From e83b0d5426a5fc392817617472852016b00a84aa Mon Sep 17 00:00:00 2001 From: Avi Kelman Date: Wed, 17 Mar 2021 15:27:59 -0600 Subject: [PATCH 1/2] :rotating_light: Lint files --- kf_utils/dataservice/descendants.py | 36 +++++++++++++++-------------- kf_utils/dataservice/patch.py | 8 +++---- kf_utils/dbgap/release.py | 10 ++++---- 3 files changed, 28 insertions(+), 26 deletions(-) diff --git a/kf_utils/dataservice/descendants.py b/kf_utils/dataservice/descendants.py index ef4ef60..dd0cce8 100644 --- a/kf_utils/dataservice/descendants.py +++ b/kf_utils/dataservice/descendants.py @@ -9,7 +9,7 @@ import psycopg2.extras from kf_utils.dataservice.patch import hide_kfids, unhide_kfids -from kf_utils.dataservice.scrape import yield_entities, yield_kfids +from kf_utils.dataservice.scrape import yield_entities def _accumulate(func, *args, **kwargs): @@ -24,11 +24,9 @@ def _accumulate(func, *args, **kwargs): # We need to specially handle getting to families from studies, because # the database layout does not match the logical data arrangement, so # just add a stub here for family. - ("family", None, None) - ], - "family": [ - ("participant", "kf_id", "family_id") + ("family", None, None), ], + "family": [("participant", "kf_id", "family_id")], "participant": [ ("family_relationship", "kf_id", "participant1_id"), ("family_relationship", "kf_id", "participant2_id"), @@ -182,9 +180,7 @@ def _find_gfs_with_extra_contributors_with_http_api( ] for f in as_completed(futures): for bg in f.result(): - gf_kfids.add( - bg["_links"]["genomic_file"].rsplit("/", 1)[1] - ) + gf_kfids.add(bg["_links"]["genomic_file"].rsplit("/", 1)[1]) else: gf_kfids = set(gf_kfids) @@ -343,7 +339,7 @@ def _inner(parent_type, parent_kfids, descendants): # special case for getting to families from studies if parent_type == "study" and child_type == "family": query = ( - f"select distinct family.* from family join participant" + "select distinct family.* from family join participant" " on participant.family_id = family.kf_id join study on" " participant.study_id = study.kf_id where study.kf_id " "in %s" @@ -402,15 +398,13 @@ def find_descendants_by_filter( filter, ignore_gfs_with_hidden_external_contribs, kfids_only=True, - db_url=None + db_url=None, ): """ Similar to find_descendants_by_kfids but starts with an API endpoint filter instead of a list of endpoint KFIDs. """ - things = list( - yield_entities(api_url, endpoint, filter, show_progress=True) - ) + things = list(yield_entities(api_url, endpoint, filter, show_progress=True)) if kfids_only: things = [t["kf_id"] for t in things] descendants = find_descendants_by_kfids( @@ -423,7 +417,9 @@ def find_descendants_by_filter( return descendants -def hide_descendants_by_filter(api_url, endpoint, filter, gf_acl=None, db_url=None): +def hide_descendants_by_filter( + api_url, endpoint, filter, gf_acl=None, db_url=None +): """ Be aware that this and unhide_descendants_by_filter are not symmetrical. @@ -432,7 +428,9 @@ def hide_descendants_by_filter(api_url, endpoint, filter, gf_acl=None, db_url=No If you anticipate needing symmetrical behavior, keep a record of what you change. """ - desc = find_descendants_by_filter(api_url, endpoint, filter, False, db_url=db_url) + desc = find_descendants_by_filter( + api_url, endpoint, filter, False, db_url=db_url + ) for v in desc.values(): hide_kfids(api_url, v, gf_acl) @@ -446,12 +444,16 @@ def unhide_descendants_by_filter(api_url, endpoint, filter, db_url=None): If you anticipate needing symmetrical behavior, keep a record of what you change. """ - desc = find_descendants_by_filter(api_url, endpoint, filter, True, db_url=db_url) + desc = find_descendants_by_filter( + api_url, endpoint, filter, True, db_url=db_url + ) for v in desc.values(): unhide_kfids(api_url, v) -def hide_descendants_by_kfids(api_url, endpoint, kfids, gf_acl=None, db_url=None): +def hide_descendants_by_kfids( + api_url, endpoint, kfids, gf_acl=None, db_url=None +): """ Be aware that this and unhide_descendants_by_kfids are not symmetrical. diff --git a/kf_utils/dataservice/patch.py b/kf_utils/dataservice/patch.py index b187ba2..4436814 100644 --- a/kf_utils/dataservice/patch.py +++ b/kf_utils/dataservice/patch.py @@ -12,13 +12,12 @@ def send_patches(host, patches): :param patches: dict mapping KFIDs to patch dicts :raises Exception: if server doesn't respond OK """ + def do_patch(url, patch): msg = f"Patching {url} with {patch}" resp = Session().patch(url, json=patch) if not resp.ok: - raise Exception( - f"{resp.status_code} -- {msg} -- {resp.json()}" - ) + raise Exception(f"{resp.status_code} -- {msg} -- {resp.json()}") return msg with ThreadPoolExecutor() as tpex: @@ -41,8 +40,7 @@ def patch_things_with_func(host, things, patch_func): a dict to patch with based on the value or contents """ patches = { - t["kf_id"] if isinstance(t, dict) else t: patch_func(t) - for t in things + t["kf_id"] if isinstance(t, dict) else t: patch_func(t) for t in things } send_patches(host, patches) diff --git a/kf_utils/dbgap/release.py b/kf_utils/dbgap/release.py index a00de27..85dea29 100644 --- a/kf_utils/dbgap/release.py +++ b/kf_utils/dbgap/release.py @@ -1,6 +1,7 @@ import xmltodict from d3b_utils.requests_retry import Session -from xml.etree import ElementTree +from xml.etree import ElementTree + # from defusedxml import ElementTree as DefusedET # from defusedxml.common import DefusedXmlException @@ -27,8 +28,7 @@ def get_latest_sample_status(phs_id, required_status="released"): if data.status_code != 200: tried[phs_string] = f"status {data.status_code}" raise Exception( - f"Request for study {phs_id} failed." - f" - Tried: {tried}" + f"Request for study {phs_id} failed." f" - Tried: {tried}" ) # try: @@ -45,7 +45,9 @@ def get_latest_sample_status(phs_id, required_status="released"): accession = study["@accession"] status = study["@registration_status"] - if (required_status is None) or (status.lower() == required_status.lower()): + if (required_status is None) or ( + status.lower() == required_status.lower() + ): break else: # try previous version From ead3c15e63627fbcef503125a647f8f7fcf36535 Mon Sep 17 00:00:00 2001 From: Avi Kelman Date: Wed, 17 Mar 2021 15:10:07 -0600 Subject: [PATCH 2/2] :construction_worker: Add CI config --- .circleci/config.yml | 73 ++++++++++++++++++++++++++++++++++++++++++++ dev-requirements.txt | 2 ++ 2 files changed, 75 insertions(+) create mode 100644 .circleci/config.yml create mode 100644 dev-requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..817c564 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,73 @@ +version: 2 +jobs: + lint: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - run: + name: black + command: | + pip install black + black --check --line-length 80 kf_utils + - run: + name: flake8 + command: | + pip install flake8 + flake8 --ignore=E501,W503,E203 kf_utils + + + test: + docker: + - image: circleci/python:3.7 + - image: postgres:11.1 + environment: + POSTGRES_USER: "postgres" + POSTGRES_DB: "test" + PG_USER: "postgres" + PG_NAME: "test" + + steps: + - checkout + + - run: + name: install dockerize + command: wget https://github.com/d3b-center/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz + environment: + DOCKERIZE_VERSION: v0.6.1 + + - run: + name: start local dataservice + command: | + git clone --depth 1 https://github.com/kids-first/kf-api-dataservice.git + cd kf-api-dataservice + python3 -m venv dataservice_venv + source dataservice_venv/bin/activate + pip install -r requirements.txt + pip install -e . + flask db upgrade + flask db migrate + flask run + background: true + environment: + FLASK_APP: "manage" + PG_USER: "postgres" + PG_NAME: "test" + + - run: + name: run tests + command: | + python3 -m venv client_venv + source client_venv/bin/activate + pip install -r dev-requirements.txt + pip install . + dockerize -wait http://localhost:5000 -timeout 5m + # pytest tests + + +workflows: + version: 2 + build: + jobs: + - lint + - test diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..26b77f6 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest