👷 Add CI config

.circleci/config.yml

@@ -0,0 +1,73 @@
+version: 2
+jobs:
+  lint:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - run:
+          name: black
+          command: |
+            pip install black
+            black --check --line-length 80 kf_utils
+      - run:
+          name: flake8
+          command: |
+            pip install flake8
+            flake8 --ignore=E501,W503,E203 kf_utils
+
+
+  test:
+    docker:
+      - image: circleci/python:3.7
+      - image: postgres:11.1
+        environment:
+          POSTGRES_USER: "postgres"
+          POSTGRES_DB: "test"
+          PG_USER: "postgres"
+          PG_NAME: "test"
+
+    steps:
+      - checkout
+
+      - run:
+          name: install dockerize
+          command: wget https://github.com/d3b-center/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz
+          environment:
+            DOCKERIZE_VERSION: v0.6.1
+
+      - run:
+          name: start local dataservice
+          command: |
+            git clone --depth 1 https://github.com/kids-first/kf-api-dataservice.git
+            cd kf-api-dataservice
+            python3 -m venv dataservice_venv
+            source dataservice_venv/bin/activate
+            pip install -r requirements.txt
+            pip install -e .
+            flask db upgrade
+            flask db migrate
+            flask run
+          background: true
+          environment:
+            FLASK_APP: "manage"
+            PG_USER: "postgres"
+            PG_NAME: "test"
+
+      - run:
+          name: run tests
+          command: |
+            python3 -m venv client_venv
+            source client_venv/bin/activate
+            pip install -r dev-requirements.txt
+            pip install .
+            dockerize -wait http://localhost:5000 -timeout 5m
+            # pytest tests
+
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - lint
+      - test

dev-requirements.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest

kf_utils/dataservice/descendants.py

@@ -9,7 +9,7 @@
 import psycopg2.extras
 
 from kf_utils.dataservice.patch import hide_kfids, unhide_kfids
-from kf_utils.dataservice.scrape import yield_entities, yield_kfids
+from kf_utils.dataservice.scrape import yield_entities
 
 
 def _accumulate(func, *args, **kwargs):
@@ -24,11 +24,9 @@ def _accumulate(func, *args, **kwargs):
         # We need to specially handle getting to families from studies, because
         # the database layout does not match the logical data arrangement, so
         # just add a stub here for family.
-        ("family", None, None)
-    ],
-    "family": [
-        ("participant", "kf_id", "family_id")
+        ("family", None, None),
     ],
+    "family": [("participant", "kf_id", "family_id")],
     "participant": [
         ("family_relationship", "kf_id", "participant1_id"),
         ("family_relationship", "kf_id", "participant2_id"),
@@ -182,9 +180,7 @@ def _find_gfs_with_extra_contributors_with_http_api(
             ]
             for f in as_completed(futures):
                 for bg in f.result():
-                    gf_kfids.add(
-                        bg["_links"]["genomic_file"].rsplit("/", 1)[1]
-                    )
+                    gf_kfids.add(bg["_links"]["genomic_file"].rsplit("/", 1)[1])
     else:
         gf_kfids = set(gf_kfids)
 
@@ -343,7 +339,7 @@ def _inner(parent_type, parent_kfids, descendants):
                 # special case for getting to families from studies
                 if parent_type == "study" and child_type == "family":
                     query = (
-                        f"select distinct family.* from family join participant"
+                        "select distinct family.* from family join participant"
                         " on participant.family_id = family.kf_id join study on"
                         " participant.study_id = study.kf_id where study.kf_id "
                         "in %s"
@@ -402,15 +398,13 @@ def find_descendants_by_filter(
     filter,
     ignore_gfs_with_hidden_external_contribs,
     kfids_only=True,
-    db_url=None
+    db_url=None,
 ):
     """
     Similar to find_descendants_by_kfids but starts with an API endpoint filter
     instead of a list of endpoint KFIDs.
     """
-    things = list(
-        yield_entities(api_url, endpoint, filter, show_progress=True)
-    )
+    things = list(yield_entities(api_url, endpoint, filter, show_progress=True))
     if kfids_only:
         things = [t["kf_id"] for t in things]
     descendants = find_descendants_by_kfids(
@@ -423,7 +417,9 @@ def find_descendants_by_filter(
     return descendants
 
 
-def hide_descendants_by_filter(api_url, endpoint, filter, gf_acl=None, db_url=None):
+def hide_descendants_by_filter(
+    api_url, endpoint, filter, gf_acl=None, db_url=None
+):
     """
     Be aware that this and unhide_descendants_by_filter are not symmetrical.
 
@@ -432,7 +428,9 @@ def hide_descendants_by_filter(api_url, endpoint, filter, gf_acl=None, db_url=No
     If you anticipate needing symmetrical behavior, keep a record of what you
     change.
     """
-    desc = find_descendants_by_filter(api_url, endpoint, filter, False, db_url=db_url)
+    desc = find_descendants_by_filter(
+        api_url, endpoint, filter, False, db_url=db_url
+    )
     for v in desc.values():
         hide_kfids(api_url, v, gf_acl)
 
@@ -446,12 +444,16 @@ def unhide_descendants_by_filter(api_url, endpoint, filter, db_url=None):
     If you anticipate needing symmetrical behavior, keep a record of what you
     change.
     """
-    desc = find_descendants_by_filter(api_url, endpoint, filter, True, db_url=db_url)
+    desc = find_descendants_by_filter(
+        api_url, endpoint, filter, True, db_url=db_url
+    )
     for v in desc.values():
         unhide_kfids(api_url, v)
 
 
-def hide_descendants_by_kfids(api_url, endpoint, kfids, gf_acl=None, db_url=None):
+def hide_descendants_by_kfids(
+    api_url, endpoint, kfids, gf_acl=None, db_url=None
+):
     """
     Be aware that this and unhide_descendants_by_kfids are not symmetrical.
 

kf_utils/dataservice/patch.py

@@ -12,13 +12,12 @@ def send_patches(host, patches):
     :param patches: dict mapping KFIDs to patch dicts
     :raises Exception: if server doesn't respond OK
     """
+
     def do_patch(url, patch):
         msg = f"Patching {url} with {patch}"
         resp = Session().patch(url, json=patch)
         if not resp.ok:
-            raise Exception(
-                f"{resp.status_code} -- {msg} -- {resp.json()}"
-            )
+            raise Exception(f"{resp.status_code} -- {msg} -- {resp.json()}")
         return msg
 
     with ThreadPoolExecutor() as tpex:
@@ -41,8 +40,7 @@ def patch_things_with_func(host, things, patch_func):
         a dict to patch with based on the value or contents
     """
     patches = {
-        t["kf_id"] if isinstance(t, dict) else t: patch_func(t)
-        for t in things
+        t["kf_id"] if isinstance(t, dict) else t: patch_func(t) for t in things
     }
     send_patches(host, patches)
 

kf_utils/dbgap/release.py

@@ -1,6 +1,7 @@
 import xmltodict
 from d3b_utils.requests_retry import Session
-from xml.etree import ElementTree 
+from xml.etree import ElementTree
+
 # from defusedxml import ElementTree as DefusedET
 # from defusedxml.common import DefusedXmlException
 
@@ -27,8 +28,7 @@ def get_latest_sample_status(phs_id, required_status="released"):
         if data.status_code != 200:
             tried[phs_string] = f"status {data.status_code}"
             raise Exception(
-                f"Request for study {phs_id} failed."
-                f" - Tried: {tried}"
+                f"Request for study {phs_id} failed." f" - Tried: {tried}"
             )
 
         # try:
@@ -45,7 +45,9 @@ def get_latest_sample_status(phs_id, required_status="released"):
         accession = study["@accession"]
         status = study["@registration_status"]
 
-        if (required_status is None) or (status.lower() == required_status.lower()):
+        if (required_status is None) or (
+            status.lower() == required_status.lower()
+        ):
             break
         else:
             # try previous version