✨ Add function for reporting S3 and DS GF entries

README.md

@@ -8,7 +8,7 @@ Python >= 3.6
 
 Using pip
 
-`pip install -e git+https://github.com/kids-first/kf-utils-python.git#egg=kf_utils`
+`pip install git+https://github.com/kids-first/kf-utils-python.git`
 
 ## Included so far
 
@@ -24,13 +24,17 @@ of the entities from all of the pages for a given query.
 
 ```Python
 from kf_utils.dataservice.scrape import *
+```
 
+```Python
 # Yield all entities from the given endpoint matching the given query
 for e in yield_entities(
     kf_api_url, "participants", {"study_id": "SD_12345678"}
 ):
   ...
+```
 
+```Python
 # Like yield_entities but just yields the kfids
 for kfid in yield_kfids(
     kf_api_url, "participants", {"study_id": "SD_12345678"}
@@ -51,7 +55,9 @@ change their descendant entities.
 
 ```Python
 from kf_utils.dataservice.descendants import *
+```
 
+```Python
 api_url = "https://kf-api-dataservice.kidsfirstdrc.org"
 db_url = f"postgres://{USER_NAME}:{PASSWORD}@kf-dataservice-api-prd-2019-9-11.c3siovbugjym.us-east-1.rds.amazonaws.com:5432/kfpostgresprd"
 
@@ -121,7 +127,9 @@ Streamline patching the dataservice quickly.
 
 ```Python
 from kf_utils.dataservice.patch import *
+```
 
+```Python
 host = "http://localhost:5000"
 
 # Patch the given KFIDs with the given changes
@@ -155,13 +163,36 @@ hide_kfids(host, ["PT_12345678", "GF_99999999"], gf_acl=[])
 unhide_kfids(host, ["PT_12345678", "BS_99999999"])
 ```
 
+#### [dataservice/reporting.py](kf_utils/dataservice/reporting.py) - Generating reports
+
+```Python
+from kf_utils.dataservice.reporting import *
+```
+
+```Python
+# Generate a report on which S3 files have been loaded into the data service
+# and which loaded files no longer exist on S3 by merging the details together.
+
+ds_url = "https://kf-api-dataservice.kidsfirstdrc.org"
+study_kfid = "SD_12345678"
+study_bucket = "kf-study-us-east-1-prd-sd-12345678"
+exclude_s3_keypaths = ("path/with_files/I_don't_care_about", "another/one")
+
+# This can take a while to run because of API limitations.
+output = merge_s3_and_kf_gfs(
+    ds_url, study_kfid, study_bucket, exclude_s3_keypaths
+)
+```
+
 ### dbGaP
 
 #### [dbgap/release.py](kf_utils/dbgap/release.py) - dbGaP release XML scraping
 
 ```Python
 from kf_utils.dbgap.release import get_latest_released_sample_status
+```
 
+```Python
 # Get the sample status data for the latest "released" version of this study
 versioned_accession, study_data = get_latest_released_sample_status("phs001138")
 ```

kf_utils/dataservice/reporting.py

@@ -0,0 +1,75 @@
+from collections import defaultdict
+
+from d3b_utils.aws_bucket_contents import fetch_bucket_obj_info
+from kf_utils.dataservice.scrape import yield_entities
+
+
+def merge_s3_and_kf_gfs(
+    ds_url, study_kfid, study_bucket, exclude_s3_keypaths=None
+):
+    """Return file data from S3 and the Kids First dataservice merged together
+    on external_id to see which S3 files have been loaded into the data service
+    and which loaded files no longer exist.
+
+    Note: You must be able to query both S3 and the dataservice
+        (VPN + chopaws if running locally)
+
+    :param study_kfid: Dataservice KFID of the study
+    :type study_kfid: string
+    :param study_bucket: Amazon S3 bucket containing study files
+    :type study_bucket: string
+    :param exclude_s3_keypaths: S3 paths starting with these strings will be
+        excluded, optional, defaults to None
+    :type exclude_s3_keypaths: string, iterable
+    :return: list of dicts
+    """
+    # Files from the dataservice
+    # We use the API because direct DB queries won't give us the gen3 fields
+    kf = {
+        e["external_id"]: {
+            f"kf_{k.lower()}": v
+            for k, v in e.items()
+            if k not in ["_links", "access_urls", "urls"]
+        }
+        for e in yield_entities(
+            ds_url,
+            "genomic-files",
+            {"study_id": study_kfid},
+            show_progress=True,
+        )
+    }
+
+    # Files from S3
+    s3 = {
+        "s3://"
+        + o["Bucket"]
+        + "/"
+        + o["Key"]: {f"s3_{k.lower()}": v for k, v in o.items()}
+        for o in fetch_bucket_obj_info(
+            study_bucket,
+            drop_folders=True,
+        )
+    }
+
+    # Sadly it's muuuuch harder to exclude paths on the S3 request side because
+    # the S3 API doesn't support it. So we're stuck for now waiting for
+    # potentially thousands of pagination requests that we don't care about,
+    # and then we remove them here.
+    if exclude_s3_keypaths:
+        if isinstance(exclude_s3_keypaths, str):
+            exclude_s3_keypaths = (exclude_s3_keypaths,)
+        elif exclude_s3_keypaths is not None:
+            exclude_s3_keypaths = tuple(exclude_s3_keypaths)
+
+        s3 = {
+            k: v
+            for k, v in s3.items()
+            if not v["s3_key"].startswith(exclude_s3_keypaths)
+        }
+
+    # Merge them together
+    s3kf = defaultdict(dict, s3)
+    for k, v in kf.items():
+        s3kf[k].update(v)
+
+    return list(s3kf.values())