✨ Add whole entity hiding function to only hide visible entities

kf_utils/dataservice/patch.py

@@ -1,7 +1,7 @@
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
 from d3b_utils.requests_retry import Session
-from kf_utils.dataservice.meta import prefix, prefix_endpoints
+from kf_utils.dataservice.meta import get_endpoint
 
 
 def send_patches(host, patches):
@@ -14,7 +14,7 @@ def send_patches(host, patches):
     """
 
     def do_patch(url, patch):
-        msg = f"Patching {url} with {patch}"
+        msg = f"Patched {url} with {patch}"
         resp = Session().patch(url, json=patch)
         if not resp.ok:
             raise Exception(f"{resp.status_code} -- {msg} -- {resp.json()}")
@@ -23,7 +23,7 @@ def do_patch(url, patch):
     with ThreadPoolExecutor() as tpex:
         futures = []
         for kfid, patch in patches.items():
-            endpoint = prefix_endpoints[prefix(kfid)]
+            endpoint = get_endpoint(kfid)
             url = f"{host}/{endpoint}/{kfid}"
             futures.append(tpex.submit(do_patch, url, patch))
         for f in as_completed(futures):
@@ -55,7 +55,7 @@ def hide_kfids(host, kfid_list, gf_acl=None):
     """
 
     def hide_function(k):
-        if prefix_endpoints[prefix(k)] == "genomic-files":
+        if get_endpoint(k) == "genomic-files":
             return {"visible": False, "acl": gf_acl or []}
         else:
             return {"visible": False}
@@ -71,3 +71,27 @@ def unhide_kfids(host, kfid_list):
     :param kfid_list: list of kfids to unhide
     """
     patch_things_with_func(host, kfid_list, lambda x: {"visible": True})
+
+
+def hide_entities(host, entities, gf_acl=None, dry_run=False):
+    """
+    Like hide_kfids but given whole entities so we can only patch the ones that
+    aren't already hidden.
+    """
+    to_hide = [e["kf_id"] for e in entities if e["visible"] is True]
+    if to_hide and not dry_run:
+        hide_kfids(host, to_hide, gf_acl)
+
+    return to_hide
+
+
+def unhide_entities(host, entities, dry_run=False):
+    """
+    Like unhide_kfids but given whole entities so we can only patch the ones that
+    aren't already visible.
+    """
+    to_show = [e["kf_id"] for e in entities if e["visible"] is False]
+    if to_show and not dry_run:
+        unhide_kfids(host, to_show)
+
+    return to_show

tests/conftest.py

@@ -1,4 +1,5 @@
 import pytest
+import requests
 from kf_utils.dataservice.delete import delete_entities
 
 DATASERVICE_URL = "http://localhost:5000"
@@ -12,3 +13,63 @@ def dataservice_setup():
     delete_entities(DATASERVICE_URL)
     yield
     delete_entities(DATASERVICE_URL)
+
+
+def create_sequencing_center():
+    kfid = "SC_11111111"
+    requests.post(
+        f"{DATASERVICE_URL}/sequencing-centers",
+        json={"kf_id": kfid, "external_id": "x", "name": "x"},
+    )
+    return kfid
+
+
+def create_study(si):
+    kfid = f"SD_{si}1111111"
+    requests.post(
+        f"{DATASERVICE_URL}/studies",
+        json={"kf_id": kfid, "external_id": f"{si}"},
+    )
+    return kfid
+
+
+def create_participant(si, pi):
+    kfid = f"PT_{si}{pi}111111"
+    requests.post(
+        f"{DATASERVICE_URL}/participants",
+        json={
+            "kf_id": kfid,
+            "study_id": f"SD_{si}1111111",
+            "external_id": f"{pi}",
+        },
+    )
+    return kfid
+
+
+def create_biospecimen(si, pi, bi):
+    kfid = f"BS_{si}{pi}{bi}11111"
+    requests.post(
+        f"{DATASERVICE_URL}/biospecimens",
+        json={
+            "kf_id": kfid,
+            "participant_id": f"PT_{si}{pi}111111",
+            "external_sample_id": f"{pi}{bi}",
+            "external_aliquot_id": f"{pi}{bi}",
+            "sequencing_center_id": "SC_11111111",
+            "analyte_type": "DNA",
+        },
+    )
+    return kfid
+
+
+def populate_data(n):
+    # Create some data in dataservice
+    kfids = []
+    kfids.append(create_sequencing_center())
+    for si in range(n):
+        kfids.append(create_study(si))
+        for pi in range(n):
+            kfids.append(create_participant(si, pi))
+            for bi in range(n):
+                kfids.append(create_biospecimen(si, pi, bi))
+    return kfids

tests/integration/test_dataservice_patch.py

@@ -0,0 +1,53 @@
+from kf_utils.dataservice.scrape import (
+    yield_entities_from_filter,
+    yield_entities_from_kfids,
+)
+from kf_utils.dataservice.patch import (
+    hide_kfids,
+    unhide_kfids,
+    hide_entities,
+    unhide_entities,
+)
+from tests.conftest import DATASERVICE_URL, populate_data
+
+
+def test_hide_unhide_kfids(dataservice_setup):
+    n = 4
+    kfids = populate_data(n)
+    to_hide = {"PT_11111111", "BS_12111111"}
+
+    hide_kfids(DATASERVICE_URL, to_hide)
+    for e in yield_entities_from_kfids(DATASERVICE_URL, kfids):
+        assert e["visible"] ^ (e["kf_id"] in to_hide)
+
+    unhide_kfids(DATASERVICE_URL, to_hide)
+    for e in yield_entities_from_kfids(DATASERVICE_URL, kfids):
+        assert e["visible"]
+
+
+def test_hide_unhide_entities(dataservice_setup):
+    n = 4
+    kfids = populate_data(n)
+    to_hide = {"PT_11111111", "BS_12111111"}
+
+    hide_kfids(DATASERVICE_URL, to_hide)
+    entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids))
+    hidden = [e for e in entities if e["kf_id"] in to_hide]
+
+    assert not hide_entities(DATASERVICE_URL, hidden)
+    entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids))
+    hidden = [e for e in entities if e["kf_id"] in to_hide]
+    for e in entities:
+        assert e["visible"] ^ (e["kf_id"] in to_hide)
+
+    assert unhide_entities(DATASERVICE_URL, hidden)
+    entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids))
+    hidden = [e for e in entities if e["kf_id"] in to_hide]
+    for e in entities:
+        assert e["visible"]
+
+    assert not unhide_entities(DATASERVICE_URL, hidden)
+    entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids))
+    hidden = [e for e in entities if e["kf_id"] in to_hide]
+    for e in entities:
+        assert e["visible"]

tests/integration/test_dataservice_scrape.py

@@ -1,56 +1,9 @@
-import requests
 from kf_utils.dataservice.scrape import (
     yield_entities,
     yield_entities_from_filter,
     yield_entities_from_kfids,
 )
-from tests.conftest import DATASERVICE_URL
-
-
-def create_sequencing_center():
-    requests.post(
-        f"{DATASERVICE_URL}/sequencing-centers",
-        json={"kf_id": "SC_11111111", "external_id": "x", "name": "x"},
-    )
-
-
-def create_study(si):
-    requests.post(
-        f"{DATASERVICE_URL}/studies",
-        json={"kf_id": f"SD_{si}1111111", "external_id": f"{si}"},
-    )
-
-
-def create_participant(si, pi):
-    requests.post(
-        f"{DATASERVICE_URL}/participants",
-        json={"kf_id": f"PT_{si}{pi}111111", "study_id": f"SD_{si}1111111", "external_id": f"{pi}"},
-    )
-
-
-def create_biospecimen(si, pi, bi):
-    requests.post(
-        f"{DATASERVICE_URL}/biospecimens",
-        json={
-            "kf_id": f"BS_{si}{pi}{bi}11111",
-            "participant_id": f"PT_{si}{pi}111111",
-            "external_sample_id": f"{pi}{bi}",
-            "external_aliquot_id": f"{pi}{bi}",
-            "sequencing_center_id": "SC_11111111",
-            "analyte_type": "DNA",
-        },
-    )
-
-
-def populate_data(n):
-    # Create some data in dataservice
-    create_sequencing_center()
-    for si in range(n):
-        create_study(si)
-        for pi in range(n):
-            create_participant(si, pi)
-            for bi in range(n):
-                create_biospecimen(si, pi, bi)
+from tests.conftest import DATASERVICE_URL, populate_data
 
 
 def test_yield_entities_from_filter(dataservice_setup):