diff --git a/kf_utils/dataservice/patch.py b/kf_utils/dataservice/patch.py index 4436814..6704059 100644 --- a/kf_utils/dataservice/patch.py +++ b/kf_utils/dataservice/patch.py @@ -1,7 +1,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from d3b_utils.requests_retry import Session -from kf_utils.dataservice.meta import prefix, prefix_endpoints +from kf_utils.dataservice.meta import get_endpoint def send_patches(host, patches): @@ -14,7 +14,7 @@ def send_patches(host, patches): """ def do_patch(url, patch): - msg = f"Patching {url} with {patch}" + msg = f"Patched {url} with {patch}" resp = Session().patch(url, json=patch) if not resp.ok: raise Exception(f"{resp.status_code} -- {msg} -- {resp.json()}") @@ -23,7 +23,7 @@ def do_patch(url, patch): with ThreadPoolExecutor() as tpex: futures = [] for kfid, patch in patches.items(): - endpoint = prefix_endpoints[prefix(kfid)] + endpoint = get_endpoint(kfid) url = f"{host}/{endpoint}/{kfid}" futures.append(tpex.submit(do_patch, url, patch)) for f in as_completed(futures): @@ -55,7 +55,7 @@ def hide_kfids(host, kfid_list, gf_acl=None): """ def hide_function(k): - if prefix_endpoints[prefix(k)] == "genomic-files": + if get_endpoint(k) == "genomic-files": return {"visible": False, "acl": gf_acl or []} else: return {"visible": False} @@ -71,3 +71,27 @@ def unhide_kfids(host, kfid_list): :param kfid_list: list of kfids to unhide """ patch_things_with_func(host, kfid_list, lambda x: {"visible": True}) + + +def hide_entities(host, entities, gf_acl=None, dry_run=False): + """ + Like hide_kfids but given whole entities so we can only patch the ones that + aren't already hidden. + """ + to_hide = [e["kf_id"] for e in entities if e["visible"] is True] + if to_hide and not dry_run: + hide_kfids(host, to_hide, gf_acl) + + return to_hide + + +def unhide_entities(host, entities, dry_run=False): + """ + Like unhide_kfids but given whole entities so we can only patch the ones that + aren't already visible. + """ + to_show = [e["kf_id"] for e in entities if e["visible"] is False] + if to_show and not dry_run: + unhide_kfids(host, to_show) + + return to_show diff --git a/tests/conftest.py b/tests/conftest.py index 5b6f88d..bcdd24d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import pytest +import requests from kf_utils.dataservice.delete import delete_entities DATASERVICE_URL = "http://localhost:5000" @@ -12,3 +13,63 @@ def dataservice_setup(): delete_entities(DATASERVICE_URL) yield delete_entities(DATASERVICE_URL) + + +def create_sequencing_center(): + kfid = "SC_11111111" + requests.post( + f"{DATASERVICE_URL}/sequencing-centers", + json={"kf_id": kfid, "external_id": "x", "name": "x"}, + ) + return kfid + + +def create_study(si): + kfid = f"SD_{si}1111111" + requests.post( + f"{DATASERVICE_URL}/studies", + json={"kf_id": kfid, "external_id": f"{si}"}, + ) + return kfid + + +def create_participant(si, pi): + kfid = f"PT_{si}{pi}111111" + requests.post( + f"{DATASERVICE_URL}/participants", + json={ + "kf_id": kfid, + "study_id": f"SD_{si}1111111", + "external_id": f"{pi}", + }, + ) + return kfid + + +def create_biospecimen(si, pi, bi): + kfid = f"BS_{si}{pi}{bi}11111" + requests.post( + f"{DATASERVICE_URL}/biospecimens", + json={ + "kf_id": kfid, + "participant_id": f"PT_{si}{pi}111111", + "external_sample_id": f"{pi}{bi}", + "external_aliquot_id": f"{pi}{bi}", + "sequencing_center_id": "SC_11111111", + "analyte_type": "DNA", + }, + ) + return kfid + + +def populate_data(n): + # Create some data in dataservice + kfids = [] + kfids.append(create_sequencing_center()) + for si in range(n): + kfids.append(create_study(si)) + for pi in range(n): + kfids.append(create_participant(si, pi)) + for bi in range(n): + kfids.append(create_biospecimen(si, pi, bi)) + return kfids diff --git a/tests/integration/test_dataservice_patch.py b/tests/integration/test_dataservice_patch.py new file mode 100644 index 0000000..a3ab95e --- /dev/null +++ b/tests/integration/test_dataservice_patch.py @@ -0,0 +1,53 @@ +from kf_utils.dataservice.scrape import ( + yield_entities_from_filter, + yield_entities_from_kfids, +) +from kf_utils.dataservice.patch import ( + hide_kfids, + unhide_kfids, + hide_entities, + unhide_entities, +) +from tests.conftest import DATASERVICE_URL, populate_data + + +def test_hide_unhide_kfids(dataservice_setup): + n = 4 + kfids = populate_data(n) + to_hide = {"PT_11111111", "BS_12111111"} + + hide_kfids(DATASERVICE_URL, to_hide) + for e in yield_entities_from_kfids(DATASERVICE_URL, kfids): + assert e["visible"] ^ (e["kf_id"] in to_hide) + + unhide_kfids(DATASERVICE_URL, to_hide) + for e in yield_entities_from_kfids(DATASERVICE_URL, kfids): + assert e["visible"] + + +def test_hide_unhide_entities(dataservice_setup): + n = 4 + kfids = populate_data(n) + to_hide = {"PT_11111111", "BS_12111111"} + + hide_kfids(DATASERVICE_URL, to_hide) + entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids)) + hidden = [e for e in entities if e["kf_id"] in to_hide] + + assert not hide_entities(DATASERVICE_URL, hidden) + entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids)) + hidden = [e for e in entities if e["kf_id"] in to_hide] + for e in entities: + assert e["visible"] ^ (e["kf_id"] in to_hide) + + assert unhide_entities(DATASERVICE_URL, hidden) + entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids)) + hidden = [e for e in entities if e["kf_id"] in to_hide] + for e in entities: + assert e["visible"] + + assert not unhide_entities(DATASERVICE_URL, hidden) + entities = list(yield_entities_from_kfids(DATASERVICE_URL, kfids)) + hidden = [e for e in entities if e["kf_id"] in to_hide] + for e in entities: + assert e["visible"] diff --git a/tests/integration/test_dataservice_scrape.py b/tests/integration/test_dataservice_scrape.py index c877c42..fa32a94 100644 --- a/tests/integration/test_dataservice_scrape.py +++ b/tests/integration/test_dataservice_scrape.py @@ -1,56 +1,9 @@ -import requests from kf_utils.dataservice.scrape import ( yield_entities, yield_entities_from_filter, yield_entities_from_kfids, ) -from tests.conftest import DATASERVICE_URL - - -def create_sequencing_center(): - requests.post( - f"{DATASERVICE_URL}/sequencing-centers", - json={"kf_id": "SC_11111111", "external_id": "x", "name": "x"}, - ) - - -def create_study(si): - requests.post( - f"{DATASERVICE_URL}/studies", - json={"kf_id": f"SD_{si}1111111", "external_id": f"{si}"}, - ) - - -def create_participant(si, pi): - requests.post( - f"{DATASERVICE_URL}/participants", - json={"kf_id": f"PT_{si}{pi}111111", "study_id": f"SD_{si}1111111", "external_id": f"{pi}"}, - ) - - -def create_biospecimen(si, pi, bi): - requests.post( - f"{DATASERVICE_URL}/biospecimens", - json={ - "kf_id": f"BS_{si}{pi}{bi}11111", - "participant_id": f"PT_{si}{pi}111111", - "external_sample_id": f"{pi}{bi}", - "external_aliquot_id": f"{pi}{bi}", - "sequencing_center_id": "SC_11111111", - "analyte_type": "DNA", - }, - ) - - -def populate_data(n): - # Create some data in dataservice - create_sequencing_center() - for si in range(n): - create_study(si) - for pi in range(n): - create_participant(si, pi) - for bi in range(n): - create_biospecimen(si, pi, bi) +from tests.conftest import DATASERVICE_URL, populate_data def test_yield_entities_from_filter(dataservice_setup):