From 32a2549867ebd533d9c2e7f8cca5671022467e8c Mon Sep 17 00:00:00 2001 From: sprenger Date: Fri, 19 Mar 2021 23:45:09 +0100 Subject: [PATCH 1/7] Add `import_metadata` functionality --- redcap/project.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++ redcap/request.py | 5 +++++ 2 files changed, 61 insertions(+) diff --git a/redcap/project.py b/redcap/project.py index 3cbf54c..cd7c5a9 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -615,6 +615,62 @@ def import_records( raise RedcapError(str(response)) return response + def import_metadata( + self, + to_import, + format="json", + return_format="json", + date_format="YMD" + ): + """ + Import metadata (DataDict) into the RedCap Project + + Parameters + ---------- + to_import : array of dicts, csv/xml string, ``pandas.DataFrame`` + :note: + If you pass a csv or xml string, you should use the + ``format`` parameter appropriately. + format : ('json'), 'xml', 'csv' + Format of incoming data. By default, to_import will be json-encoded + return_format : ('json'), 'csv', 'xml' + Response format. By default, response will be json-decoded. + date_format : ('YMD'), 'DMY', 'MDY' + Describes the formatting of dates. By default, date strings + are formatted as 'YYYY-MM-DD' corresponding to 'YMD'. If date + strings are formatted as 'MM/DD/YYYY' set this parameter as + 'MDY' and if formatted as 'DD/MM/YYYY' set as 'DMY'. No + other formattings are allowed. + + Returns + ------- + response : dict, str + response from REDCap API, json-decoded if ``return_format`` == ``'json'`` + If successful, the number of imported fields + """ + payload = self.__basepl("metadata") + # pylint: disable=comparison-with-callable + if hasattr(to_import, "to_csv"): + # We'll assume it's a df + buf = StringIO() + to_import.to_csv(buf, index=False) + payload["data"] = buf.getvalue() + buf.close() + format = "csv" + elif format == "json": + payload["data"] = json.dumps(to_import, separators=(",", ":")) + else: + # don't do anything to csv/xml + payload["data"] = to_import + # pylint: enable=comparison-with-callable + payload["format"] = format + payload["returnFormat"] = return_format + payload["dateFormat"] = date_format + response = self._call_api(payload, "imp_metadata")[0] + if "error" in str(response): + raise RedcapError(str(response)) + return response + def export_file(self, record, field, event=None, return_format="json"): """ Export the contents of a file stored for a particular record diff --git a/redcap/request.py b/redcap/request.py index 3985791..53efe29 100644 --- a/redcap/request.py +++ b/redcap/request.py @@ -82,6 +82,11 @@ def validate(self): "record", "Importing record but content is not record", ), + "imp_metadata": ( + ["type", "data", "format"], + "metadata", + "Importing record but content is not record", + ), "metadata": ( ["format"], "metadata", From ae676e61f512c7496e6cd71c97655b2676a8d1bc Mon Sep 17 00:00:00 2001 From: sprenger Date: Mon, 22 Mar 2021 15:03:25 +0100 Subject: [PATCH 2/7] add import metadata test --- test/test_project.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/test_project.py b/test/test_project.py index 1d1f53f..54a465d 100644 --- a/test/test_project.py +++ b/test/test_project.py @@ -405,6 +405,17 @@ def test_import_exception(self): exc = assert_context.exception self.assertIn("error", exc.args[0]) + @responses.activate + def test_import_metadata(self): + "Test metadata import" + self.add_normalproject_response() + data = self.reg_proj.export_metadata() + response = self.reg_proj.import_metadata(data) + for field_dict in response: + for key in ["field_name", "field_label", "form_name", "arm_num", "name"]: + self.assertIn(key, field_dict) + self.assertNotIn("error", response) + @staticmethod def is_good_csv(csv_string): "Helper to test csv strings" From 1820956124b2b2e5042943bc17a84e5fdbbf58f2 Mon Sep 17 00:00:00 2001 From: sprenger Date: Wed, 24 Mar 2021 15:23:33 +0100 Subject: [PATCH 3/7] Add `_initialize_import_payload` utility method --- redcap/project.py | 71 ++++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/redcap/project.py b/redcap/project.py index cd7c5a9..62e6573 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -585,27 +585,9 @@ def import_records( response : dict, str response from REDCap API, json-decoded if ``return_format`` == ``'json'`` """ - payload = self.__basepl("record") - # pylint: disable=comparison-with-callable - if hasattr(to_import, "to_csv"): - # We'll assume it's a df - buf = StringIO() - if self.is_longitudinal(): - csv_kwargs = {"index_label": [self.def_field, "redcap_event_name"]} - else: - csv_kwargs = {"index_label": self.def_field} - to_import.to_csv(buf, **csv_kwargs) - payload["data"] = buf.getvalue() - buf.close() - format = "csv" - elif format == "json": - payload["data"] = json.dumps(to_import, separators=(",", ":")) - else: - # don't do anything to csv/xml - payload["data"] = to_import - # pylint: enable=comparison-with-callable + payload = self._initialize_import_payload(to_import, format, 'record') + payload["overwriteBehavior"] = overwrite - payload["format"] = format payload["returnFormat"] = return_format payload["returnContent"] = return_content payload["dateFormat"] = date_format @@ -648,12 +630,49 @@ def import_metadata( response from REDCap API, json-decoded if ``return_format`` == ``'json'`` If successful, the number of imported fields """ - payload = self.__basepl("metadata") + payload = self._initialize_import_payload(to_import, format, "metadata") + payload["returnFormat"] = return_format + payload["dateFormat"] = date_format + response = self._call_api(payload, "imp_metadata")[0] + if "error" in str(response): + raise RedcapError(str(response)) + return response + + def _initialize_import_payload(self, to_import, format, data_type): + """ + Standardize the data to be imported and add it to the payload + + Parameters + ---------- + to_import : array of dicts, csv/xml string, ``pandas.DataFrame`` + :note: + If you pass a csv or xml string, you should use the + ``format`` parameter appropriately. + format : ('json'), 'xml', 'csv' + Format of incoming data. By default, to_import will be json-encoded + data_type: 'record', 'metadata' + The kind of data that are imported + + Returns + ------- + payload : (dict, str) + The initialized payload dictionary and updated format + """ + + payload = self.__basepl(data_type) # pylint: disable=comparison-with-callable if hasattr(to_import, "to_csv"): # We'll assume it's a df buf = StringIO() - to_import.to_csv(buf, index=False) + if data_type == "record": + if self.is_longitudinal(): + csv_kwargs = {"index_label": [self.def_field, + "redcap_event_name"]} + else: + csv_kwargs = {"index_label": self.def_field} + elif data_type == "metadata": + csv_kwargs = {"index": False} + to_import.to_csv(buf, **csv_kwargs) payload["data"] = buf.getvalue() buf.close() format = "csv" @@ -663,13 +682,9 @@ def import_metadata( # don't do anything to csv/xml payload["data"] = to_import # pylint: enable=comparison-with-callable + payload["format"] = format - payload["returnFormat"] = return_format - payload["dateFormat"] = date_format - response = self._call_api(payload, "imp_metadata")[0] - if "error" in str(response): - raise RedcapError(str(response)) - return response + return payload def export_file(self, record, field, event=None, return_format="json"): """ From c5fe5c24a5988fde47cf54c3b1f9cc8f8c281130 Mon Sep 17 00:00:00 2001 From: sprenger Date: Wed, 24 Mar 2021 17:33:40 +0100 Subject: [PATCH 4/7] Follow Black format requirements --- redcap/project.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/redcap/project.py b/redcap/project.py index 62e6573..b8e0119 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -585,7 +585,7 @@ def import_records( response : dict, str response from REDCap API, json-decoded if ``return_format`` == ``'json'`` """ - payload = self._initialize_import_payload(to_import, format, 'record') + payload = self._initialize_import_payload(to_import, format, "record") payload["overwriteBehavior"] = overwrite payload["returnFormat"] = return_format @@ -598,11 +598,7 @@ def import_records( return response def import_metadata( - self, - to_import, - format="json", - return_format="json", - date_format="YMD" + self, to_import, format="json", return_format="json", date_format="YMD" ): """ Import metadata (DataDict) into the RedCap Project @@ -666,8 +662,7 @@ def _initialize_import_payload(self, to_import, format, data_type): buf = StringIO() if data_type == "record": if self.is_longitudinal(): - csv_kwargs = {"index_label": [self.def_field, - "redcap_event_name"]} + csv_kwargs = {"index_label": [self.def_field, "redcap_event_name"]} else: csv_kwargs = {"index_label": self.def_field} elif data_type == "metadata": From 979ad5aff2bce5f26e3d992e74a91d21e4979d04 Mon Sep 17 00:00:00 2001 From: sprenger Date: Wed, 24 Mar 2021 17:36:15 +0100 Subject: [PATCH 5/7] List new method in documentation --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 030756b..dbca0f0 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,7 @@ Currently, these API calls are available: - Export Records - Export Metadata +- Import Metadata - Delete Records - Import Records - Export File From 5a3c80b6efe9b4effe8168fdbaf1aa2788d9afa8 Mon Sep 17 00:00:00 2001 From: sprenger Date: Thu, 25 Mar 2021 11:13:23 +0100 Subject: [PATCH 6/7] Pylint: Ignore too many lines in project.py --- redcap/project.py | 1 + 1 file changed, 1 insertion(+) diff --git a/redcap/project.py b/redcap/project.py index b8e0119..c3606d5 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -17,6 +17,7 @@ __license__ = "MIT" __copyright__ = "2014, Vanderbilt University" +# pylint: disable=too-many-lines # pylint: disable=too-many-instance-attributes # pylint: disable=too-many-arguments # pylint: disable=too-many-public-methods From 81792e79d9e072243df4d049dfdf31ec9aee4221 Mon Sep 17 00:00:00 2001 From: sprenger Date: Thu, 25 Mar 2021 11:22:02 +0100 Subject: [PATCH 7/7] Add test that is failing on test server --- test/test_project.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_project.py b/test/test_project.py index 54a465d..caa0ace 100644 --- a/test/test_project.py +++ b/test/test_project.py @@ -416,6 +416,18 @@ def test_import_metadata(self): self.assertIn(key, field_dict) self.assertNotIn("error", response) + @unittest.skip("Fails on test server for unknown reason") + @responses.activate + def test_import_reduced_metadata(self): + "Test import of a reduced set of metadata" + self.add_normalproject_response() + original_data = self.reg_proj.export_metadata() + # reducing the metadata + reduced_data = original_data[0:1] + imported_data = self.reg_proj.import_metadata(reduced_data) + + self.assertEqual(len(imported_data), len(reduced_data)) + @staticmethod def is_good_csv(csv_string): "Helper to test csv strings"