From 2d9f17ff33e2925f7990b88a1b22674481fade26 Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Wed, 19 Aug 2020 15:06:17 -0600 Subject: [PATCH 1/8] Added support for the dateRangeBegin and dateRangeEnd API Parameters --- redcap/__init__.py | 2 +- redcap/project.py | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/redcap/__init__.py b/redcap/__init__.py index d1ae61f..6ab7f23 100644 --- a/redcap/__init__.py +++ b/redcap/__init__.py @@ -4,7 +4,7 @@ __author__ = 'Scott Burns ' __license__ = 'MIT' __copyright__ = '2014, Vanderbilt University' -__version__ = '1.1.1' +__version__ = '1.1.2' """ This module exposes the REDCap API through the Project class. Instantiate the diff --git a/redcap/project.py b/redcap/project.py index 5a76717..08636ef 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -6,6 +6,7 @@ __copyright__ = '2014, Vanderbilt University' import json +from dateutil.parser import parse import warnings from .request import RCRequest, RedcapError, RequestException @@ -267,7 +268,8 @@ def export_records(self, records=None, fields=None, forms=None, events=None, raw_or_label='raw', event_name='label', format='json', export_survey_fields=False, export_data_access_groups=False, df_kwargs=None, - export_checkbox_labels=False, filter_logic=None): + export_checkbox_labels=False, filter_logic=None, + date_begin=None, date_end=None): """ Export data from the REDCap project. @@ -319,6 +321,10 @@ def export_records(self, records=None, fields=None, forms=None, export. filter_logic : string specify the filterLogic to be sent to the API. + date_begin : string + for the dateRangeStart filtering of the API + date_end : string + for the dateRangeEnd filtering snet to the API Returns ------- @@ -332,10 +338,18 @@ def export_records(self, records=None, fields=None, forms=None, fields = self.backfill_fields(fields, forms) keys_to_add = (records, fields, forms, events, raw_or_label, event_name, export_survey_fields, - export_data_access_groups, export_checkbox_labels) + export_data_access_groups, export_checkbox_labels, + date_begin, date_end) str_keys = ('records', 'fields', 'forms', 'events', 'rawOrLabel', 'eventName', 'exportSurveyFields', 'exportDataAccessGroups', - 'exportCheckboxLabel') + 'exportCheckboxLabel', 'dateRangeBegin', 'dateRangeEnd') + + if date_begin: + date_begin = parse(date_begin).strftime('%Y-%m-%d %H:%M:%S') + + if date_end: + date_end = parse(date_end).strftime('%Y-%m-%d %H:%M:%S') + for key, data in zip(str_keys, keys_to_add): if data: if key in ('fields', 'records', 'forms', 'events'): @@ -699,7 +713,7 @@ def export_survey_participant_list(self, instrument, event=None, format='json'): if event: pl['event'] = event return self._call_api(pl, 'exp_survey_participant_list') - + def generate_next_record_name(self): pl = self.__basepl(content='generateNextRecordName') From 2e59ac9a1c01d7814c16f6805ad3754eba97bb1a Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Wed, 19 Aug 2020 15:17:34 -0600 Subject: [PATCH 2/8] updated requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 84253b4..4d2d6f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ semantic-version==2.3.1 requests>=1.1.0 wheel==0.22.0 responses==0.9.0 +python-dateutil==2.8.1 From a7029bde022118f660081797ee4c6ba5320ff8dc Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Wed, 19 Aug 2020 15:48:22 -0600 Subject: [PATCH 3/8] refactoring date filtering to be more in line with filter_logic parameter --- redcap/project.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/redcap/project.py b/redcap/project.py index 08636ef..ab112d8 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -338,17 +338,11 @@ def export_records(self, records=None, fields=None, forms=None, fields = self.backfill_fields(fields, forms) keys_to_add = (records, fields, forms, events, raw_or_label, event_name, export_survey_fields, - export_data_access_groups, export_checkbox_labels, - date_begin, date_end) + export_data_access_groups, export_checkbox_labels) + str_keys = ('records', 'fields', 'forms', 'events', 'rawOrLabel', 'eventName', 'exportSurveyFields', 'exportDataAccessGroups', - 'exportCheckboxLabel', 'dateRangeBegin', 'dateRangeEnd') - - if date_begin: - date_begin = parse(date_begin).strftime('%Y-%m-%d %H:%M:%S') - - if date_end: - date_end = parse(date_end).strftime('%Y-%m-%d %H:%M:%S') + 'exportCheckboxLabel') for key, data in zip(str_keys, keys_to_add): if data: @@ -358,6 +352,13 @@ def export_records(self, records=None, fields=None, forms=None, else: pl[key] = data + if date_begin: + pl["dateRangeBegin"] = \ + parse(date_begin).strftime('%Y-%m-%d %H:%M:%S') + + if date_end: + pl["dateRangeEnd"] = parse(date_end).strftime('%Y-%m-%d %H:%M:%S') + if filter_logic: pl["filterLogic"] = filter_logic response, _ = self._call_api(pl, 'exp_record') From 1d54271203b51d4b68df5684bc3d1fb0da77d1fa Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Thu, 20 Aug 2020 08:09:59 -0600 Subject: [PATCH 4/8] changed date_end, date_begin to datetime objs and removed dateutil requirement --- redcap/project.py | 10 ++++------ requirements.txt | 1 - 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/redcap/project.py b/redcap/project.py index ab112d8..96d460f 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -6,7 +6,6 @@ __copyright__ = '2014, Vanderbilt University' import json -from dateutil.parser import parse import warnings from .request import RCRequest, RedcapError, RequestException @@ -321,9 +320,9 @@ def export_records(self, records=None, fields=None, forms=None, export. filter_logic : string specify the filterLogic to be sent to the API. - date_begin : string + date_begin : datetime for the dateRangeStart filtering of the API - date_end : string + date_end : datetime for the dateRangeEnd filtering snet to the API Returns @@ -353,11 +352,10 @@ def export_records(self, records=None, fields=None, forms=None, pl[key] = data if date_begin: - pl["dateRangeBegin"] = \ - parse(date_begin).strftime('%Y-%m-%d %H:%M:%S') + pl["dateRangeBegin"] = date_begin.strftime('%Y-%m-%d %H:%M:%S') if date_end: - pl["dateRangeEnd"] = parse(date_end).strftime('%Y-%m-%d %H:%M:%S') + pl["dateRangeEnd"] = date_end.strftime('%Y-%m-%d %H:%M:%S') if filter_logic: pl["filterLogic"] = filter_logic diff --git a/requirements.txt b/requirements.txt index 4d2d6f2..84253b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,3 @@ semantic-version==2.3.1 requests>=1.1.0 wheel==0.22.0 responses==0.9.0 -python-dateutil==2.8.1 From ab0182ee0de1b7f604e95ad1c7d0dc8ef6597ff2 Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Fri, 28 Aug 2020 21:12:49 -0600 Subject: [PATCH 5/8] updating to add export_feature_names method --- redcap/__init__.py | 2 +- redcap/project.py | 40 ++++++++++++++++++++++++++++++++++++++++ redcap/request.py | 2 ++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/redcap/__init__.py b/redcap/__init__.py index 6ab7f23..7449862 100644 --- a/redcap/__init__.py +++ b/redcap/__init__.py @@ -4,7 +4,7 @@ __author__ = 'Scott Burns ' __license__ = 'MIT' __copyright__ = '2014, Vanderbilt University' -__version__ = '1.1.2' +__version__ = '1.1.3' """ This module exposes the REDCap API through the Project class. Instantiate the diff --git a/redcap/project.py b/redcap/project.py index 96d460f..a07189e 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -192,6 +192,46 @@ def export_fem(self, arms=None, format='json', df_kwargs=None): return self.read_csv(StringIO(response), **df_kwargs) + def export_field_names(self, field=None, format='json', df_kwargs=None): + """ + Export the project's field names + + Parameters + ---------- + fields : str + Limit exported field name to this field (only single field supported). + When not provided, all fields returned. + format : (``'json'``), ``'csv'``, ``'xml'``, ``'df'`` + Return the metadata in native objects, csv or xml. + ``'df'`` will return a ``pandas.DataFrame``. + df_kwargs : dict + Passed to ``pandas.read_csv`` to control construction of + returned DataFrame. + by default ``{'index_col': 'original_field_name'}`` + + Returns + ------- + metadata : list, str, ``pandas.DataFrame`` + metadata sttructure for the project. + """ + ret_format = format + if format == 'df': + ret_format = 'csv' + + pl = self.__basepl('exportFieldNames', format=ret_format) + + if field: + pl['field'] = field + + response, _ = self._call_api(pl, 'exp_field_names') + + if format in ('json', 'csv', 'xml'): + return response + elif format == 'df': + if not df_kwargs: + df_kwargs = {'index_col': 'original_field_name'} + return self.read_csv(StringIO(response), **df_kwargs) + def export_metadata(self, fields=None, forms=None, format='json', df_kwargs=None): """ diff --git a/redcap/request.py b/redcap/request.py index bf6cce4..f5e9714 100644 --- a/redcap/request.py +++ b/redcap/request.py @@ -69,6 +69,8 @@ def validate(self): valid_data = { 'exp_record': (['type', 'format'], 'record', 'Exporting record but content is not record'), + 'exp_field_names': (['format'], 'exportFieldNames', + 'Exporting field names, but content is not exportFieldNames'), 'del_record': (['format'], 'record', 'Deleting record but content is not record'), 'imp_record': (['type', 'overwriteBehavior', 'data', 'format'], From bdd9dce6a38e3790c423bfbb1a888d0326e595f6 Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Mon, 31 Aug 2020 09:09:43 -0600 Subject: [PATCH 6/8] adding logging to instantiation --- redcap/project.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/redcap/project.py b/redcap/project.py index a07189e..2a8ee2a 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -11,6 +11,8 @@ from .request import RCRequest, RedcapError, RequestException import semantic_version +import logging + try: from StringIO import StringIO except ImportError: @@ -49,26 +51,51 @@ def __init__(self, url, token, name='', verify_ssl=True, lazy=False): self.arm_names = None self.configured = False + self.logger = logging.getLogger(__name__) + if not lazy: self.configure() def configure(self): + + self.logger.debug("Configuring project.") + + self.logger.debug("Exporting metadata.") try: self.metadata = self.__md() except RequestException: raise RedcapError("Exporting metadata failed. Check your URL and token.") + self.logger.debug("Metadata exported.") + self.logger.debug("Exporting REDCap version") try: self.redcap_version = self.__rcv() except: raise RedcapError("Determination of REDCap version failed") + self.logger.debug("Version exported.") + + self.logger.debug("Filtering metadata on field_name") self.field_names = self.filter_metadata('field_name') + self.logger.debug("metadata filtering done") + + self.logger.debug("setting default field") # we'll use the first field as the default id for each row self.def_field = self.field_names[0] + + self.logger.debug("Filtering metadata on field_label") self.field_labels = self.filter_metadata('field_label') + + self.logger.debug("Grabbing form names") self.forms = tuple(set(c['form_name'] for c in self.metadata)) + + self.logger.debug("determining whether logitudinal") # determine whether longitudinal + self.logger.debug("exporting events") ev_data = self._call_api(self.__basepl('event'), 'exp_event')[0] + self.logger.debug("exporting events COMPLETE") + self.logger.debug("exporting arms") + arm_data = self._call_api(self.__basepl('arm'), 'exp_arm')[0] + self.logger.debug("exporting arms COMPLETE") if isinstance(ev_data, dict) and ('error' in ev_data.keys()): events = tuple([]) From bb98c14d6d202a06df8a4d69bb8d1f9e4ec41d02 Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Wed, 16 Sep 2020 12:28:24 -0600 Subject: [PATCH 7/8] removing logging for pr --- redcap/__init__.py | 2 +- redcap/project.py | 27 --------------------------- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/redcap/__init__.py b/redcap/__init__.py index 7449862..6ab7f23 100644 --- a/redcap/__init__.py +++ b/redcap/__init__.py @@ -4,7 +4,7 @@ __author__ = 'Scott Burns ' __license__ = 'MIT' __copyright__ = '2014, Vanderbilt University' -__version__ = '1.1.3' +__version__ = '1.1.2' """ This module exposes the REDCap API through the Project class. Instantiate the diff --git a/redcap/project.py b/redcap/project.py index 2a8ee2a..a07189e 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -11,8 +11,6 @@ from .request import RCRequest, RedcapError, RequestException import semantic_version -import logging - try: from StringIO import StringIO except ImportError: @@ -51,51 +49,26 @@ def __init__(self, url, token, name='', verify_ssl=True, lazy=False): self.arm_names = None self.configured = False - self.logger = logging.getLogger(__name__) - if not lazy: self.configure() def configure(self): - - self.logger.debug("Configuring project.") - - self.logger.debug("Exporting metadata.") try: self.metadata = self.__md() except RequestException: raise RedcapError("Exporting metadata failed. Check your URL and token.") - self.logger.debug("Metadata exported.") - self.logger.debug("Exporting REDCap version") try: self.redcap_version = self.__rcv() except: raise RedcapError("Determination of REDCap version failed") - self.logger.debug("Version exported.") - - self.logger.debug("Filtering metadata on field_name") self.field_names = self.filter_metadata('field_name') - self.logger.debug("metadata filtering done") - - self.logger.debug("setting default field") # we'll use the first field as the default id for each row self.def_field = self.field_names[0] - - self.logger.debug("Filtering metadata on field_label") self.field_labels = self.filter_metadata('field_label') - - self.logger.debug("Grabbing form names") self.forms = tuple(set(c['form_name'] for c in self.metadata)) - - self.logger.debug("determining whether logitudinal") # determine whether longitudinal - self.logger.debug("exporting events") ev_data = self._call_api(self.__basepl('event'), 'exp_event')[0] - self.logger.debug("exporting events COMPLETE") - self.logger.debug("exporting arms") - arm_data = self._call_api(self.__basepl('arm'), 'exp_arm')[0] - self.logger.debug("exporting arms COMPLETE") if isinstance(ev_data, dict) and ('error' in ev_data.keys()): events = tuple([]) From 5a034d894ce9ba48f943d117b0b91510a54fd626 Mon Sep 17 00:00:00 2001 From: Chris Greer Date: Thu, 17 Sep 2020 16:03:39 -0600 Subject: [PATCH 8/8] Update redcap/project.py Co-authored-by: pwildenhain <35195136+pwildenhain@users.noreply.github.com> --- redcap/project.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redcap/project.py b/redcap/project.py index a07189e..9f96db6 100755 --- a/redcap/project.py +++ b/redcap/project.py @@ -194,7 +194,7 @@ def export_fem(self, arms=None, format='json', df_kwargs=None): def export_field_names(self, field=None, format='json', df_kwargs=None): """ - Export the project's field names + Export the project's export field names Parameters ---------- @@ -212,7 +212,7 @@ def export_field_names(self, field=None, format='json', df_kwargs=None): Returns ------- metadata : list, str, ``pandas.DataFrame`` - metadata sttructure for the project. + metadata structure for the project. """ ret_format = format if format == 'df':