Merge branch 'release/1.14.0'

s-emerson · s-emerson · commit e495d8016706 · 2024-08-26T13:48:41.000-04:00
diff --git a/CHANGELOG b/CHANGELOG
@@ -1,5 +1,19 @@
 Changelog
 =========
+## [1.14.0] - 2024-08-26
+### Summary
+This release adds support for multiple REDCap projects, in case the Optional Modules such as LBD or FTLD are broken out into their own REDCap projects. To pull from multiple REDCap projects into the same `redcap_input.csv` file, create a list of API tokens in your config file. This update also changes NACCulator's event detection to require the presence of both the Z1X and the A1 forms for UDS visits.
+
+### Added
+ * Add additional unit test (Michael Bentz)
+ * Add pandas to setup.py (Michael Bentz)
+ * Add script to combine REDCap project data (Michael Bentz)
+
+### Updated
+ * Update README with new requirements for ivp and fvp processing (Samantha Emerson)
+ * Add A1 to required forms for IVP and FVP processing (Samantha Emerson)
+ * Bump report_handler dependency to 1.3.0 (Michael Bentz)
+
 ## [1.13.1] - 2024-04-15
 ### Summary
 This release removes the `.vscode` sub-module from the NACCulator repo, since it points to a private repository and makes NACCulator difficult to install for those who do not have permissions for the .vscode repo. 
diff --git a/README.md b/README.md
@@ -45,6 +45,9 @@ marked "Unverified" or "Complete" for NACCulator to recognize the record, and
 each optional form must be marked as submitted within the Z1X for NACCulator to
 find those forms.
 
+_Note: For UDS visits (the -ivp and -fvp flags), NACCulator also expects the
+A1 subject demographics form to be either Unverified or Complete._
+
 _Note: output is written to `STDOUT`; errors are written to `STDERR`; input is
 expected to be from `STDIN` (the command line) unless a file is specified using
 the `-file` flag._
@@ -351,7 +354,7 @@ This is not exhaustive, but here is an explanation of some important files.
 
 * `nacc/run_filters.py` and `tools/preprocess/run_filters.sh`:
     pulls data from REDCap based on the settings found in `nacculator_cfg.ini`
-    (for .py) and `filters_config.cfg` (for .sh).
+    (for .py) and `filters_config.cfg` (for .sh). Supports exporting data from multiple REDCap projects by adding a comma-delimited list of tokens without spaces e.g., `token=token1,token2` to `token` in the `nacculator_cfg.ini` config file.
 
 
 ### Testing
diff --git a/nacc/redcap2nacc.py b/nacc/redcap2nacc.py
@@ -237,7 +237,12 @@ def check_redcap_event(
             form_match_z1 = ''
             record['ivp_z1_complete'] = ''
         form_match_z1x = record['ivp_z1x_complete']
-        if form_match_z1 in ['0', ''] and form_match_z1x in ['0', '']:
+        try:
+            form_match_a1 = record['ivp_a1_complete']
+        except KeyError:
+            form_match_a1 = record['ivp_a1_subject_demographics_complete']
+        if (form_match_z1 in ['0', ''] and form_match_z1x in ['0', '']) or \
+           form_match_a1 in ['0', '']:
             return False
     elif options.fvp:
         event_name = 'follow'
@@ -247,7 +252,12 @@ def check_redcap_event(
             form_match_z1 = ''
             record['fvp_z1_complete'] = ''
         form_match_z1x = record['fvp_z1x_complete']
-        if form_match_z1 in ['0', ''] and form_match_z1x in ['0', '']:
+        try:
+            form_match_a1 = record['fvp_a1_complete']
+        except KeyError:
+            form_match_a1 = record['fvp_a1_subject_demographics_complete']
+        if (form_match_z1 in ['0', ''] and form_match_z1x in ['0', '']) or \
+           form_match_a1 in ['0', '']:
             return False
     # TODO: add -csf option if/when it is added to the full ADRC project.
     elif options.cv:
diff --git a/nacc/run_filters.py b/nacc/run_filters.py
@@ -6,6 +6,7 @@
 from redcap import Project
 from nacc.uds3.filters import *
 import logging
+import pandas as pd
 
 
 # Creating a folder which contains Intermediate files
@@ -112,7 +113,7 @@ def get_data_from_redcap_pycap(folder_name, config):
     """
     # Enter the path for filters_config
     try:
-        token = config.get('pycap', 'token')
+        tokens = config.get('pycap', 'token').split(',')
         redcap_url = config.get('pycap', 'redcap_server')
     except Exception as e:
         print("Please check the config file and validate all the proper fields exist", file=sys.stderr)
@@ -130,7 +131,32 @@ def get_data_from_redcap_pycap(folder_name, config):
         print(e)
         raise e
 
-    redcap_project = Project(redcap_url, token)
+    try:
+        df_all_project_data = pd.DataFrame()
+        for token in tokens:
+            df_project_data = _get_project_data(redcap_url, token)
+            df_all_project_data = pd.concat([df_all_project_data, df_project_data], ignore_index=True)
+
+        # Ignore index to remove the record number column
+        df_all_project_data.to_csv(os.path.join(folder_name, "redcap_input.csv"), index=False)
+    except Exception as e:
+        print("Error in exporting project data")
+        logging.error('Error in processing project data',
+                      extra={
+                          "report_handler": {
+                              "data": {"ptid": None, "error": f'Error in exporting project data: {e}'},
+                              "sheet": 'ERROR'
+                          }
+                      }
+                      )
+        print(e)
+
+    return
+
+
+def _get_project_data(url: str, token: str) -> pd.DataFrame:
+    df_project_data = pd.DataFrame()
+    redcap_project = Project(url, token)
 
     # Get list of all fieldnames in project to create a csv header
     assert hasattr(redcap_project, 'field_names')
@@ -146,8 +172,13 @@ def get_data_from_redcap_pycap(folder_name, config):
 
     # Get list of all records present in project to iterate over
     list_of_records = []
+
+    # If there are multiple records with the same ptid only keep the first occurence
+    # export all records only grabbing the ptid field
     all_records = redcap_project.export_records(fields=['ptid'])
+    # iterate through all records
     for record in all_records:
+        # if the ptid is not already in the list then append it
         if record['ptid'] not in list_of_records:
             list_of_records.append(record['ptid'])
 
@@ -157,43 +188,11 @@ def get_data_from_redcap_pycap(folder_name, config):
     for i in range(0, len(list_of_records), n):
         chunked_records.append(list_of_records[i:i + n])
 
-    try:
-
-        try:
-            with open(os.path.join(folder_name, "redcap_input.csv"), "w") as redcap_export:
-                writer = csv.DictWriter(redcap_export, fieldnames=header_full)
-                writer.writeheader()
-                # header_mapping = next(reader)
-                for current_record_chunk in chunked_records:
-                    data = redcap_project.export_records(
-                        records=current_record_chunk)
-                    for row in data:
-                        writer.writerow(row)
-        except Exception as e:
-            print("Error in Writing")
-            logging.error('Error in writing',
-                          extra={
-                              "report_handler": {
-                                  "data": {"ptid": None, "error": f'Error in writing: {e}'},
-                                  "sheet": 'ERROR'
-                              }
-                          }
-                          )
-            print(e)
+    for current_record_chunk in chunked_records:
+        chunked_data = redcap_project.export_records(records=current_record_chunk)
+        df_project_data = pd.concat([df_project_data, pd.DataFrame(chunked_data)], ignore_index=True)
 
-    except Exception as e:
-        print("Error in CSV file")
-        logging.error('Error in CSV file',
-                      extra={
-                          "report_handler": {
-                              "data": {"ptid": None,
-                                       "error": f'Error in CSV file: {e}'},
-                              "sheet": 'ERROR'
-                          }
-                      }
-                      )
-
-    return
+    return df_project_data
 
 
 def main():
diff --git a/nacculator_cfg.ini.example b/nacculator_cfg.ini.example
@@ -1,8 +1,9 @@
 [DEFAULT]
 
 [pycap]
+redcap_server: http://<redcap_server>/api/
+# token now supports multiple tokens as a comma-delimeted list without spaces e.g., token1,token2,token3,token4
 token: Your REDCAP Token
-redcap_server: Your Redcap Server
 
 # [filters] - Each section is named after the corresponding function name
 # in filters.py
diff --git a/setup.py b/setup.py
@@ -6,7 +6,7 @@
 
 from setuptools import setup, find_packages
 
-VERSION = "1.13.1"
+VERSION = "1.14.0"
 
 setup(
     name="nacculator",
@@ -33,7 +33,8 @@
 
     install_requires=[
         "PyCap>=2.1.0",
-        "report_handler @ git+https://git@github.com:/ctsit/report_handler.git"
+        "pandas>=2.2.0",
+        "report_handler @ git+https://git@github.com:/ctsit/report_handler.git@1.3.0"
     ],
 
     python_requires=">=3.6.0",
diff --git a/tests/test_check_redcap_event.py b/tests/test_check_redcap_event.py
@@ -35,18 +35,26 @@ def test_for_ivp(self):
         '''
         self.options.ivp = True
         record = {'redcap_event_name': 'initial_visit_year_arm_1',
-                  'ivp_z1_complete': '', 'ivp_z1x_complete': '2'}
+                  'ivp_z1_complete': '', 'ivp_z1x_complete': '2',
+                  'ivp_a1_complete': '2'}  # condition met to return True
         result = check_redcap_event(self.options, record)
         self.assertTrue(result)
 
+        record = {'redcap_event_name': 'initial_visit_year_arm_1',
+                  'ivp_z1_complete': '', 'ivp_z1x_complete': '2',
+                  'ivp_a1_complete': ''}  # condition met to return False
+        result2 = check_redcap_event(self.options, record)
+        self.assertFalse(result2)
+
     def test_for_not_ivp(self):
         '''
         Checks that the initial_visit is not returned when the -ivp flag is not
         set.
         '''
         self.options.fvp = True
         record = {'redcap_event_name': 'initial_visit_year_arm_1',
-                  'fvp_z1_complete': '', 'fvp_z1x_complete': ''}
+                  'fvp_z1_complete': '', 'fvp_z1x_complete': '',
+                  'fvp_a1_complete': '2'}
         result = check_redcap_event(self.options, record)
         self.assertFalse(result)
 
@@ -68,7 +76,7 @@ def test_for_not_multiple_flags(self):
         self.options.ivp = True
         record = {'redcap_event_name': 'initial_visit_year_arm_1',
                   'ivp_z1_complete': '', 'ivp_z1x_complete': '',
-                  'lbd_ivp_b1l_complete': '2'}
+                  'lbd_ivp_b1l_complete': '2', 'ivp_a1_complete': '2'}
         incorrect = check_redcap_event(self.options, record)
 
         self.options.lbd = True
