Skip to content

Commit 51deedf

Browse files
author
mbentz-uf
committed
Merge branch 'release/1.10.0'
2 parents a4e3310 + 3eb8f9c commit 51deedf

File tree

10 files changed

+188
-19
lines changed

10 files changed

+188
-19
lines changed

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,7 @@ build/
1313

1414
# Anything else is probably a preference. Think twice and then ask three people
1515
# before adding something else.
16+
17+
run_*/
18+
nacculator_cfg.ini
19+
.DS_STORE

‎AUTHORS

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ Contributors at the University of Florida
2222

2323
* Tarun Gupta Akirala
2424
* Christopher P. Barnes
25+
* Michael Bentz
2526
* Naomi Braun
2627
* Philip Chase
2728
* Samantha Emerson

‎CHANGELOG

+14
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,19 @@
11
Changelog
22
=========
3+
## [1.10.0] - 2022-08-31
4+
### Summary
5+
This release introduces minor fixes to the FTLD and LBD Short Version modules.
6+
7+
### Added
8+
* Add missing field for #3 in LBD Short Version FVP form E2L in the builder file (Samantha Emerson)
9+
* Add ability to run filters on a local redcap export csv rather than automatically downloading via redcap api (Samantha Emerson)
10+
11+
### Changed
12+
* Update gitignore (mbentz-uf)
13+
* Update allowable_favlues on FTDRATIO in ftld forms.py files to account for error code (Samantha Emerson)
14+
* Re-adjust Z1X spacing on FVP LBDSV to account for invisible deprecated C1 form (Samantha Emerson)
15+
* Update Z1X positions in forms.py for lbd short version (Samantha Emerson)
16+
317
## [1.9.0] - 2022-06-24
418

519
### Summary

‎README.md

+30-1
Original file line numberDiff line numberDiff line change
@@ -120,8 +120,37 @@ The filters can be run all at once with your REDCap API token using:
120120
You can find more details on `nacculator_filters` under the section:
121121
HOW TO Acquire current-db-subjects.csv for the filters
122122

123-
Or they can be run one at a time on a `.csv` file with the `-f` and `-meta`
123+
124+
RUNNING ALL FILTERS ON A LOCAL FILE
125+
------------------------------------------------------
126+
127+
REDCap has an export size limit that can be exceeded with a large project like
128+
the ADRC. When the size of the project surpasses the REDCap limit, the
129+
`nacculator_filters` command will no longer work. The data must be manually
130+
exported from the project in chunks (whether by event or by ptid). However you
131+
choose to export the data, keep in mind that all of the fields in a packet need
132+
to be present in the input csv you use. So, for example, the A1 and A2 forms in
133+
the IVP cannot be exported and run separately through NACCulator.
134+
135+
You can still run all the filters using your config file on a REDCap-exported
136+
csv, even when not using `nacculator_filters`. The command to use this filter
137+
locally is:
138+
139+
$ python3 nacc/local_filters.py nacculator_cfg.ini redcap_input.csv
140+
141+
where `redcap_input.csv` is the location of the file you want to filter. The
142+
filter will then run as normal, creating a `run_CURRENT-DATE` folder and
143+
depositing each stage of the filter process in this folder. The final output
144+
of the filter process is a csv file called `final_Update.csv` which can then
145+
be run through NACCulator.
146+
147+
148+
RUNNING INDIVIDUAL FILTERS
149+
------------------------------------------------------
150+
151+
The filters can also be run one at a time on a `.csv` file with the `-f` and `-meta`
124152
flags.
153+
125154
For example, to run the fixHeaders filter:
126155

127156
$ redcap2nacc -f fixHeaders -meta nacculator_cfg.ini <data_input.csv >filtered_output.csv

‎nacc/ftld/fvp/forms.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,7 @@ def __init__(self):
170170
self.fields['FTDSENPR'] = nacc.uds3.Field(name='FTDSENPR', typename='Num', position=(96, 97), length=2, inclusive_range=(0, 20), allowable_values=[], blanks=['Blank if Question 5a FTDSENAS = 95-98'])
171171
self.fields['FTDNOUNC'] = nacc.uds3.Field(name='FTDNOUNC', typename='Num', position=(99, 100), length=2, inclusive_range=(0, 16), allowable_values=['95', '96', '97', '98'], blanks=[])
172172
self.fields['FTDVERBC'] = nacc.uds3.Field(name='FTDVERBC', typename='Num', position=(102, 103), length=2, inclusive_range=(0, 16), allowable_values=[], blanks=['Blank if Question 6a FTDNOUNC = 95-98'])
173-
self.fields['FTDRATIO'] = nacc.uds3.Field(name='FTDRATIO', typename='Num', position=(105, 109), length=5, inclusive_range=(0,16), allowable_values=[], blanks=['Blank if Question 6a FTDNOUNC = 95-98'])
173+
self.fields['FTDRATIO'] = nacc.uds3.Field(name='FTDRATIO', typename='Num', position=(105, 109), length=5, inclusive_range=(0,16), allowable_values=[88.88], blanks=['Blank if Question 6a FTDNOUNC = 95-98'])
174174
self.fields['FTDREAAS'] = nacc.uds3.Field(name='FTDREAAS', typename='Num', position=(111, 112), length=2, inclusive_range=(0, 5), allowable_values=['95', '96', '97', '98'], blanks=[])
175175
self.fields['FTDREAOS'] = nacc.uds3.Field(name='FTDREAOS', typename='Num', position=(114, 115), length=2, inclusive_range=(0, 37), allowable_values=[], blanks=['Blank if Question 7a FTDREAAS = 95-98'])
176176
self.fields['FTDREASR'] = nacc.uds3.Field(name='FTDREASR', typename='Num', position=(117, 118), length=2, inclusive_range=(0, 20), allowable_values=[], blanks=['Blank if Question 7a FTDREAAS = 95-98'])

‎nacc/ftld/ivp/forms.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ def __init__(self):
171171
self.fields['FTDSENPR'] = nacc.uds3.Field(name='FTDSENPR', typename='Num', position=(96, 97), length=2, inclusive_range=(0, 20), allowable_values=[], blanks=['Blank if Question 5a FTDSENAS = 95-98'])
172172
self.fields['FTDNOUNC'] = nacc.uds3.Field(name='FTDNOUNC', typename='Num', position=(99, 100), length=2, inclusive_range=(0, 16), allowable_values=['95', '96', '97', '98'], blanks=[])
173173
self.fields['FTDVERBC'] = nacc.uds3.Field(name='FTDVERBC', typename='Num', position=(102, 103), length=2, inclusive_range=(0, 16), allowable_values=[], blanks=['Blank if Question 6a FTDNOUNC = 95-98'])
174-
self.fields['FTDRATIO'] = nacc.uds3.Field(name='FTDRATIO', typename='Num', position=(105, 109), length=5, inclusive_range=(0,16), allowable_values=[], blanks=['Blank if Question 6a FTDNOUNC = 95-98'])
174+
self.fields['FTDRATIO'] = nacc.uds3.Field(name='FTDRATIO', typename='Num', position=(105, 109), length=5, inclusive_range=(0,16), allowable_values=[88.88], blanks=['Blank if Question 6a FTDNOUNC = 95-98'])
175175
self.fields['FTDREAAS'] = nacc.uds3.Field(name='FTDREAAS', typename='Num', position=(111, 112), length=2, inclusive_range=(0, 5), allowable_values=['95', '96', '97', '98'], blanks=[])
176176
self.fields['FTDREAOS'] = nacc.uds3.Field(name='FTDREAOS', typename='Num', position=(114, 115), length=2, inclusive_range=(0, 37), allowable_values=[], blanks=['Blank if Question 7a FTDREAAS = 95-98'])
177177
self.fields['FTDREASR'] = nacc.uds3.Field(name='FTDREASR', typename='Num', position=(117, 118), length=2, inclusive_range=(0, 20), allowable_values=[], blanks=['Blank if Question 7a FTDREAAS = 95-98'])

‎nacc/lbd/v3_1/fvp/builder.py

+1
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,7 @@ def add_e2l(record,packet):
447447
E2L.LBIFPtPP = record['fu_LBIFPtPP'.lower()]
448448
E2L.LBIFPISL = record['fu_LBIFPISL'.lower()]
449449
E2L.LBIFPAVL = record['fu_LBIFPAVL'.lower()]
450+
E2L.LBIAPet = record['fu_LBIAPet'.lower()]
450451
E2L.LBIAPAVL = record['fu_LBIAPAVL'.lower()]
451452
E2L.LBItPet = record['fu_LBItPet'.lower()]
452453
E2L.LBItPAVL = record['fu_LBItPAVL'.lower()]

‎nacc/lbd/v3_1/fvp/forms.py

+15-15
Original file line numberDiff line numberDiff line change
@@ -46,21 +46,21 @@ def __init__(self):
4646
self.fields['LANGA4'] = nacc.uds3.Field(name='LANGA4', typename='Num', position=(61, 61), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 4b A4SUB = 0 (No)'])
4747
self.fields['A4SUB'] = nacc.uds3.Field(name='A4SUB', typename='Num', position=(63, 63), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
4848
self.fields['A4NOT'] = nacc.uds3.Field(name='A4NOT', typename='Num', position=(65, 66), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 4b A4SUB = 1 (Yes)'])
49-
self.fields['LANGB1'] = nacc.uds3.Field(name='LANGB1', typename='Num', position=(70, 70), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 6b B1SUB = 0 (No)'])
50-
self.fields['B1SUB'] = nacc.uds3.Field(name='B1SUB', typename='Num', position=(72, 72), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
51-
self.fields['B1NOT'] = nacc.uds3.Field(name='B1NOT', typename='Num', position=(74, 75), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 6b B1SUB = 1 (Yes)'])
52-
self.fields['LANGB4'] = nacc.uds3.Field(name='LANGB4', typename='Num', position=(77, 77), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
53-
self.fields['LANGB5'] = nacc.uds3.Field(name='LANGB5', typename='Num', position=(79, 79), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 8b B5SUB = 0 (No)'])
54-
self.fields['B5SUB'] = nacc.uds3.Field(name='B5SUB', typename='Num', position=(81, 81), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
55-
self.fields['B5NOT'] = nacc.uds3.Field(name='B5NOT', typename='Num', position=(83, 84), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 8b B5SUB = 1 (Yes)'])
56-
self.fields['LANGB6'] = nacc.uds3.Field(name='LANGB6', typename='Num', position=(86, 86), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 9b B6SUB = 0 (No)'])
57-
self.fields['B6SUB'] = nacc.uds3.Field(name='B6SUB', typename='Num', position=(88, 88), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
58-
self.fields['B6NOT'] = nacc.uds3.Field(name='B6NOT', typename='Num', position=(90, 91), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 9b B6SUB = 1 (Yes)'])
59-
self.fields['LANGB7'] = nacc.uds3.Field(name='LANGB7', typename='Num', position=(93, 93), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 10b B7SUB = 0 (No)'])
60-
self.fields['B7SUB'] = nacc.uds3.Field(name='B7SUB', typename='Num', position=(95, 95), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
61-
self.fields['B7NOT'] = nacc.uds3.Field(name='B7NOT', typename='Num', position=(97, 98), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 10b B7SUB = 1 (Yes)'])
62-
self.fields['LANGB8'] = nacc.uds3.Field(name='LANGB8', typename='Num', position=(100, 100), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
63-
self.fields['LANGB9'] = nacc.uds3.Field(name='LANGB9', typename='Num', position=(102, 102), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
49+
self.fields['LANGB1'] = nacc.uds3.Field(name='LANGB1', typename='Num', position=(68, 68), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 6b B1SUB = 0 (No)'])
50+
self.fields['B1SUB'] = nacc.uds3.Field(name='B1SUB', typename='Num', position=(70, 70), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
51+
self.fields['B1NOT'] = nacc.uds3.Field(name='B1NOT', typename='Num', position=(72, 73), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 6b B1SUB = 1 (Yes)'])
52+
self.fields['LANGB4'] = nacc.uds3.Field(name='LANGB4', typename='Num', position=(75, 75), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
53+
self.fields['LANGB5'] = nacc.uds3.Field(name='LANGB5', typename='Num', position=(77, 77), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 8b B5SUB = 0 (No)'])
54+
self.fields['B5SUB'] = nacc.uds3.Field(name='B5SUB', typename='Num', position=(79, 79), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
55+
self.fields['B5NOT'] = nacc.uds3.Field(name='B5NOT', typename='Num', position=(81, 82), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 8b B5SUB = 1 (Yes)'])
56+
self.fields['LANGB6'] = nacc.uds3.Field(name='LANGB6', typename='Num', position=(84, 84), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 9b B6SUB = 0 (No)'])
57+
self.fields['B6SUB'] = nacc.uds3.Field(name='B6SUB', typename='Num', position=(86, 86), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
58+
self.fields['B6NOT'] = nacc.uds3.Field(name='B6NOT', typename='Num', position=(88, 89), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 9b B6SUB = 1 (Yes)'])
59+
self.fields['LANGB7'] = nacc.uds3.Field(name='LANGB7', typename='Num', position=(91, 91), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=['Blank if Question 10b B7SUB = 0 (No)'])
60+
self.fields['B7SUB'] = nacc.uds3.Field(name='B7SUB', typename='Num', position=(93, 93), length=1, inclusive_range=(0, 1), allowable_values=['0', '1'], blanks=[])
61+
self.fields['B7NOT'] = nacc.uds3.Field(name='B7NOT', typename='Num', position=(95, 96), length=2, inclusive_range=None, allowable_values=['95', '96', '97', '98'], blanks=['Blank if Question 10b B7SUB = 1 (Yes)'])
62+
self.fields['LANGB8'] = nacc.uds3.Field(name='LANGB8', typename='Num', position=(98, 98), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
63+
self.fields['LANGB9'] = nacc.uds3.Field(name='LANGB9', typename='Num', position=(100, 100), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
6464
self.fields['LANGC2'] = nacc.uds3.Field(name='LANGC2', typename='Num', position=(104, 104), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
6565
self.fields['LANGD1'] = nacc.uds3.Field(name='LANGD1', typename='Num', position=(106, 106), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])
6666
self.fields['LANGD2'] = nacc.uds3.Field(name='LANGD2', typename='Num', position=(108, 108), length=1, inclusive_range=(1, 2), allowable_values=['1', '2'], blanks=[])

‎nacc/local_filters.py

+120
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import os
2+
import sys
3+
import csv
4+
import datetime
5+
import configparser
6+
from nacc.uds3.filters import *
7+
8+
9+
# Creating a folder which contains Intermediate files
10+
def recent_run_folder(out_dir):
11+
# Check if directory exists. If not, create it.
12+
if not os.path.exists(out_dir):
13+
try:
14+
os.makedirs(out_dir)
15+
except Exception as e:
16+
raise e
17+
18+
19+
def get_headers(input_ptr):
20+
reader = csv.DictReader(input_ptr)
21+
headers = reader.fieldnames
22+
print(headers)
23+
24+
25+
def run_all_filters(folder_name, config, input_name):
26+
# Calling Filters
27+
try:
28+
print("--------------Removing subjects already in current--------------------", file=sys.stderr)
29+
if input_name:
30+
input_path = input_name
31+
else:
32+
input_path = os.path.join(folder_name, "redcap_input.csv")
33+
output_path = os.path.join(folder_name, "clean.csv")
34+
print("Processing", file=sys.stderr)
35+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
36+
filter_clean_ptid(input_ptr, config, output_ptr)
37+
38+
print("--------------Replacing drug IDs--------------------", file=sys.stderr)
39+
input_path = os.path.join(folder_name, "clean.csv")
40+
output_path = os.path.join(folder_name, "drugs.csv")
41+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
42+
filter_replace_drug_id(input_ptr, config, output_ptr)
43+
44+
print("--------------Fixing Headers--------------------", file=sys.stderr)
45+
input_path = os.path.join(folder_name, "drugs.csv")
46+
output_path = os.path.join(folder_name, "clean_headers.csv")
47+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
48+
filter_fix_headers(input_ptr, config, output_ptr)
49+
50+
print("--------------Filling in Defaults--------------------", file=sys.stderr)
51+
input_path = os.path.join(folder_name, "clean_headers.csv")
52+
output_path = os.path.join(folder_name, "default.csv")
53+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
54+
filter_fill_default(input_ptr, config, output_ptr)
55+
56+
print("--------------Updating fields--------------------", file=sys.stderr)
57+
input_path = os.path.join(folder_name, "default.csv")
58+
output_path = os.path.join(folder_name, "update_fields.csv")
59+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
60+
filter_update_field(input_ptr, config, output_ptr)
61+
62+
print("--------------Fixing Visit Dates--------------------", file=sys.stderr)
63+
input_path = os.path.join(folder_name, "update_fields.csv")
64+
output_path = os.path.join(folder_name, "proper_visitdate.csv")
65+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
66+
filter_fix_visitdate(input_ptr, config, output_ptr)
67+
68+
print("--------------Removing Unnecessary Records--------------------", file=sys.stderr)
69+
input_path = os.path.join(folder_name, "proper_visitdate.csv")
70+
output_path = os.path.join(folder_name, "CleanedPtid_Update.csv")
71+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
72+
filter_remove_ptid(input_ptr, config, output_ptr)
73+
74+
print("--------------Removing Records without VisitDate--------------------", file=sys.stderr)
75+
input_path = os.path.join(folder_name, "CleanedPtid_Update.csv")
76+
output_path = os.path.join(folder_name, "final_Update.csv")
77+
with open(output_path, 'w') as output_ptr, open(input_path, 'r') as input_ptr:
78+
filter_eliminate_empty_date(input_ptr, config, output_ptr)
79+
80+
except Exception as e:
81+
print("Error in Opening a file")
82+
print(e)
83+
84+
return
85+
86+
87+
def read_config(config_path):
88+
config = configparser.ConfigParser()
89+
config.read(config_path)
90+
return config
91+
92+
93+
def main():
94+
currentdate = datetime.datetime.now().strftime('%m-%d-%Y')
95+
folder_name = "run_" + currentdate
96+
input_name = ""
97+
print("Recent folder " + folder_name, file=sys.stderr)
98+
99+
current_directory = os.getcwd()
100+
identified_folder = os.path.join(current_directory, folder_name)
101+
102+
if not os.path.exists(identified_folder):
103+
recent_run_folder(identified_folder)
104+
105+
# Reading from Config and Accessing the necessary Data
106+
config_path = sys.argv[1]
107+
try:
108+
if sys.argv[2]:
109+
input_name = sys.argv[2]
110+
except IndexError:
111+
pass
112+
# config = read_config(config_path)
113+
114+
run_all_filters(folder_name, config_path, input_name)
115+
116+
exit()
117+
118+
119+
if __name__ == '__main__':
120+
main()

‎setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
from setuptools import setup, find_packages
88

9-
VERSION = "1.9.0"
9+
VERSION = "1.10.0"
1010

1111
setup(
1212
name="nacculator",

0 commit comments

Comments
 (0)