Skip to content

Commit 27e64a5

Browse files
KevinHansonKevinHanson
committed
Merge branch 'release/1.7.0'
2 parents 71b2515 + e65fc80 commit 27e64a5

19 files changed

+4640
-55
lines changed

‎CHANGELOG

+13-1
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,20 @@
11
Changelog
22
=========
3+
## [1.7.0] - 2021-01-15
4+
### Summary
5+
This release incorporates the telephone followup packet 3.2 module which
6+
includes the C2T form. Centers will be able to process data from forms like
7+
the T1 and C2T, along with the expanded telehealth regimen in the other
8+
forms provided by NACC due to the COVID-19 pandemic.
9+
10+
### Added
11+
* Unit tests for TFP module functionality
12+
* Re-add skipping logic to TFP form a3 from the updated DED for TFP
13+
* Added the tele_ prefix to drug id filter for form A4
14+
315
## [1.6.0] - 2020-12-15
416
### Summary
5-
This change was implemented after getting feedback from OHSU. Some ADRCs do
17+
This change was implemented after getting feedback from OHSU. Some ADRCs do
618
not have the optional forms like A2 or A3 in their REDCap project at all, since
719
they will not be used for that center. NACCulator used to run with the
820
requirement that all forms be present in a REDCap project, whether they were

‎README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,8 @@ the `-file` flag._
5757
-h, --help show this help message and exit
5858
-fvp Set this flag to process as FVP data
5959
-ivp Set this flag to process as IVP data
60-
-tfp Set this flag to process as Telephone Followup Packet data
60+
-tfp Set this flag to process as Telephone Followup Packet v3.2 data
61+
-tfp3 Set this flag to process as TFP v3.0 (pre-2020) data
6162
-np Set this flag to process as Neuropathology data
6263
-m Set this flag to process as Milestone data
6364
-csf Set this flag to process as NACC BIDSS CSF data

‎nacc/ftld/fvp/builder.py

+2
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,8 @@ def update_header(record, packet):
472472
for header in packet:
473473
header.PACKET = "FF"
474474
header.FORMID = header.form_name
475+
if header.FORMID == "Z1X":
476+
header.PACKET = "F"
475477
header.FORMVER = 3
476478
header.ADCID = record['adcid']
477479
header.PTID = record['ptid']

‎nacc/ftld/ivp/builder.py

+2
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,8 @@ def update_header(record, packet):
473473
for header in packet:
474474
header.PACKET = "IF"
475475
header.FORMID = header.form_name
476+
if header.FORMID == "Z1X":
477+
header.PACKET = "I"
476478
header.FORMVER = 3
477479
header.ADCID = record['adcid']
478480
header.PTID = record['ptid']

‎nacc/redcap2nacc.py

+81-35
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env python
22

33
###############################################################################
4-
# Copyright 2015-2020 University of Florida. All rights reserved.
4+
# Copyright 2015-2021 University of Florida. All rights reserved.
55
# This file is part of UF CTS-IT's NACCulator project.
66
# Use of this source code is governed by the license found in the LICENSE file.
77
###############################################################################
@@ -21,6 +21,7 @@
2121
from nacc.uds3.np import builder as np_builder
2222
from nacc.uds3.fvp import builder as fvp_builder
2323
from nacc.uds3.tfp import builder as tfp_builder
24+
from nacc.uds3.tfp.v3_2 import builder as tfp_new_builder
2425
from nacc.uds3.m import builder as m_builder
2526
from nacc.lbd.ivp import builder as lbd_ivp_builder
2627
from nacc.lbd.fvp import builder as lbd_fvp_builder
@@ -216,7 +217,26 @@ def check_redcap_event(options, record) -> bool:
216217
elif options.np:
217218
event_name = 'neuropath'
218219
elif options.tfp:
219-
event_name = 'telephone'
220+
event_name = 'follow'
221+
try:
222+
followup_match = record['tvp_z1x_checklist_complete']
223+
if followup_match in ['', '0']:
224+
return False
225+
except KeyError:
226+
try:
227+
followup_match = record['tfp_z1x_complete']
228+
if followup_match in ['', '0']:
229+
return False
230+
except KeyError:
231+
try:
232+
followup_match = record['tele_z1x_complete']
233+
if followup_match in ['', '0']:
234+
return False
235+
except KeyError:
236+
print("Could not find a REDCap field for TFP Z1X form.")
237+
return False
238+
elif options.tfp3:
239+
event_name = 'tele'
220240
elif options.m:
221241
event_name = 'milestone'
222242

@@ -284,44 +304,66 @@ def set_to_zero_if_blank(*field_names):
284304
field.value = 0
285305

286306
# B8 2.
287-
if packet['PARKSIGN'] == 1:
288-
set_to_zero_if_blank(
289-
'RESTTRL', 'RESTTRR', 'SLOWINGL', 'SLOWINGR', 'RIGIDL', 'RIGIDR',
290-
'BRADY', 'PARKGAIT', 'POSTINST')
307+
try:
308+
if packet['PARKSIGN'] == 1:
309+
set_to_zero_if_blank(
310+
'RESTTRL', 'RESTTRR', 'SLOWINGL', 'SLOWINGR', 'RIGIDL', 'RIGIDR',
311+
'BRADY', 'PARKGAIT', 'POSTINST')
312+
except KeyError:
313+
pass
291314

292315
# B8 3.
293-
if packet['CVDSIGNS'] == 1:
294-
set_to_zero_if_blank('CORTDEF', 'SIVDFIND', 'CVDMOTL', 'CVDMOTR',
295-
'CORTVISL', 'CORTVISR', 'SOMATL', 'SOMATR')
316+
try:
317+
if packet['CVDSIGNS'] == 1:
318+
set_to_zero_if_blank('CORTDEF', 'SIVDFIND', 'CVDMOTL', 'CVDMOTR',
319+
'CORTVISL', 'CORTVISR', 'SOMATL', 'SOMATR')
320+
except KeyError:
321+
pass
296322

297323
# B8 5.
298-
if packet['PSPCBS'] == 1:
299-
set_to_zero_if_blank(
300-
'PSPCBS', 'EYEPSP', 'DYSPSP', 'AXIALPSP', 'GAITPSP', 'APRAXSP',
301-
'APRAXL', 'APRAXR', 'CORTSENL', 'CORTSENR', 'ATAXL', 'ATAXR',
302-
'ALIENLML', 'ALIENLMR', 'DYSTONL', 'DYSTONR', 'MYOCLLT', 'MYOCLRT')
324+
try:
325+
if packet['PSPCBS'] == 1:
326+
set_to_zero_if_blank(
327+
'PSPCBS', 'EYEPSP', 'DYSPSP', 'AXIALPSP', 'GAITPSP', 'APRAXSP',
328+
'APRAXL', 'APRAXR', 'CORTSENL', 'CORTSENR', 'ATAXL', 'ATAXR',
329+
'ALIENLML', 'ALIENLMR', 'DYSTONL', 'DYSTONR', 'MYOCLLT',
330+
'MYOCLRT')
331+
except KeyError:
332+
pass
303333

304334
# D1 4.
305-
if packet['DEMENTED'] == 1:
306-
set_to_zero_if_blank(
307-
'AMNDEM', 'PCA', 'PPASYN', 'FTDSYN', 'LBDSYN', 'NAMNDEM')
335+
try:
336+
if packet['DEMENTED'] == 1:
337+
set_to_zero_if_blank(
338+
'AMNDEM', 'PCA', 'PPASYN', 'FTDSYN', 'LBDSYN', 'NAMNDEM')
339+
except KeyError:
340+
pass
308341

309342
# D1 5.
310-
if packet['DEMENTED'] == 0:
311-
set_to_zero_if_blank(
312-
'MCIAMEM', 'MCIAPLUS', 'MCINON1', 'MCINON2', 'IMPNOMCI')
343+
try:
344+
if packet['DEMENTED'] == 0:
345+
set_to_zero_if_blank(
346+
'MCIAMEM', 'MCIAPLUS', 'MCINON1', 'MCINON2', 'IMPNOMCI')
347+
except KeyError:
348+
pass
313349

314350
# D1 11-39.
315-
set_to_zero_if_blank(
316-
'ALZDIS', 'LBDIS', 'MSA', 'PSP', 'CORT', 'FTLDMO', 'FTLDNOS', 'CVD',
317-
'ESSTREM', 'DOWNS', 'HUNT', 'PRION', 'BRNINJ', 'HYCEPH', 'EPILEP',
318-
'NEOP', 'HIV', 'OTHCOG', 'DEP', 'BIPOLDX', 'SCHIZOP', 'ANXIET',
319-
'DELIR', 'PTSDDX', 'OTHPSY', 'ALCDEM', 'IMPSUB', 'DYSILL', 'MEDS',
320-
'COGOTH', 'COGOTH2', 'COGOTH3')
351+
try:
352+
set_to_zero_if_blank(
353+
'ALZDIS', 'LBDIS', 'MSA', 'PSP', 'CORT', 'FTLDMO', 'FTLDNOS', 'CVD',
354+
'ESSTREM', 'DOWNS', 'HUNT', 'PRION', 'BRNINJ', 'HYCEPH', 'EPILEP',
355+
'NEOP', 'HIV', 'OTHCOG', 'DEP', 'BIPOLDX', 'SCHIZOP', 'ANXIET',
356+
'DELIR', 'PTSDDX', 'OTHPSY', 'ALCDEM', 'IMPSUB', 'DYSILL', 'MEDS',
357+
'COGOTH', 'COGOTH2', 'COGOTH3')
358+
except KeyError:
359+
pass
321360

322361
# D2 11.
323-
if packet['ARTH'] == 1:
324-
set_to_zero_if_blank('ARTUPEX', 'ARTLOEX', 'ARTSPIN', 'ARTUNKN')
362+
try:
363+
if packet['ARTH'] == 1:
364+
set_to_zero_if_blank('ARTUPEX', 'ARTLOEX', 'ARTSPIN', 'ARTUNKN')
365+
except KeyError:
366+
pass
325367

326368

327369
def convert(fp, options, out=sys.stdout, err=sys.stderr):
@@ -358,6 +400,8 @@ def convert(fp, options, out=sys.stdout, err=sys.stderr):
358400
elif options.fvp:
359401
packet = fvp_builder.build_uds3_fvp_form(record)
360402
elif options.tfp:
403+
packet = tfp_new_builder.build_uds3_tfp_new_form(record)
404+
elif options.tfp3:
361405
packet = tfp_builder.build_uds3_tfp_form(record)
362406
elif options.m:
363407
packet = m_builder.build_uds3_m_form(record)
@@ -369,12 +413,11 @@ def convert(fp, options, out=sys.stdout, err=sys.stderr):
369413
traceback.print_exc()
370414
continue
371415

372-
if not options.np and not options.m and not options.tfp and not \
373-
options.lbd and not options.lbdsv and not options.ftld and not \
374-
options.csf:
416+
if not (options.np or options.m or options.lbd or options.lbdsv or
417+
options.ftld or options.csf):
375418
set_blanks_to_zero(packet)
376419

377-
if options.m:
420+
if options.m or options.tfp:
378421
blanks_uds3.set_zeros_to_blanks(packet)
379422

380423
warnings = []
@@ -439,7 +482,10 @@ def parse_args(args=None):
439482
help='Set this flag to process as ivp data')
440483
option_group.add_argument(
441484
'-tfp', action='store_true', dest='tfp',
442-
help='Set this flag to process as tfp data')
485+
help='Set this flag to process as tfp version 3.2 data')
486+
option_group.add_argument(
487+
'-tfp3', action='store_true', dest='tfp3',
488+
help='Set this flag to process as tfp version 3.0 (pre-June 2020) data')
443489
option_group.add_argument(
444490
'-np', action='store_true', dest='np',
445491
help='Set this flag to process as np data')
@@ -483,8 +529,8 @@ def parse_args(args=None):
483529
options = parser.parse_args(args)
484530
# Defaults to processing of ivp.
485531
# TODO this can be changed in future to process fvp by default.
486-
if not (options.ivp or options.fvp or options.tfp or options.np or
487-
options.m or options.csf or options.filter):
532+
if not (options.ivp or options.fvp or options.tfp or options.tfp3 or
533+
options.np or options.m or options.csf or options.filter):
488534
options.ivp = True
489535

490536
return options

‎nacc/uds3/__init__.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
###############################################################################
2-
# Copyright 2015-2020 University of Florida. All rights reserved.
2+
# Copyright 2015-2021 University of Florida. All rights reserved.
33
# This file is part of UF CTS-IT's NACCulator project.
44
# Use of this source code is governed by the license found in the LICENSE file.
55
###############################################################################

‎nacc/uds3/blanks.py

+35-15
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
###############################################################################
2-
# Copyright 2015-2016 University of Florida. All rights reserved.
2+
# Copyright 2015-2021 University of Florida. All rights reserved.
33
# This file is part of UF CTS-IT's NACCulator project.
44
# Use of this source code is governed by the license found in the LICENSE file.
55
###############################################################################
@@ -53,13 +53,15 @@ def convert_rule_to_python(name: str, rule: str) -> bool:
5353
'NPPATH9': _blanking_rule_dummy,
5454
'NPPATH10': _blanking_rule_dummy,
5555
'NPPATH11': _blanking_rule_dummy,
56+
# TFP 3.2 skip rules
57+
'TELMILE': _blanking_rule_telmile,
5658
}
5759

5860
single_value = re.compile(
59-
r"Blank if( Question(s?))? *\w+ (?P<key>\w+) *(?P<eq>=|ne)"
61+
r"Blank if( Question(s?))? *\w+\.? (?P<key>\w+) *(?P<eq>=|ne)"
6062
r" (?P<value>\d+)([^-]|$)")
6163
range_values = re.compile(
62-
r"Blank if( Question(s?))? *\w+ (?P<key>\w+) *(?P<eq>=|ne)"
64+
r"Blank if( Question(s?))? *\w+\.? (?P<key>\w+) *(?P<eq>=|ne)"
6365
r" (?P<start>\d+)-(?P<stop>\d+)( |$)")
6466

6567
# First, check to see if the rule is a "Special Case"
@@ -140,6 +142,13 @@ def _blanking_rule_learned():
140142
return lambda packet: packet['REFERSC'] in (3, 4, 5, 6, 8, 9)
141143

142144

145+
def _blanking_rule_telmile():
146+
# 'Blank if Question 3 TELINPER = 1 (Yes)'
147+
# 'Blank if Question 3 TELINPER = 9 (Unknown)'
148+
# 'Blank if this is the first telephone packet submitted for the subject.'
149+
return lambda packet: packet['TELINPER'] in (1, 9)
150+
151+
143152
def set_zeros_to_blanks(packet):
144153
""" Sets specific fields to zero if they meet certain criteria """
145154
def set_to_blank_if_zero(*field_names):
@@ -148,18 +157,29 @@ def set_to_blank_if_zero(*field_names):
148157
if field == 0:
149158
field.value = ''
150159
# M1
151-
if packet['DECEASED'] == 1 or packet['DISCONT'] == 1:
152-
set_to_blank_if_zero(
153-
'RENURSE', 'RENAVAIL', 'RECOGIM', 'REJOIN', 'REPHYILL',
154-
'REREFUSE', 'FTLDDISC', 'CHANGEMO', 'CHANGEDY', 'CHANGEYR',
155-
'PROTOCOL', 'ACONSENT', 'RECOGIM', 'REPHYILL', 'NURSEMO',
156-
'NURSEDY', 'NURSEYR', 'FTLDREAS', 'FTLDREAX')
157-
elif packet['DECEASED'] == 1:
158-
# for just dead
159-
set_to_blank_if_zero('DISCONT')
160-
elif packet['DISCONT'] == 1:
161-
# for just discont
162-
set_to_blank_if_zero('DECEASED')
160+
try:
161+
if packet['DECEASED'] == 1 or packet['DISCONT'] == 1:
162+
set_to_blank_if_zero(
163+
'RENURSE', 'RENAVAIL', 'RECOGIM', 'REJOIN', 'REPHYILL',
164+
'REREFUSE', 'FTLDDISC', 'CHANGEMO', 'CHANGEDY', 'CHANGEYR',
165+
'PROTOCOL', 'ACONSENT', 'RECOGIM', 'REPHYILL', 'NURSEMO',
166+
'NURSEDY', 'NURSEYR', 'FTLDREAS', 'FTLDREAX')
167+
elif packet['DECEASED'] == 1:
168+
# for just dead
169+
set_to_blank_if_zero('DISCONT')
170+
elif packet['DISCONT'] == 1:
171+
# for just discont
172+
set_to_blank_if_zero('DECEASED')
173+
except KeyError:
174+
pass
175+
# TFP
176+
try:
177+
if packet['RESPVAL'] == 1:
178+
set_to_blank_if_zero(
179+
'RESPHEAR', 'RESPDIST', 'RESPINTR', 'RESPDISN', 'RESPFATG',
180+
'RESPEMOT', 'RESPASST', 'RESPOTH')
181+
except KeyError:
182+
pass
163183

164184

165185
def main():

‎nacc/uds3/filters.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ def filter_replace_drug_id_do(input_ptr, output_ptr):
140140
write_headers(reader, output)
141141
for record in reader:
142142
count = 0
143-
prefixes = ['', 'fu_']
143+
prefixes = ['', 'fu_', 'tele_']
144144
for prefix in prefixes:
145145
for i in range(1, 31):
146146
col_name = prefix + 'drugid_' + str(i)

‎nacc/uds3/tfp/v3_2/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)