Merge branch 'main' into nibabies-24

.circleci/config.yml

@@ -5,7 +5,7 @@ orbs:
 .dockersetup:
   &dockersetup
   docker:
-    - image: pennlinc/xcp_d_build:0.0.10
+    - image: pennlinc/xcp_d_build:0.0.11
   working_directory: /src/xcp_d
 
 runinstall:
@@ -73,6 +73,25 @@ jobs:
     # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
     resource_class: large
 
+  download_data_ukbiobank:
+    <<: *dockersetup
+    steps:
+      - checkout
+      - restore_cache:
+          key: ukbiobank-08
+      - run: *runinstall
+      - run:
+          name: Download ukbiobank test data
+          command: |
+            cd /src/xcp_d/.circleci
+            python get_data.py $PWD/data ukbiobank
+      - save_cache:
+          key: ukbiobank-08
+          paths:
+            - /src/xcp_d/.circleci/data/ukbiobank
+    # The resource_class feature allows configuring CPU and RAM resources for each job. Different resource classes are available for different executors. https://circleci.com/docs/2.0/configuration-reference/#resourceclass
+    resource_class: large
+
   download_data_fmriprepwithoutfreesurfer:
     <<: *dockersetup
     steps:
@@ -200,36 +219,36 @@ jobs:
       - store_artifacts:
           path: /src/xcp_d/.circleci/out/test_ds001419_cifti/xcp_d/
 
-  pnc_nifti:
+  ukbiobank:
     <<: *dockersetup
     steps:
       - checkout
       - run:
           name: Check whether build should be skipped
           command: |
             cd /src/xcp_d
-            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?pnc_nifti\]' )" != "" ]]; then
-              echo "Skipping pnc_nifti build"
+            if [[ "$( git log --format=oneline -n 1 $CIRCLE_SHA1 | grep -i -E '\[skip[ _]?ukbiobank\]' )" != "" ]]; then
+              echo "Skipping ukbiobank build"
               circleci step halt
             fi
       - restore_cache:
-          key: pnc-02
+          key: ukbiobank-08
       - run: *runinstall
       - run:
           name: Run full xcp_d on nifti with freesurfer
           no_output_timeout: 1h
           command: |
-            pytest -rP -o log_cli=true -m "pnc_nifti" --cov-append --cov-report term-missing --cov=xcp_d --data_dir=/src/xcp_d/.circleci/data --output_dir=/src/xcp_d/.circleci/out --working_dir=/src/xcp_d/.circleci/work xcp_d
+            pytest -rP -o log_cli=true -m "ukbiobank" --cov-append --cov-report term-missing --cov=xcp_d --data_dir=/src/xcp_d/.circleci/data --output_dir=/src/xcp_d/.circleci/out --working_dir=/src/xcp_d/.circleci/work xcp_d
             mkdir /src/coverage
-            mv /src/xcp_d/.coverage /src/coverage/.coverage.pnc_nifti
+            mv /src/xcp_d/.coverage /src/coverage/.coverage.ukbiobank
             # remove nifti files before uploading artifacts
             find /src/xcp_d/.circleci/out/ -name "*.nii.gz" -type f -delete
       - persist_to_workspace:
           root: /src/coverage/
           paths:
-            - .coverage.pnc_nifti
+            - .coverage.ukbiobank
       - store_artifacts:
-          path: /src/xcp_d/.circleci/out/test_pnc_nifti/xcp_d/
+          path: /src/xcp_d/.circleci/out/test_ukbiobank/xcp_d/
 
   nibabies:
     <<: *dockersetup
@@ -450,6 +469,13 @@ workflows:
             tags:
               only: /.*/
 
+      - download_data_ukbiobank:
+          requires:
+            - build
+          filters:
+            tags:
+              only: /.*/
+
       - download_data_fmriprepwithoutfreesurfer:
           requires:
             - build
@@ -486,9 +512,9 @@ workflows:
             tags:
               only: /.*/
 
-      - pnc_nifti:
+      - ukbiobank:
           requires:
-            - download_data_pnc
+            - download_data_ukbiobank
           filters:
             branches:
               ignore:
@@ -561,7 +587,7 @@ workflows:
             - pnc_cifti_t2wonly
             - ds001419_nifti
             - ds001419_cifti
-            - pnc_nifti
+            - ukbiobank
             - nibabies
             - nifti_without_freesurfer
             - pytests
@@ -577,7 +603,7 @@ workflows:
           requires:
             - ds001419_nifti
             - ds001419_cifti
-            - pnc_nifti
+            - ukbiobank
             - nifti_without_freesurfer
             - nibabies
             - pnc_cifti

Dockerfile

@@ -1,4 +1,4 @@
-FROM pennlinc/xcp_d_build:0.0.10
+FROM pennlinc/xcp_d_build:0.0.11
 
 # Install xcp_d
 COPY . /src/xcp_d

docs/api.rst

@@ -88,8 +88,6 @@ xcp_d-combineqc
    xcp_d.utils.bids
    xcp_d.utils.concatenation
    xcp_d.utils.confounds
-   xcp_d.utils.dcan2fmriprep
-   xcp_d.utils.hcp2fmriprep
    xcp_d.utils.doc
    xcp_d.utils.execsummary
    xcp_d.utils.filemanip
@@ -100,3 +98,23 @@ xcp_d-combineqc
    xcp_d.utils.sentry
    xcp_d.utils.utils
    xcp_d.utils.write_save
+
+
+***********************************************************************
+:mod:`xcp_d.ingression`: Functions to Ingress Preprocessing Derivatives
+***********************************************************************
+
+.. automodule:: xcp_d.ingression
+   :no-members:
+   :no-inherited-members:
+
+.. currentmodule:: xcp_d
+
+.. autosummary::
+   :toctree: generated/
+   :template: module.rst
+
+   xcp_d.ingression.abcdbids
+   xcp_d.ingression.hcpya
+   xcp_d.ingression.ukbiobank
+   xcp_d.ingression.utils

docs/outputs.rst

@@ -18,7 +18,7 @@ The  *XCP-D* outputs are written out in BIDS format and consist of three main pa
    `BEP012: Functional preprocessing derivatives <https://github.com/bids-standard/bids-specification/pull/519>`_,
    `BEP017: BIDS connectivity matrix data schema <https://docs.google.com/document/d/1ugBdUF6dhElXdj3u9vw0iWjE6f_Bibsro3ah7sRV0GA/edit?usp=sharing>`_,
    and
-   `BEPXXX: Atlas Specification <https://docs.google.com/document/d/1RxW4cARr3-EiBEcXjLpSIVidvnUSHE7yJCUY91i5TfM/edit?usp=sharing>`_
+   `BEP038: Atlas Specification <https://docs.google.com/document/d/1RxW4cARr3-EiBEcXjLpSIVidvnUSHE7yJCUY91i5TfM/edit?usp=sharing>`_
    (currently unnumbered).
 
    In cases where a derivative type is not covered by an existing BEP,
@@ -72,6 +72,22 @@ The 4S atlas is used in the same manner across three PennLINC BIDS Apps:
 XCP-D, QSIPrep_, and ASLPrep_, to produce synchronized outputs across modalities.
 For more information about the 4S atlas, please see https://github.com/PennLINC/AtlasPack.
 
+Atlases are written out to the ``atlases`` subfolder, following BEP038.
+
+.. code-block::
+
+   xcp_d/
+      atlases/
+         atlas-<label>/
+            atlas-<label>_dseg.json
+            atlas-<label>_dseg.tsv
+
+            # NIfTI
+            space-<label>_atlas-<label>_dseg.nii.gz
+
+            # CIFTI
+            space-<label>_atlas-<label>_dseg.dlabel.nii
+
 
 ******************
 Anatomical Outputs
@@ -208,12 +224,6 @@ This includes the atlases used to extract the timeseries.
 .. code-block::
 
    xcp_d/
-      # NIfTI
-      space-<label>_atlas-<label>_dseg.nii.gz
-
-      # CIFTI
-      space-<label>_atlas-<label>_dseg.dlabel.nii
-
       sub-<label>/[ses-<label>/]
          func/
             # NIfTI

docs/workflows.rst

@@ -246,6 +246,88 @@ For more information about confound regressor selection, please refer to :footci
    options.
 
 
+.. list-table:: Preprocessing Pipeline Support
+
+   *  - Nuisance Strategy
+      - 24P
+      - 27P
+      - 36P
+      - acompcor
+      - acompcor_gsr
+      - aroma
+      - aroma_gsr
+      - gsr_only
+      - none
+   *  - fMRIPrep (>=23.1.0)
+      - X
+      - X
+      - X
+      - X
+      - X
+      -
+      -
+      - X
+      - X
+   *  - fMRIPrep (<23.1.0)
+      - X
+      - X
+      - X
+      - X
+      - X
+      - X
+      - X
+      - X
+      - X
+   *  - Nibabies
+      - X
+      - X
+      - X
+      - X
+      - X
+      -
+      -
+      - X
+      - X
+   *  - ABCD-BIDS (DCAN)
+      - X
+      - X
+      - X
+      -
+      -
+      -
+      -
+      - X
+      - X
+   *  - HCP-YA
+      - X
+      - X
+      - X
+      -
+      -
+      -
+      -
+      - X
+      - X
+   *  - UK Biobank
+      - X
+      -
+      -
+      -
+      -
+      -
+      -
+      - X
+      - X
+
+.. important::
+   fMRIPrep removed AROMA support in 23.1.0.
+   In the future, there will be an fMRIPost-AROMA BIDS App that runs AROMA on fMRIPrep outputs.
+
+.. warning::
+   The strategy ``gsr_only`` is only appropriate for UK Biobank data,
+   as those data have already been denoised with FSL FIX.
+
+
 Dummy scan removal [OPTIONAL]
 =============================
 :func:`~xcp_d.workflows.postprocessing.init_prepare_confounds_wf`,

pyproject.toml

@@ -165,11 +165,11 @@ per-file-ignores = [
 ]
 
 [tool.pytest.ini_options]
-addopts = '-m "not ds001419_nifti and not ds001419_cifti and not pnc_nifti and not pnc_cifti and not pnc_cifti_t2wonly and not fmriprep_without_freesurfer and not nibabies"'
+addopts = '-m "not ds001419_nifti and not ds001419_cifti and not ukbiobank and not pnc_cifti and not pnc_cifti_t2wonly and not fmriprep_without_freesurfer and not nibabies"'
 markers = [
     "ds001419_nifti: mark NIfTI integration test for fMRIPrep derivatives from ds001419",
     "ds001419_cifti: mark CIFTI integration test for fMRIPrep derivatives from ds001419",
-    "pnc_nifti: mark integration test for fMRIPrep derivatives with NIFTI settings",
+    "ukbiobank: mark integration test for UK Biobank derivatives with NIfTI settings",
     "pnc_cifti: mark integration test for fMRIPrep derivatives with CIFTI settings",
     "pnc_cifti_t2wonly: mark integration test for fMRIPrep derivatives with CIFTI settings and a simulated T2w file",
     "fmriprep_without_freesurfer: mark integration test for fMRIPrep derivatives without FreeSurfer",

xcp_d/cli/run.py

@@ -175,11 +175,11 @@ def get_parser():
         "--input_type",
         required=False,
         default="fmriprep",
-        choices=["fmriprep", "dcan", "hcp", "nibabies"],
+        choices=["fmriprep", "dcan", "hcp", "nibabies", "ukb"],
         help=(
             "The pipeline used to generate the preprocessed derivatives. "
             "The default pipeline is 'fmriprep'. "
-            "The 'dcan', 'hcp', and 'nibabies' pipelines are also supported. "
+            "The 'dcan', 'hcp', 'nibabies', and 'ukb' pipelines are also supported. "
             "'nibabies' assumes the same structure as 'fmriprep'."
         ),
     )
@@ -218,6 +218,8 @@ def get_parser():
             "aroma_gsr",
             "custom",
             "none",
+            # GSR-only for UKB
+            "gsr_only",
         ],
         default="36P",
         type=str,
@@ -797,6 +799,21 @@ def _validate_parameters(opts, build_log):
             )
             opts.process_surfaces = True
 
+    elif opts.input_type == "ukb":
+        if opts.cifti:
+            build_log.warning(
+                f"With input_type {opts.input_type}, cifti processing (--cifti) will be "
+                "disabled automatically."
+            )
+            opts.cifti = False
+
+        if opts.process_surfaces:
+            build_log.warning(
+                f"With input_type {opts.input_type}, surface normalization "
+                "(--warp-surfaces-native2std) will be disabled automatically."
+            )
+            opts.process_surfaces = False
+
     # process_surfaces and nifti processing are incompatible.
     if opts.process_surfaces and not opts.cifti:
         build_log.error(
@@ -852,11 +869,13 @@ def build_workflow(opts, retval):
     retval["work_dir"] = str(opts.work_dir)
 
     # First check that fmriprep_dir looks like a BIDS folder
-    if opts.input_type in ("dcan", "hcp"):
+    if opts.input_type in ("dcan", "hcp", "ukb"):
         if opts.input_type == "dcan":
-            from xcp_d.utils.dcan2fmriprep import convert_dcan2bids as convert_to_bids
+            from xcp_d.ingression.abcdbids import convert_dcan2bids as convert_to_bids
         elif opts.input_type == "hcp":
-            from xcp_d.utils.hcp2fmriprep import convert_hcp2bids as convert_to_bids
+            from xcp_d.ingression.hcpya import convert_hcp2bids as convert_to_bids
+        elif opts.input_type == "ukb":
+            from xcp_d.ingression.ukbiobank import convert_ukb2bids as convert_to_bids
 
         NIWORKFLOWS_LOG.info(f"Converting {opts.input_type} to fmriprep format")
         converted_fmri_dir = os.path.join(

xcp_d/data/README

@@ -0,0 +1,8 @@
+MNI152_T1_2mm.nii.gz comes from the FSL standard templates folder.
+I copied it here to use in the UK Biobank ingression step,
+instead of TemplateFlow's tpl-MNI152NLin6Asym_res-2 template,
+because FSL's version is in LAS+, while TemplateFlow's version is in RAS+.
+
+FSL apparently can't handle differently-oriented files,
+and really only works with LAS+ files (yay),
+so it just seems easier to have an LAS+ copy of the template on hand.

xcp_d/data/boilerplate.bib

@@ -725,6 +725,17 @@ @article{najdenovska2018vivo
   doi={10.1038/sdata.2018.270}
 }
 
+@article{miller2016multimodal,
+  title={Multimodal population brain imaging in the UK Biobank prospective epidemiological study},
+  author={Miller, Karla L and Alfaro-Almagro, Fidel and Bangerter, Neal K and Thomas, David L and Yacoub, Essa and Xu, Junqian and Bartsch, Andreas J and Jbabdi, Saad and Sotiropoulos, Stamatios N and Andersson, Jesper LR and others},
+  journal={Nature neuroscience},
+  volume={19},
+  number={11},
+  pages={1523--1536},
+  year={2016},
+  publisher={Nature Publishing Group US New York}
+}
+
 @article{taylorlomb,
   title={Lomb-Scargle your way to RSFC parameter estimation in AFNI-FATCAT},
   author={Taylor, Paul A and Chen, Gang and Glen, Daniel R and Reynolds, Richard C and Cox, Robert W},