RFC95: Generating Study data files

.github/workflows/integration-test.yml

@@ -45,7 +45,9 @@ jobs:
           cat $PORTAL_SOURCE_DIR/src/main/resources/application.properties | \
               sed 's|spring.datasource.url=.*|spring.datasource.url=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false|' | \
               sed 's|spring.datasource.username=.*|spring.datasource.username=cbio_user|' | \
-              sed 's|spring.datasource.password=.*|spring.datasource.password=somepassword|' \
+              sed 's|spring.datasource.password=.*|spring.datasource.password=somepassword|' | \
+              sed 's|session.service.url=.*|session.service.url=http://cbioportal-session:5001/api/sessions/my_portal/|' | \
+              sed 's|dynamic_study_export_mode=.*|dynamic_study_export_mode=true|' \
               > application.properties
       - name: 'Copy cgds.sql file into Docker Compose'
         run: cp ./cbioportal/src/main/resources/db-scripts/cgds.sql ./cbioportal-docker-compose/data/.
@@ -73,6 +75,11 @@ jobs:
         working-directory: ./cbioportal-docker-compose
         run: |
           $PORTAL_SOURCE_DIR/test/integration/test_load_study.sh
+      - name: 'TEST - Import and Export of study_es_0_import_export'
+        if: steps.startup.conclusion == 'success'
+        working-directory: ./cbioportal-docker-compose
+        run: |
+          $PORTAL_SOURCE_DIR/test/integration/test_import_export.sh
       - name: 'TEST - Add OncoKB annotations to study'
         if: steps.startup.conclusion == 'success'
         working-directory: ./cbioportal-docker-compose

pom.xml

@@ -182,6 +182,10 @@
 			<artifactId>mybatis-spring-boot-starter</artifactId>
 			<version>${mybatis.starter.version}</version>
 		</dependency>
+		<dependency>
+			<groupId>com.zaxxer</groupId>
+			<artifactId>HikariCP</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>org.redisson</groupId>
 			<artifactId>redisson</artifactId>

src/main/java/org/cbioportal/application/file/README.md

@@ -0,0 +1,15 @@
+# Study Data Export
+
+This package contains the code for exporting study data from the database to a file format.  The export process involves several steps, including:
+1. Retrieving the study data from the database.
+2. Transforming the data into a suitable format for export.
+3. Writing the transformed data to a file.
+
+The implementation is done with minimum dependencies on the rest of the code to ensure that the code is lightweight, performant and easy to move to a separate web application if needed.
+To make export process take less RAM, the code uses a streaming approach to read and write data. On the database side, the code uses a cursor to read data in chunks, and on the web controller side, the code uses a streaming response to write data in chunks.
+This allows the code to handle large datasets without running out of memory. 
+
+## Usage
+
+Set `dynamic_study_export_mode` to `true` in the application properties file to enable the dynamic study export mode. 
+This mode allows the user to export study with `/export/study/{studyId}.zip` link.

src/main/java/org/cbioportal/application/file/export/ExportController.java

@@ -0,0 +1,48 @@
+package org.cbioportal.application.file.export;
+
+import org.cbioportal.application.file.export.exporters.ExportDetails;
+import org.cbioportal.application.file.export.services.CancerStudyMetadataService;
+import org.cbioportal.application.file.export.services.ExportService;
+import org.cbioportal.application.file.export.services.VirtualStudyAwareExportService;
+import org.cbioportal.application.file.utils.ZipOutputStreamWriterFactory;
+import org.cbioportal.legacy.utils.config.annotation.ConditionalOnProperty;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody;
+
+import java.io.BufferedOutputStream;
+
+@RestController
+//How to have only one conditional on property in the config only
+// https://stackoverflow.com/questions/62355615/define-a-spring-restcontroller-via-java-configuration
+@ConditionalOnProperty(name = "dynamic_study_export_mode", havingValue = "true")
+public class ExportController {
+
+    private final VirtualStudyAwareExportService exportService;
+
+    public ExportController(VirtualStudyAwareExportService exportService) {
+        this.exportService = exportService;
+    }
+
+    @GetMapping("/export/study/{studyId}.zip")
+    public ResponseEntity<StreamingResponseBody> downloadStudyData(@PathVariable String studyId) throws Exception {
+        if (!exportService.isStudyExportable(studyId)) {
+            return ResponseEntity.notFound().build();
+        }
+
+        StreamingResponseBody stream = outputStream -> {
+            try (BufferedOutputStream bos = new BufferedOutputStream(outputStream);
+                 ZipOutputStreamWriterFactory zipFactory = new ZipOutputStreamWriterFactory(bos)) {
+                exportService.exportData(zipFactory, new ExportDetails(studyId));
+            }
+        };
+
+        return ResponseEntity.ok()
+            .contentType(new MediaType("application", "zip"))
+            .header("Content-Disposition", "attachment; filename=\"" + studyId + ".zip\"")
+            .body(stream);
+    }
+}

src/main/java/org/cbioportal/application/file/export/exporters/CancerStudyMetadataExporter.java

@@ -0,0 +1,46 @@
+package org.cbioportal.application.file.export.exporters;
+
+import org.cbioportal.application.file.export.services.CancerStudyMetadataService;
+import org.cbioportal.application.file.model.CancerStudyMetadata;
+
+import java.util.Optional;
+import java.util.SequencedMap;
+
+/**
+ * Exports metadata for a cancer study
+ */
+public class CancerStudyMetadataExporter extends MetadataExporter<CancerStudyMetadata> {
+
+    private final CancerStudyMetadataService cancerStudyMetadataService;
+
+    public CancerStudyMetadataExporter(CancerStudyMetadataService cancerStudyMetadataService) {
+        this.cancerStudyMetadataService = cancerStudyMetadataService;
+    }
+
+    @Override
+    public String getMetaFilename(CancerStudyMetadata metadata) {
+        return "meta_study.txt";
+    }
+
+    @Override
+    protected Optional<CancerStudyMetadata> getMetadata(String studyId) {
+        return Optional.ofNullable(cancerStudyMetadataService.getCancerStudyMetadata(studyId));
+    }
+
+    @Override
+    protected void updateMetadata(ExportDetails exportDetails, SequencedMap<String, String> metadataSeqMap) {
+        super.updateMetadata(exportDetails, metadataSeqMap);
+        // used primarily while downloading a Virtual Study
+        CancerStudyMetadata alternativeCancerStudyMetadata = new CancerStudyMetadata();
+        alternativeCancerStudyMetadata.setCancerStudyIdentifier(exportDetails.getExportWithStudyId());
+        alternativeCancerStudyMetadata.setName(exportDetails.getExportWithStudyName());
+        alternativeCancerStudyMetadata.setDescription(exportDetails.getExportAsStudyDescription());
+        alternativeCancerStudyMetadata.setPmid(exportDetails.getExportWithStudyPmid());
+        alternativeCancerStudyMetadata.setTypeOfCancer(exportDetails.getExportWithStudyTypeOfCancerId());
+        alternativeCancerStudyMetadata.toMetadataKeyValues().forEach((key, value) -> {
+            if (value != null && metadataSeqMap.containsKey(key)) {
+                metadataSeqMap.put(key, value);
+            }
+        });
+    }
+}

src/main/java/org/cbioportal/application/file/export/exporters/CancerTypeDataTypeExporter.java

@@ -0,0 +1,61 @@
+package org.cbioportal.application.file.export.exporters;
+
+import org.cbioportal.application.file.export.services.CancerStudyMetadataService;
+import org.cbioportal.application.file.model.CancerType;
+import org.cbioportal.application.file.model.ClinicalAttributesMetadata;
+import org.cbioportal.application.file.model.Table;
+import org.cbioportal.application.file.model.TableRow;
+import org.cbioportal.application.file.utils.CloseableIterator;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+public class CancerTypeDataTypeExporter extends DataTypeExporter<ClinicalAttributesMetadata, Table> {
+
+    private final CancerStudyMetadataService cancerStudyMetadataService;
+
+    public CancerTypeDataTypeExporter(CancerStudyMetadataService cancerStudyMetadataService) {
+        this.cancerStudyMetadataService = cancerStudyMetadataService;
+    }
+
+    @Override
+    protected Optional<ClinicalAttributesMetadata> getMetadata(String studyId, Set<String> sampleIds) {
+        return Optional.of(new ClinicalAttributesMetadata(
+            studyId,
+            "CANCER_TYPE",
+            "CANCER_TYPE"
+        ));
+    }
+
+    @Override
+    public String getDataFilename(ClinicalAttributesMetadata metadata) {
+        return "data_cancer_type.txt";
+    }
+
+    @Override
+    public String getMetaFilename(ClinicalAttributesMetadata metadata) {
+        return "meta_cancer_type.txt";
+    }
+
+    @Override
+    protected Table getData(String studyId, Set<String> sampleIds) {
+        List<CancerType> cancerTypes = cancerStudyMetadataService.getCancerTypeHierarchy(studyId);
+        var iterator = cancerTypes.iterator();
+        return new Table(new CloseableIterator<>() {
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public boolean hasNext() {
+                return iterator.hasNext();
+            }
+
+            @Override
+            public TableRow next() {
+                return iterator.next();
+            }
+        });
+    }
+}

src/main/java/org/cbioportal/application/file/export/exporters/CaseListsExporter.java

@@ -0,0 +1,62 @@
+package org.cbioportal.application.file.export.exporters;
+
+import org.cbioportal.application.file.export.services.CaseListMetadataService;
+import org.cbioportal.application.file.model.CaseListMetadata;
+import org.cbioportal.application.file.utils.FileWriterFactory;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.SequencedMap;
+
+/**
+ * Exports all case lists for a study
+ */
+public class CaseListsExporter implements Exporter {
+
+    private final CaseListMetadataService caseListMetadataService;
+
+    public CaseListsExporter(CaseListMetadataService caseListMetadataService) {
+        this.caseListMetadataService = caseListMetadataService;
+    }
+
+    @Override
+    public boolean exportData(FileWriterFactory fileWriterFactory, ExportDetails exportDetails) {
+        List<CaseListMetadata> caseLists = caseListMetadataService.getCaseListsMetadata(exportDetails.getStudyId(), exportDetails.getSampleIds());
+        boolean exported = false;
+        for (CaseListMetadata metadata : caseLists) {
+            exported |= new CaseListExporter(metadata).exportData(fileWriterFactory, exportDetails);
+        }
+        return exported;
+    }
+
+    /**
+     * Exports a case list metadata to a file
+     */
+    public static class CaseListExporter extends MetadataExporter<CaseListMetadata> {
+
+        private final CaseListMetadata caseListMetadata;
+
+        public CaseListExporter(CaseListMetadata caseListMetadata) {
+            this.caseListMetadata = caseListMetadata;
+        }
+
+        @Override
+        public String getMetaFilename(CaseListMetadata metadata) {
+            return "case_lists/cases_" + metadata.getCaseListTypeStableId() + ".txt";
+        }
+
+        @Override
+        protected Optional<CaseListMetadata> getMetadata(String studyId) {
+            return Optional.of(caseListMetadata);
+        }
+
+
+        @Override
+        protected void updateMetadata(ExportDetails exportDetails, SequencedMap<String, String> metadataSeqMap) {
+            super.updateMetadata(exportDetails, metadataSeqMap);
+            if (exportDetails.getExportWithStudyId() != null) {
+                metadataSeqMap.put("stable_id", exportDetails.getExportWithStudyId() + "_" + caseListMetadata.getCaseListTypeStableId());
+            }
+        }
+    }
+}

src/main/java/org/cbioportal/application/file/export/exporters/ClinicalPatientAttributesDataTypeExporter.java

@@ -0,0 +1,42 @@
+package org.cbioportal.application.file.export.exporters;
+
+import org.cbioportal.application.file.export.services.ClinicalAttributeDataService;
+import org.cbioportal.application.file.model.ClinicalAttribute;
+import org.cbioportal.application.file.model.ClinicalAttributeValue;
+import org.cbioportal.application.file.model.ClinicalAttributesMetadata;
+import org.cbioportal.application.file.model.ClinicalAttributesTable;
+import org.cbioportal.application.file.utils.CloseableIterator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Export metadata and data for clinical patient attributes.
+ */
+public class ClinicalPatientAttributesDataTypeExporter extends DataTypeExporter<ClinicalAttributesMetadata, ClinicalAttributesTable> {
+
+    private final ClinicalAttributeDataService clinicalDataAttributeDataService;
+
+    public ClinicalPatientAttributesDataTypeExporter(ClinicalAttributeDataService clinicalDataAttributeDataService) {
+        this.clinicalDataAttributeDataService = clinicalDataAttributeDataService;
+    }
+
+    @Override
+    protected Optional<ClinicalAttributesMetadata> getMetadata(String studyId, Set<String> sampleIds) {
+        if (!clinicalDataAttributeDataService.hasClinicalPatientAttributes(studyId, sampleIds)) {
+            return Optional.empty();
+        }
+        return Optional.of(new ClinicalAttributesMetadata(studyId, "CLINICAL", "PATIENT_ATTRIBUTES"));
+    }
+
+    @Override
+    protected ClinicalAttributesTable getData(String studyId, Set<String> sampleIds) {
+        List<ClinicalAttribute> clinicalPatientAttributes = new ArrayList<>();
+        clinicalPatientAttributes.add(ClinicalAttribute.PATIENT_ID);
+        clinicalPatientAttributes.addAll(clinicalDataAttributeDataService.getClinicalPatientAttributes(studyId));
+        CloseableIterator<ClinicalAttributeValue> clinicalPatientAttributeValues = clinicalDataAttributeDataService.getClinicalPatientAttributeValues(studyId, sampleIds);
+        return new ClinicalAttributesTable(clinicalPatientAttributes, clinicalPatientAttributeValues);
+    }
+}

src/main/java/org/cbioportal/application/file/export/exporters/ClinicalSampleAttributesDataTypeExporter.java

@@ -0,0 +1,43 @@
+package org.cbioportal.application.file.export.exporters;
+
+import org.cbioportal.application.file.export.services.ClinicalAttributeDataService;
+import org.cbioportal.application.file.model.ClinicalAttribute;
+import org.cbioportal.application.file.model.ClinicalAttributeValue;
+import org.cbioportal.application.file.model.ClinicalAttributesMetadata;
+import org.cbioportal.application.file.model.ClinicalAttributesTable;
+import org.cbioportal.application.file.utils.CloseableIterator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Export metadata and data for clinical sample attributes.
+ */
+public class ClinicalSampleAttributesDataTypeExporter extends DataTypeExporter<ClinicalAttributesMetadata, ClinicalAttributesTable> {
+
+    private final ClinicalAttributeDataService clinicalDataAttributeDataService;
+
+    public ClinicalSampleAttributesDataTypeExporter(ClinicalAttributeDataService clinicalDataAttributeDataService) {
+        this.clinicalDataAttributeDataService = clinicalDataAttributeDataService;
+    }
+
+    @Override
+    protected Optional<ClinicalAttributesMetadata> getMetadata(String studyId, Set<String> sampleIds) {
+        if (!clinicalDataAttributeDataService.hasClinicalSampleAttributes(studyId, sampleIds)) {
+            return Optional.empty();
+        }
+        return Optional.of(new ClinicalAttributesMetadata(studyId, "CLINICAL", "SAMPLE_ATTRIBUTES"));
+    }
+
+    @Override
+    protected ClinicalAttributesTable getData(String studyId, Set<String> sampleIds) {
+        List<ClinicalAttribute> clinicalSampleAttributes = new ArrayList<>();
+        clinicalSampleAttributes.add(ClinicalAttribute.PATIENT_ID);
+        clinicalSampleAttributes.add(ClinicalAttribute.SAMPLE_ID);
+        clinicalSampleAttributes.addAll(clinicalDataAttributeDataService.getClinicalSampleAttributes(studyId));
+        CloseableIterator<ClinicalAttributeValue> clinicalSampleAttributeValues = clinicalDataAttributeDataService.getClinicalSampleAttributeValues(studyId, sampleIds);
+        return new ClinicalAttributesTable(clinicalSampleAttributes, clinicalSampleAttributeValues);
+    }
+}