[bot] Fast-forward for 24.11.10

panoramapublic/src/org/labkey/panoramapublic/proteomexchange/validator/SpecLibValidator.java

@@ -31,8 +31,10 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -210,6 +212,16 @@ private void validateLibrarySources(List<LibSourceFile> sources, FileContentServ
         // folder as well as any subfolders containing documents that have the library
         Set<Container> containers = getDocsWithLibrary().stream().map(dl -> dl.getRun().getContainer()).collect(Collectors.toSet());
         containers.add(expAnnotations.getContainer());
+        Set<Path> rawFilesDirPaths = new HashSet<>();
+        for (Container container: containers)
+        {
+            var rawFilesDirPath = DataValidator.getRawFilesDirPath(container, fcs);
+            // We will look in the "RawFiles" directory only if it exists.
+            if (Files.exists(rawFilesDirPath))
+            {
+                rawFilesDirPaths.add(rawFilesDirPath);
+            }
+        }
 
         List<SpecLibSourceFile> spectrumFiles = new ArrayList<>();
         List<SpecLibSourceFile> idFiles = new ArrayList<>();
@@ -220,7 +232,7 @@ private void validateLibrarySources(List<LibSourceFile> sources, FileContentServ
             if (source.hasSpectrumSourceFile() && !checkedFiles.contains(ssf))
             {
                 checkedFiles.add(ssf);
-                Path path = getPath(ssf, containers, source.isMaxQuantSearch(), fcs);
+                Path path = getPath(ssf, rawFilesDirPaths, source.isMaxQuantSearch(), fcs);
                 SpecLibSourceFile sourceFile = new SpecLibSourceFile(ssf, SPECTRUM);
                 sourceFile.setSpecLibValidationId(getId());
                 sourceFile.setPath(path != null ? path.toString() : DataFile.NOT_FOUND);
@@ -230,7 +242,7 @@ private void validateLibrarySources(List<LibSourceFile> sources, FileContentServ
             if (source.hasIdFile() && !checkedFiles.contains(idFile))
             {
                 checkedFiles.add(idFile);
-                Path path = getPath(idFile, containers, false, fcs);
+                Path path = getPath(idFile, rawFilesDirPaths, false, fcs);
                 SpecLibSourceFile sourceFile = new SpecLibSourceFile(idFile, PEPTIDE_ID);
                 sourceFile.setSpecLibValidationId(getId());
                 sourceFile.setPath(path != null ? path.toString() : DataFile.NOT_FOUND);
@@ -241,11 +253,10 @@ private void validateLibrarySources(List<LibSourceFile> sources, FileContentServ
         setIdFiles(idFiles);
     }
 
-    private Path getPath(String name, Set<Container> containers, boolean isMaxquant, FileContentService fcs)
+    private Path getPath(String name, Set<Path> rawFilesDirPaths, boolean isMaxquant, FileContentService fcs)
     {
-        for (Container container: containers)
+        for (Path rawFilesDir: rawFilesDirPaths)
         {
-            java.nio.file.Path rawFilesDir = DataValidator.getRawFilesDirPath(container, fcs);
             Path path = findInDirectoryTree(rawFilesDir, name, isMaxquant);
             if (path != null)
             {

panoramapublic/test/sampledata/TargetedMS/panoramapublic/LibraryTest-telomerasehcmvprm/README.txt

@@ -0,0 +1,4 @@
+This folder contains dummy files for testing that all the source files used for building a Bibliospec spectral library are available in the folder being submitted to Panorama Public.
+Test is: PanoramaPublicValidationTest.testLibraryValidationWithSubfolders()
+Original files are in the Panorama Public data folder https://panoramaweb.org/telomerasehcmvprm.url
+Replicates and targets were removed from the test document, Telomerase_HCMV_PRM_Skyline_2024-11-22_21-30-46.sky.zip, to keep the size small.  The library, 20211118_SIRT-PRM_HSV1_rescheduling_v2.blib, was rebuilt as test_library.blib with "Filter for document peptides" checked to keep the size of the library small. 

panoramapublic/test/src/org/labkey/test/pages/panoramapublic/DataValidationPage.java

@@ -167,11 +167,11 @@ public void verifySpectralLibraryStatus(String libraryFile, String fileSize, Str
         expandLibraryRow(panel, libraryFile, fileSize);
         verifySpecLibStatus(libraryFile, fileSize,
                 (spectrumFiles.size() == 0 || idFiles.size() == 0
-                        || spectrumFilesMissing.size() > 0 || idFilesMissing.size() > 0) ? "INCOMPLETE" : "CPMPLETE");
+                        || spectrumFilesMissing.size() > 0 || idFilesMissing.size() > 0) ? "INCOMPLETE" : "COMPLETE");
 
         var panelText = panel.getText();
         List<String> expectedTexts = new ArrayList<>(skylineDocNames);
-        expectedTexts.add(statusText);
+        if (statusText != null) expectedTexts.add(statusText);
         assertTextPresent(new TextSearcher(panelText), expectedTexts.toArray(new String[]{}));
 
         verifyLibrarySourceFiles(libraryFile, spectrumFiles, spectrumFilesMissing, panel, "lib-spectrum-files-status");

panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java

@@ -202,6 +202,11 @@ private void setupSourceFolder(String projectName, String folderName, FolderType
     {
         setupSubfolder(projectName, folderName, folderType); // Create the subfolder
 
+        updatePermissions(projectName, folderName, adminUsers);
+    }
+
+    private void updatePermissions(String projectName, String folderName, String[] adminUsers)
+    {
         ApiPermissionsHelper permissionsHelper = new ApiPermissionsHelper(this);
         _userHelper.ensureUsersExist(List.of(adminUsers));
         for(String user: adminUsers)
@@ -210,6 +215,12 @@ private void setupSourceFolder(String projectName, String folderName, FolderType
         }
     }
 
+    void setupSubfolder(String projectName, String parentFolderName, String folderName, FolderType folderType, String ... adminUsers)
+    {
+        super.setupSubfolder(projectName, parentFolderName, folderName, folderType);
+        updatePermissions(projectName, parentFolderName + "/" + folderName, adminUsers);
+    }
+
     void updateSubmitterAccountInfo(String lastName)
     {
         goToMyAccount();

panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicValidationTest.java

@@ -6,6 +6,7 @@
 import org.labkey.test.Locator;
 import org.labkey.test.categories.External;
 import org.labkey.test.categories.MacCossLabModules;
+import org.labkey.test.components.panoramapublic.TargetedMsExperimentWebPart;
 import org.labkey.test.pages.panoramapublic.DataValidationPage;
 import org.labkey.test.util.DataRegionTable;
 import org.labkey.test.util.Ext4Helper;
@@ -42,6 +43,17 @@ public class PanoramaPublicValidationTest extends PanoramaPublicBaseTest
     private static final String AGILENT_DATA_2_ZIP = AGILENT_DATA_2 + ".zip";
     private static final String SKY_FILE_8 = "Study9S_Site52_v1_with_library.sky.zip";
 
+    // Raw files used to build the maxquant.blib library used with Olga_srm_course_heavy_light_w_maxquant_lib.sky.zip
+    private static final List<String> maxQuantLibRawSources = List.of("BBM_332_P110_C04_PRM_007.raw",
+            "BBM_332_P110_C04_PRM_006.raw",
+            "BBM_332_P110_C04_PRM_005.raw",
+            "BBM_332_P110_C04_PRM_004.raw",
+            "BBM_332_P110_C04_PRM_003.raw");
+    // Peptide Id files used to build the maxquant.blib library used with Olga_srm_course_heavy_light_w_maxquant_lib.sky.zip
+    private static final List<String> maxQuantLibPeptideIdSources = List.of("evidence.txt", "mqpar.xml", "msms.txt");
+
+    private static final String SKY_FILE_9 = "Telomerase_HCMV_PRM_Skyline-NO-RESULTS.sky.zip";
+
     @Override
     public String getSampleDataFolder()
     {
@@ -177,6 +189,96 @@ public void testValidationUpdate()
         goToValidationDetails().verifyCompleteStatus();
     }
 
+    @Test
+    public void testLibraryValidationWithSubfolders()
+    {
+        // Set up our source folder.
+        log("Creating experiment folder");
+        String projectName = getProjectName();
+        String folderName = "Experiment Folder";
+        setupSourceFolder(projectName, folderName, SUBMITTER);
+        log("Creating subfolder where Skyline documents will be uploaded");
+        String subfolderName = "Skyline Documents Folder";
+        setupSubfolder(projectName, folderName, subfolderName, FolderType.Experiment, SUBMITTER);
+
+        impersonate(SUBMITTER);
+        updateSubmitterAccountInfo("One");
+
+        String testFilesFolder = "LibraryTest-telomerasehcmvprm";
+        String testSkyZip = testFilesFolder + "/" + SKY_FILE_9;
+
+        // Upload document and raw data files
+        log("Uploading and importing Skyline document " + testSkyZip + " into folder " + folderName + "/" + subfolderName);
+        goToProjectFolder(projectName, folderName + "/" + subfolderName);
+        importData(testSkyZip, 1);
+
+        // Add the "Targeted MS Experiment" webpart
+        log("Creating TargetedMS Experiment in folder " + folderName);
+        goToProjectFolder(projectName, folderName);
+        String experimentTitle = "This is an experiment to test validation of Bibliospec library source files";
+        TargetedMsExperimentWebPart expWebPart = createExperimentCompleteMetadata(experimentTitle);
+        // Include subfolders
+        log("Including subfolders in experiment");
+        goToDashboard();
+        expWebPart.clickMoreDetails();
+        clickButton("Include Subfolders");
+
+        String libraryName = "test_library.blib";
+        List<String> rawSources = List.of("20210719_SIRT-PRM_non-SIRT4_unsched_01.raw",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_02.raw",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_03.raw",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_04.raw",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_05.raw",
+                "20210719_SIRT-PRM_SIRT4pep_unsched.raw");
+        List<String> peptideIdSources = List.of("20210719_SIRT-PRM_non-SIRT4_unsched_01.msf",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_02.msf",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_03.msf",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_04.msf",
+                "20210719_SIRT-PRM_non-SIRT4_unsched_05.msf",
+                "20210719_SIRT-PRM_SIRT4pep_unsched.msf");
+
+        goToDashboard();
+        // Upload the source files used to build test_library.blib to the parent experiment folder.
+        // Since spectral libraries can be used with multiple documents that may be uploaded to different subfolders,
+        // we check all subfolders, as well as the parent experiment folder for library source files.
+        log("Uploading library source files to parent experiment folder - " + folderName);
+        uploadRawFiles(rawSources.stream()
+                .map(file -> testFilesFolder + "/" + file)
+                .toArray(String[]::new));
+        uploadRawFiles(peptideIdSources.stream()
+                .map(file -> testFilesFolder + "/" + file)
+                .toArray(String[]::new));
+
+        // Run validation job and verify the results
+        log("Running data validation job; expect COMPLETE status");
+        DataValidationPage validationPage = submitValidationJob();
+        validationPage.verifyCompleteStatus();
+        verifySpecLibSourceFiles(validationPage, libraryName, SKY_FILE_9, "100 KB", true, rawSources, peptideIdSources);
+
+        // Delete the "RawFiles" folder in the parent experiment folder
+        log("Deleting RawFiles directory from the parent experiment folder");
+        goToModule("FileContent");
+        _fileBrowserHelper.deleteFile("RawFiles");
+
+        log("Running data validation job; expect INCOMPLETE status");
+        validationPage = submitValidationJob();
+        validationPage.verifyIncompleteStatus();
+        verifySpecLibSourceFiles(validationPage, libraryName, SKY_FILE_9, "100 KB", false, rawSources, peptideIdSources);
+    }
+
+    private static void verifySpecLibSourceFiles(DataValidationPage validationPage, String libraryFileName, String skyZipName, String libraryFileSize,
+                                                 boolean allSourcesFound, List<String> rawFiles, List<String> peptideIdFiles)
+    {
+        String expectedStatus = allSourcesFound ? null : "Missing spectrum and peptide Id files";
+        validationPage.verifySpectralLibraryStatus(libraryFileName, libraryFileSize, expectedStatus,
+                List.of(skyZipName),
+                allSourcesFound ? rawFiles : Collections.emptyList(),
+                !allSourcesFound ? rawFiles : Collections.emptyList(),
+                allSourcesFound ? peptideIdFiles : Collections.emptyList(),
+                !allSourcesFound ? peptideIdFiles : Collections.emptyList()
+                );
+    }
+
     private void clickExperimentDetailsLink()
     {
         var exptDetailsLink = Locator.XPathLocator.tag("a").withText("[View Experiment Details]").findElement(getDriver());
@@ -352,12 +454,8 @@ private int verifyIncompleteStatus(int jobCount)
 
         // Verify library built with MaxQuant results. Expect to see evidence.xml, mqpar.xml in the Peptide ID files list
         // even though these files are not named in the .blib
-        validationPage.verifySpectralLibraryStatus("maxquant.blib", "104 KB", "Missing spectrum and peptide Id files",
-                List.of(SKY_FILE_2),
-                Collections.emptyList(),
-                List.of("BBM_332_P110_C04_PRM_007.raw", "BBM_332_P110_C04_PRM_006.raw", "BBM_332_P110_C04_PRM_005.raw", "BBM_332_P110_C04_PRM_004.raw", "BBM_332_P110_C04_PRM_003.raw"),
-                Collections.emptyList(),
-                List.of("evidence.txt", "mqpar.xml", "msms.txt"));
+        verifySpecLibSourceFiles(validationPage, "maxquant.blib", SKY_FILE_2,  "104 KB",
+                false, maxQuantLibRawSources, maxQuantLibPeptideIdSources);
 
         // .blib does not have any source files in the SpectrumSourceFiles table.
         validationPage.verifySpectralLibraryStatus("Qtrap_DP-PA_cons_P0836.blib", "61 KB",
@@ -383,12 +481,8 @@ private int verifyMissingBlibAndUnsupportedLibrary(int jobCount)
 
         // Verify library built with MaxQuant results. Expect to see evidence.xml, mqpar.xml in the Peptide ID files list
         // even though these files are not named in the .blib
-        validationPage.verifySpectralLibraryStatus("maxquant.blib", "104 KB", "Missing spectrum and peptide Id files",
-                List.of(SKY_FILE_2),
-                Collections.emptyList(),
-                List.of("BBM_332_P110_C04_PRM_007.raw", "BBM_332_P110_C04_PRM_006.raw", "BBM_332_P110_C04_PRM_005.raw", "BBM_332_P110_C04_PRM_004.raw", "BBM_332_P110_C04_PRM_003.raw"),
-                Collections.emptyList(),
-                List.of("evidence.txt", "mqpar.xml", "msms.txt"));
+        verifySpecLibSourceFiles(validationPage, "maxquant.blib", SKY_FILE_2, "104 KB",
+                false, maxQuantLibRawSources, maxQuantLibPeptideIdSources);
 
         // .blib does not have any source files in the SpectrumSourceFiles table.
         validationPage.verifySpectralLibraryStatus("Qtrap_DP-PA_cons_P0836.blib", "61 KB",