diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java index ffc03592..6dc5ea79 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicController.java @@ -1499,6 +1499,20 @@ private boolean validateAction(CopyExperimentForm form, BindException errors) return true; } + private Path getExportFilesDir(Container c) + { + FileContentService fcs = FileContentService.get(); + if(fcs != null) + { + Path fileRoot = fcs.getFileRootPath(c, FileContentService.ContentType.files); + if (fileRoot != null) + { + return fileRoot.resolve(PipelineService.EXPORT_DIR); + } + } + return null; + } + @Override public boolean handlePost(CopyExperimentForm form, BindException errors) { @@ -1539,13 +1553,15 @@ public boolean handlePost(CopyExperimentForm form, BindException errors) return false; } + String previousVersionName = null; Submission previousSubmission = _journalSubmission.getLatestCopiedSubmission(); if (previousSubmission != null) { // Target folder name is automatically populated in the copy experiment form. Unless the admin making the copy changed the // folder name we expect the previous copy of the data to have the same folder name. Rename the old folder so that we can // use the same folder name for the new copy. - if (!renamePreviousFolder(previousSubmission, destinationFolder, errors)) + previousVersionName = renamePreviousFolder(previousSubmission, destinationFolder, errors); + if (previousVersionName == null) { return false; } @@ -1573,8 +1589,13 @@ public boolean handlePost(CopyExperimentForm form, BindException errors) job.setUsePxTestDb(form.isUsePxTestDb()); job.setAssignDoi(form.isAssignDoi()); job.setUseDataCiteTestApi(form.isUseDataCiteTestApi()); + job.setMoveAndSymlink(form.isMoveAndSymlink()); job.setReviewerEmailPrefix(form.getReviewerEmailPrefix()); job.setDeletePreviousCopy(form.isDeleteOldCopy()); + job.setPreviousVersionName(previousVersionName); + job.setExportTargetPath(getExportFilesDir(target)); + job.setExportSourceContainer(form.getContainer()); + PipelineService.get().queueJob(job); _successURL = PageFlowUtil.urlProvider(PipelineStatusUrls.class).urlBegin(target); @@ -1586,12 +1607,14 @@ public boolean handlePost(CopyExperimentForm form, BindException errors) } } - private boolean renamePreviousFolder(Submission previousSubmission, String targetContainerName, BindException errors) + private String renamePreviousFolder(Submission previousSubmission, String targetContainerName, BindException errors) { + String newPath = null; ExperimentAnnotations previousCopy = ExperimentAnnotationsManager.get(previousSubmission.getCopiedExperimentId()); if (previousCopy != null) { Container previousContainer = previousCopy.getContainer(); + newPath = previousContainer.getPath(); if (targetContainerName.equals(previousContainer.getName())) { try (DbScope.Transaction transaction = PanoramaPublicManager.getSchema().getScope().ensureTransaction()) @@ -1601,21 +1624,23 @@ private boolean renamePreviousFolder(Submission previousSubmission, String targe { errors.reject(ERROR_MSG, "Previous experiment copy (Id: " + previousCopy.getId() + ") does not have a version. " + "Cannot rename previous folder."); - return false; + return null; } // Rename the container where the old copy lives so that the same folder name can be used for the new copy. String newName = previousContainer.getName() + " V." + version; if (ContainerManager.getChild(previousContainer.getParent(), newName) != null) { errors.reject(ERROR_MSG, "Cannot rename previous folder to '" + newName + "'. A folder with that name already exists."); - return false; + return null; } ContainerManager.rename(previousContainer, getUser(), newName); + + newPath = FileContentService.get().getFileRoot(previousContainer.getParent()) + File.separator + newName; transaction.commit(); } } } - return true; + return newPath; } private ValidEmail getValidEmail(String email, String errMsg, BindException errors) @@ -1683,6 +1708,8 @@ public static class CopyExperimentForm extends ExperimentIdForm private boolean _usePxTestDb; // Use the test database for getting a PX ID if true private boolean _assignDoi; private boolean _useDataCiteTestApi; + + private boolean _moveAndSymlink; private boolean _deleteOldCopy; static void setDefaults(CopyExperimentForm form, ExperimentAnnotations sourceExperiment, Submission currentSubmission) @@ -1696,6 +1723,7 @@ static void setDefaults(CopyExperimentForm form, ExperimentAnnotations sourceExp form.setUsePxTestDb(false); form.setAssignDoi(true); + form.setMoveAndSymlink(true); form.setUseDataCiteTestApi(false); Container sourceExptContainer = sourceExperiment.getContainer(); @@ -1819,6 +1847,16 @@ public void setUseDataCiteTestApi(boolean useDataCiteTestApi) _useDataCiteTestApi = useDataCiteTestApi; } + public boolean isMoveAndSymlink() + { + return _moveAndSymlink; + } + + public void setMoveAndSymlink(boolean moveAndSymlink) + { + _moveAndSymlink = moveAndSymlink; + } + public boolean isDeleteOldCopy() { return _deleteOldCopy; @@ -5626,7 +5664,7 @@ public ExperimentAnnotationsDetails(User user, ExperimentAnnotations exptAnnotat { // Display the version only if there is more than one version of this dataset on Panorama Public _version = _experimentAnnotations.getStringVersion(maxVersion); - if (_experimentAnnotations.getDataVersion().equals(maxVersion)) + if (_experimentAnnotations.getDataVersion() != null && _experimentAnnotations.getDataVersion().equals(maxVersion)) { // This is the current version; Display a link to see all published versions _versionsUrl = new ActionURL(PanoramaPublicController.ShowPublishedVersions.class, _experimentAnnotations.getContainer()); @@ -9050,6 +9088,20 @@ public Pair getAttachment(AttachmentForm form) } } + @RequiresPermission(ReadPermission.class) + public static class VerifySymlinksAction extends ReadOnlyApiAction + { + @Override + public Object execute(CatalogForm catalogForm, BindException errors) throws Exception + { + if (PanoramaPublicSymlinkManager.get().verifySymlinks()) + return success(); + + errors.reject(ERROR_MSG, "Problems with symlink registration. See log for details."); + return null; + } + } + @RequiresPermission(ReadPermission.class) public static class GetCatalogApiAction extends ReadOnlyApiAction { diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java new file mode 100644 index 00000000..15d37187 --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java @@ -0,0 +1,171 @@ +package org.labkey.panoramapublic; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.AbstractFolderImportFactory; +import org.labkey.api.admin.FolderImportContext; +import org.labkey.api.admin.FolderImporter; +import org.labkey.api.admin.ImportException; +import org.labkey.api.admin.SubfolderWriter; +import org.labkey.api.data.Container; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileContentService; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.security.User; +import org.labkey.api.writer.VirtualFile; +import org.labkey.panoramapublic.pipeline.CopyExperimentPipelineJob; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Objects; + +/** + * This importer does a file move instead of copy to the temp directory and creates a symlink in place of the original + * file. + */ +public class PanoramaPublicFileImporter implements FolderImporter +{ + @Override + public String getDataType() + { + return PanoramaPublicManager.PANORAMA_PUBLIC_FILES; + } + + @Override + public String getDescription() + { + return "Panorama Public Files"; + } + + @Override + public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualFile root) throws Exception + { + Logger log = ctx.getLogger(); + + FileContentService fcs = FileContentService.get(); + if (null == fcs) + return; + + File targetRoot = fcs.getFileRoot(ctx.getContainer()); + + if (null == targetRoot) + { + log.error("File copy target folder not found: " + ctx.getContainer().getPath()); + return; + } + + if (null == job) + { + log.error("Pipeline job not found."); + return; + } + + if (job instanceof CopyExperimentPipelineJob expJob) + { + File targetFiles = new File(targetRoot.getPath(), FileContentService.FILES_LINK); + + // Get source files including resolving subfolders + String divider = FileContentService.FILES_LINK + File.separator + PipelineService.EXPORT_DIR; + String subProject = root.getLocation().substring(root.getLocation().lastIndexOf(divider) + divider.length()); + subProject = subProject.replace(File.separator + SubfolderWriter.DIRECTORY_NAME, ""); + + Path sourcePath = Paths.get(fcs.getFileRoot(expJob.getExportSourceContainer()).getPath(), subProject); + File sourceFiles = Paths.get(sourcePath.toString(), FileContentService.FILES_LINK).toFile(); + + if (!targetFiles.exists()) + { + log.warn("Panorama public file copy target not found. Creating directory: " + targetFiles); + Files.createDirectories(targetFiles.toPath()); + } + + log.info("Moving files and creating sym links in folder " + ctx.getContainer().getPath()); + PanoramaPublicSymlinkManager.get().moveAndSymLinkDirectory(expJob, sourceFiles, targetFiles, false, log); + + alignDataFileUrls(expJob.getUser(), ctx.getContainer(), log); + } + } + + private void alignDataFileUrls(User user, Container targetContainer, Logger log) throws BatchValidationException, ImportException + { + log.info("Aligning data files urls in folder: " + targetContainer.getPath()); + + FileContentService fcs = FileContentService.get(); + if (null == fcs) + return; + + ExperimentService expService = ExperimentService.get(); + List runs = expService.getExpRuns(targetContainer, null, null); + boolean errors = false; + + Path fileRootPath = fcs.getFileRootPath(targetContainer, FileContentService.ContentType.files); + if(fileRootPath == null || !Files.exists(fileRootPath)) + { + throw new ImportException("File root path for container " + targetContainer.getPath() + " does not exist: " + fileRootPath); + } + + for (ExpRun run : runs) + { + run.setFilePathRootPath(fileRootPath); + run.save(user); + log.debug("Setting filePathRoot on copied run: " + run.getName() + " to: " + fileRootPath); + + for (ExpData data : run.getAllDataUsedByRun()) + { + if (null != data.getRun() && data.getDataFileUrl().contains(FileContentService.FILES_LINK)) + { + String[] parts = Objects.requireNonNull(data.getFilePath()).toString().split("Run\\d+"); + + if (parts.length > 1) + { + String fileName = parts[1]; + Path newDataPath = Paths.get(fileRootPath.toString(), fileName); + + if (newDataPath.toFile().exists()) + { + data.setDataFileURI(newDataPath.toUri()); + data.save(user); + log.debug("Setting dataFileUri on copied data: " + data.getName() + " to: " + newDataPath); + } + else + { + log.error("Data file not found: " + newDataPath.toUri()); + errors = true; + } + } + else + { + log.error("Unexpected data file path. Could not align dataFileUri. " + data.getFilePath().toString()); + errors = true; + } + } + } + } + if (errors) + { + throw new ImportException("Data files urls could not be aligned."); + } + } + + public static class Factory extends AbstractFolderImportFactory + { + @Override + public FolderImporter create() + { + return new PanoramaPublicFileImporter(); + } + + @Override + public int getPriority() + { + // We want this to run last to do exp.data.datafileurl cleanup + return PanoramaPublicManager.PRIORITY_PANORAMA_PUBLIC_FILES; + } + } +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java new file mode 100644 index 00000000..c334b425 --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java @@ -0,0 +1,65 @@ +package org.labkey.panoramapublic; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileListener; +import org.labkey.api.security.User; + +import java.io.File; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; + +public class PanoramaPublicFileListener implements FileListener +{ + + @Override + public String getSourceName() + { + return null; + } + + @Override + public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) + { + + } + + @Override + public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) + { + // Update any symlinks targeting the file + PanoramaPublicSymlinkManager.get().fireSymlinkUpdate(src.toPath(), dest.toPath(), container); + + ExpData data = ExperimentService.get().getExpDataByURL(src, null); + if (null != data) + data.setDataFileURI(dest.toURI()); + + return 0; + } + + @Override + public void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable Container container) + { + ExpData data = ExperimentService.get().getExpDataByURL(deleted, container); + + if (null != data) + data.delete(user); + } + + @Override + public Collection listFiles(@Nullable Container container) + { + return Collections.emptyList(); + } + + @Override + public SQLFragment listFilesQuery() + { + throw new UnsupportedOperationException("Not implemented"); + } +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java index 464d8ee2..4bcb576e 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java @@ -23,6 +23,7 @@ import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExperimentListener; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileContentService; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; import org.labkey.api.security.User; @@ -39,6 +40,7 @@ import org.labkey.panoramapublic.query.SubmissionManager; import java.beans.PropertyChangeEvent; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -72,11 +74,14 @@ public void containerCreated(Container c, User user) public void containerDeleted(Container c, User user) { JournalManager.deleteProjectJournal(c, user); + + PanoramaPublicSymlinkManager.get().beforeContainerDeleted(c); } @Override public void containerMoved(Container c, Container oldParent, User user) { + PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldParent, c); } @Override @@ -88,6 +93,26 @@ public void containerMoved(Container c, Container oldParent, User user) @Override public void propertyChange(PropertyChangeEvent evt) { + // Needed for container rename to realign symlinks + if (evt.getPropertyName().equals(ContainerManager.Property.Name.name()) + && evt instanceof ContainerManager.ContainerPropertyChangeEvent ce) + { + Container c = ce.container; + + if (PanoramaPublicManager.canBeSymlinkTarget(c)) // Fire the event only if a folder in the Panorama Public project is being renamed. + { + FileContentService fcs = FileContentService.get(); + if (fcs != null) + { + Container parent = c.getParent(); + Path parentPath = fcs.getFileRootPath(parent); + // ce.getOldValue() and ce.getNewValue() are just the names of the old and new containers. We need the full path. + Path oldPath = parentPath.resolve((String) ce.getOldValue()); + Path newPath = parentPath.resolve((String) ce.getNewValue()); + PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldPath.toString(), newPath.toString(), c); + } + } + } } // ShortURLListener diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicManager.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicManager.java index 1c87a2c6..48070d09 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicManager.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicManager.java @@ -16,6 +16,8 @@ package org.labkey.panoramapublic; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.labkey.api.data.Container; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbSchemaType; @@ -25,11 +27,19 @@ import org.labkey.api.targetedms.TargetedMSService; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; +import org.labkey.panoramapublic.query.JournalManager; public class PanoramaPublicManager { private static final PanoramaPublicManager _instance = new PanoramaPublicManager(); + private static final Logger _log = LogManager.getLogger(PanoramaPublicManager.class); + + public static String PANORAMA_PUBLIC_FILES = "Panorama Public Files"; + public static String PANORAMA_PUBLIC_METADATA = "Panorama Public Experiment Metadata"; + public static int PRIORITY_PANORAMA_PUBLIC_METADATA = 1000; + public static int PRIORITY_PANORAMA_PUBLIC_FILES = PRIORITY_PANORAMA_PUBLIC_METADATA + 1; + private PanoramaPublicManager() { // prevent external construction with a private default constructor @@ -144,4 +154,12 @@ public static ActionURL getRawDataTabUrl(Container container) { return PageFlowUtil.urlProvider(ProjectUrls.class).getBeginURL(container, TargetedMSService.RAW_FILES_TAB); } + + public static boolean canBeSymlinkTarget(Container container) + { + // Folders in a journal project (e.g. Panorama Public) are the only ones that can have symlink targets. + // Folders in other projects can contain symlinks but no symlink targets. + Container project = container.getProject(); + return project != null ? JournalManager.isJournalProject(project) : false; + } } \ No newline at end of file diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicMetadataImporter.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicMetadataImporter.java new file mode 100644 index 00000000..ac13766a --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicMetadataImporter.java @@ -0,0 +1,227 @@ +package org.labkey.panoramapublic; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.AbstractFolderImportFactory; +import org.labkey.api.admin.FolderImportContext; +import org.labkey.api.admin.FolderImporter; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.DbScope; +import org.labkey.api.exp.api.ExpExperiment; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.pipeline.PipelineJob; +import org.labkey.api.pipeline.PipelineJobException; +import org.labkey.api.security.User; +import org.labkey.api.writer.VirtualFile; +import org.labkey.panoramapublic.model.ExperimentAnnotations; +import org.labkey.panoramapublic.model.JournalSubmission; +import org.labkey.panoramapublic.model.speclib.SpecLibInfo; +import org.labkey.panoramapublic.pipeline.CopyExperimentJobSupport; +import org.labkey.panoramapublic.pipeline.CopyExperimentPipelineJob; +import org.labkey.panoramapublic.query.ExperimentAnnotationsManager; +import org.labkey.panoramapublic.query.ModificationInfoManager; +import org.labkey.panoramapublic.query.SpecLibInfoManager; +import org.labkey.panoramapublic.query.SubmissionManager; +import org.labkey.panoramapublic.query.modification.ExperimentIsotopeModInfo; +import org.labkey.panoramapublic.query.modification.ExperimentStructuralModInfo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + Creates a new row in panoramapublic.ExperimentAnnotations and links it to the new experiment created during folder import. + This importer should run before {@link PanoramaPublicFileImporter} so that if there is an error in aligning datafileurls in + {@link PanoramaPublicFileImporter#alignDataFileUrls(User, Container, VirtualFile, Logger)} we can delete the container and + get all the files copied back to the source container. {@link PanoramaPublicListener#containerDeleted(Container, User)} + will fire fireSymlinkCopiedExperimentDelete only if there is an entry in panoramapublic.experimentannotations. + + Copies other metadata related to spectral libraries and modifications from the source container. + */ +public class PanoramaPublicMetadataImporter implements FolderImporter +{ + @Override + public String getDataType() + { + return PanoramaPublicManager.PANORAMA_PUBLIC_METADATA; + } + + @Override + public String getDescription() + { + return "Panorama Public Metadata"; + } + + @Override + public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualFile root) throws Exception + { + if (job instanceof CopyExperimentPipelineJob) + { + Logger log = ctx.getLogger(); + CopyExperimentJobSupport jobSupport = job.getJobSupport(CopyExperimentJobSupport.class); + + Container container = job.getContainer(); + + // If a row already exists in panoramapublic.experimentannotations then don't do anything. + if (ExperimentAnnotationsManager.getExperimentInContainer(container) != null) + { + return; + } + + // Get the experiment that was just created in the target folder as part of folder import. + User user = job.getUser(); + List experiments = ExperimentService.get().getExperiments(container, user, false, false); + if (experiments.size() == 0) + { + throw new PipelineJobException("No experiments found in the folder " + container.getPath()); + } + else if (experiments.size() > 1) + { + throw new PipelineJobException("More than one experiment found in the folder " + container.getPath()); + } + ExpExperiment experiment = experiments.get(0); + + try (DbScope.Transaction transaction = PanoramaPublicManager.getSchema().getScope().ensureTransaction()) + { + // Get the ExperimentAnnotations in the source container + ExperimentAnnotations sourceExperiment = jobSupport.getExpAnnotations(); + // Get the submission request + JournalSubmission js = SubmissionManager.getJournalSubmission(sourceExperiment.getId(), jobSupport.getJournal().getId(), sourceExperiment.getContainer()); + if (js == null) + { + throw new PipelineJobException("Could not find a submission request for experiment Id " + sourceExperiment.getId() + " and journal Id " + jobSupport.getJournal().getId() + + " in the folder '" + sourceExperiment.getContainer() + "'"); + } + + ExperimentAnnotations targetExperiment = createNewExperimentAnnotations(experiment, sourceExperiment, user, log); + + // Get a list of all the ExpRuns imported to subfolders of this folder. + int[] runRowIdsInSubfolders = getAllExpRunRowIdsInSubfolders(container); + if (runRowIdsInSubfolders.length > 0) + { + // The folder export and import process, creates a new experiment in exp.experiment. + // However, only runs in the top-level folder are added to this experiment. + // We will add to the experiment all the runs imported to subfolders. + log.info("Adding runs imported in subfolders."); + ExperimentAnnotationsManager.addSelectedRunsToExperiment(experiment, runRowIdsInSubfolders, user); + } + + // Copy any Spectral library information provided by the user in the source container + copySpecLibInfos(sourceExperiment, targetExperiment, user); + + // Copy any modifications related information provided by the user in the source container + copyModificationInfos(sourceExperiment, targetExperiment, user); + + transaction.commit(); + } + } + } + + private static int[] getAllExpRunRowIdsInSubfolders(Container container) + { + Set children = ContainerManager.getAllChildren(container); + ExperimentService expService = ExperimentService.get(); + List expRunRowIds = new ArrayList<>(); + for(Container child: children) + { + if(container.equals(child)) + { + continue; + } + List runs = expService.getExpRuns(child, null, null); + for(ExpRun run: runs) + { + expRunRowIds.add(run.getRowId()); + } + } + int[] intIds = new int[expRunRowIds.size()]; + for(int i = 0; i < expRunRowIds.size(); i++) + { + intIds[i] = expRunRowIds.get(i); + } + return intIds; + } + + private ExperimentAnnotations createNewExperimentAnnotations(ExpExperiment experiment, ExperimentAnnotations sourceExperiment, + User user, Logger log) + { + log.info("Creating a new TargetedMS experiment entry in panoramapublic.ExperimentAnnotations."); + ExperimentAnnotations targetExperiment = createExperimentCopy(sourceExperiment); + targetExperiment.setExperimentId(experiment.getRowId()); + targetExperiment.setContainer(experiment.getContainer()); + targetExperiment.setSourceExperimentId(sourceExperiment.getId()); + targetExperiment.setSourceExperimentPath(sourceExperiment.getContainer().getPath()); + + return ExperimentAnnotationsManager.save(targetExperiment, user); + } + + private ExperimentAnnotations createExperimentCopy(ExperimentAnnotations source) + { + ExperimentAnnotations copy = new ExperimentAnnotations(); + copy.setTitle(source.getTitle()); + copy.setExperimentDescription(source.getExperimentDescription()); + copy.setSampleDescription(source.getSampleDescription()); + copy.setOrganism(source.getOrganism()); + copy.setInstrument(source.getInstrument()); + copy.setSpikeIn(source.getSpikeIn()); + copy.setCitation(source.getCitation()); + copy.setAbstract(source.getAbstract()); + copy.setPublicationLink(source.getPublicationLink()); + copy.setIncludeSubfolders(source.isIncludeSubfolders()); + copy.setKeywords(source.getKeywords()); + copy.setLabHead(source.getLabHead()); + copy.setSubmitter(source.getSubmitter()); + copy.setLabHeadAffiliation(source.getLabHeadAffiliation()); + copy.setSubmitterAffiliation(source.getSubmitterAffiliation()); + copy.setPubmedId(source.getPubmedId()); + return copy; + } + + private void copySpecLibInfos(ExperimentAnnotations sourceExperiment, ExperimentAnnotations targetExperiment, User user) + { + List specLibInfos = SpecLibInfoManager.getForExperiment(sourceExperiment.getId(), sourceExperiment.getContainer()); + for (SpecLibInfo info: specLibInfos) + { + info.setId(0); + info.setExperimentAnnotationsId(targetExperiment.getId()); + SpecLibInfoManager.save(info, user); + } + } + + private void copyModificationInfos(ExperimentAnnotations sourceExperiment, ExperimentAnnotations targetExperiment, User user) + { + List strModInfos = ModificationInfoManager.getStructuralModInfosForExperiment(sourceExperiment.getId(), sourceExperiment.getContainer()); + for (ExperimentStructuralModInfo info: strModInfos) + { + info.setId(0); + info.setExperimentAnnotationsId(targetExperiment.getId()); + ModificationInfoManager.saveStructuralModInfo(info, user); + } + + List isotopeModInfos = ModificationInfoManager.getIsotopeModInfosForExperiment(sourceExperiment.getId(), sourceExperiment.getContainer()); + for (ExperimentIsotopeModInfo info: isotopeModInfos) + { + info.setId(0); + info.setExperimentAnnotationsId(targetExperiment.getId()); + ModificationInfoManager.saveIsotopeModInfo(info, user); + } + } + + public static class Factory extends AbstractFolderImportFactory + { + @Override + public FolderImporter create() + { + return new PanoramaPublicMetadataImporter(); + } + + @Override + public int getPriority() + { + // We want this to run AFTER an entry is created in exp.experiment and BEFORE PanoramaPublicFileImporter + return PanoramaPublicManager.PRIORITY_PANORAMA_PUBLIC_METADATA; + } + } +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java index 4929bd17..38c85239 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicModule.java @@ -18,11 +18,13 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderSerializationRegistry; import org.labkey.api.attachments.AttachmentService; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.SimpleFilter; import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileContentService; import org.labkey.api.module.ModuleContext; import org.labkey.api.module.SpringModule; import org.labkey.api.pipeline.PipelineService; @@ -133,6 +135,15 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) TargetedMSService.get().addModificationSearchResultCustomizer(ExperimentTitleDisplayColumn.getModSearchTableCustomizer()); TargetedMSService.get().addPeptideSearchResultCustomizers(ExperimentTitleDisplayColumn.getPeptideGroupJoinTableCustomizer()); TargetedMSService.get().addProteinSearchResultCustomizer(ExperimentTitleDisplayColumn.getPeptideGroupJoinTableCustomizer()); + + FolderSerializationRegistry.get().addImportFactory(new PanoramaPublicFileImporter.Factory()); + FolderSerializationRegistry.get().addImportFactory(new PanoramaPublicMetadataImporter.Factory()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new PanoramaPublicFileListener()); + } } @NotNull diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java new file mode 100644 index 00000000..5607bf1e --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java @@ -0,0 +1,9 @@ +package org.labkey.panoramapublic; + +import java.io.IOException; +import java.nio.file.Path; + +public interface PanoramaPublicSymlinkHandler +{ + void handleSymlink(Path link, Path target) throws IOException; +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java new file mode 100644 index 00000000..9e753672 --- /dev/null +++ b/panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java @@ -0,0 +1,448 @@ +package org.labkey.panoramapublic; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.SystemUtils; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerService; +import org.labkey.api.files.FileContentService; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.panoramapublic.model.ExperimentAnnotations; +import org.labkey.panoramapublic.pipeline.CopyExperimentPipelineJob; +import org.labkey.panoramapublic.query.ExperimentAnnotationsManager; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +public class PanoramaPublicSymlinkManager +{ + private static final Logger _log = LogHelper.getLogger(PanoramaPublicSymlinkManager.class, "Handling symlinks between public and private folders"); + + // Note: The production server running this code is Linux. Windows requires extra permissions to create symlinks. + // If running this code on a Windows server, the Panorama Public copy will not use symlinks unless this flag is set to true. + // Additionally, you will have to run the server (or IntelliJ) as an administrator to run with symlinks. + private static final boolean DEBUG_SYMLINKS_ON_WINDOWS = false; + + private static final PanoramaPublicSymlinkManager _instance = new PanoramaPublicSymlinkManager(); + + // Manage symlinks created when copying files to Panorama Public + private PanoramaPublicSymlinkManager() + { + // prevent external construction with a private default constructor + } + + public static PanoramaPublicSymlinkManager get() + { + return _instance; + } + + + private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler handler) + { + for (File file : Objects.requireNonNull(source.listFiles())) + { + if (file.isDirectory()) + { + handleContainerSymlinks(file, handler); + } + else + { + Path filePath = file.toPath(); + if (Files.isSymbolicLink(filePath)) + { + try { + Path target = Files.readSymbolicLink(filePath); + handler.handleSymlink(filePath, target); + } catch (IOException x) { + _log.error("Unable to resolve symlink target for symlink at " + filePath); + } + } + } + } + } + + public void handleContainerSymlinks(Container container, PanoramaPublicSymlinkHandler handler) + { + FileContentService fcs = FileContentService.get(); + if (null != fcs) + { + File root = fcs.getFileRoot(container); + if (null != root) + { + handleContainerSymlinks(root, handler); + } + } + } + + private void handleAllSymlinks(Set containers, PanoramaPublicSymlinkHandler handler) + { + for (Container container : containers) + { + Set tree = ContainerManager.getAllChildren(container); + for (Container node : tree) + { + handleContainerSymlinks(node, handler); + } + } + } + + private String normalizeContainerPath(String path) + { + File file = new File(path); + if (file.isAbsolute() || path.startsWith(File.separator)) + return path + File.separator; + + return File.separator + path + File.separator; + } + + public void beforeContainerDeleted(Container container) + { + if (PanoramaPublicManager.canBeSymlinkTarget(container)) // Fire the event only if the container being deleted is in the Panorama Public project. + { + // Look for an experiment that includes data in the given container. This could be an experiment defined + // in the given container, or in an ancestor container that has 'IncludeSubfolders' set to true. + ExperimentAnnotations expAnnot = ExperimentAnnotationsManager.getExperimentIncludesContainer(container); + if (null != expAnnot) + { + fireSymlinkCopiedExperimentDelete(expAnnot, container); + } + } + } + + /** + * + * @param expAnnot experiment associated with the container being deleted + * @param container container being deleted. This could be a subfolder of the experiment container if the experiment + * is configured to include subfolders. + */ + private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, Container container) + { + if (expAnnot.getDataVersion() != null && !ExperimentAnnotationsManager.isCurrentVersion(expAnnot)) + { + // This is an older version of the data on Panorama Public, so the folder should not have any files that are symlink targets. + return; + } + + Path deletedContainerPath = FileContentService.get().getFileRootPath(container, FileContentService.ContentType.files); + + List versions = ExperimentAnnotationsManager.getPublishedVersionsOfExperiment(expAnnot.getSourceExperimentId()); + + if (versions.size() > 1) + { + ExperimentAnnotations nextHighestVersion = versions.stream() + .filter(version -> version.getDataVersion() != null && !version.getDataVersion().equals(expAnnot.getDataVersion())) + .max(Comparator.comparing(ExperimentAnnotations::getDataVersion)).orElse(null); + + if (nextHighestVersion != null) + { + Container versionContainer = nextHighestVersion.getContainer(); + handleContainerSymlinks(versionContainer, (link, target) -> { + if (!target.startsWith(deletedContainerPath)) + { + return; + } + Files.move(target, link, REPLACE_EXISTING); // Move the files back to the next highest version of the experiment + _log.info("File moved from " + target + " to " + link); + + // This should update the symlinks in the submitted folder as well as + // symlinks in versions older than this one to point to the files in the next highest version. + fireSymlinkUpdate(target, link, container); + }); + } + } + + // If there were no previous versions then move the files back to the submitted folder. + // This will also take care of the case where there is a previous version but some files in the folder being + // deleted are not in the previous version (e.g. new files added to the source folder when data was resubmitted). + // These files should be moved back to the submitted folder + Container sourceContainer = ExperimentAnnotationsManager.getSourceExperimentContainer(expAnnot); + if (null == sourceContainer && null != expAnnot.getSourceExperimentPath()) + { + // Submitter may have deleted the ExperimentAnnotations in their folder. Try to lookup by sourceExperimentPath + sourceContainer = ContainerService.get().getForPath(expAnnot.getSourceExperimentPath()); + } + if (null != sourceContainer) + { + handleContainerSymlinks(sourceContainer, (link, target) -> { + if (!target.startsWith(deletedContainerPath)) + { + return; + } + Files.move(target, link, REPLACE_EXISTING); + _log.info("File moved from " + target + " to " + link); + + // Symlinks in the source container point to -> current version container on Panorama Public + // Symlinks in previous versions of the data on Panorama Public point to -> current version container on Panorama Public + // Only the container with the current version of the data on Panorama Public can have symlink targets. + // Here we are moving the file back to the source container, which means none of the previous versions had this file. + // So we don't need to fireSymlinkUpdate since there should not be any other symlinks targeting this file. + }); + } + } + + public void fireSymlinkUpdateContainer(Container oldContainer, Container newContainer) + { + // Update symlinks to new target + FileContentService fcs = FileContentService.get(); + if (fcs != null) + { + if (fcs.getFileRoot(oldContainer) != null && fcs.getFileRoot(newContainer) != null) + { + fireSymlinkUpdateContainer(fcs.getFileRoot(oldContainer).getPath(), fcs.getFileRoot(newContainer).getPath(), oldContainer); + } + } + } + + public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, Container container) + { + if (PanoramaPublicManager.canBeSymlinkTarget(container)) + { + String oldContainerPath = normalizeContainerPath(oldContainer); + String newContainerPath = normalizeContainerPath(newContainer); + + Set containers = getSymlinkContainers(container); + handleAllSymlinks(containers, (link, target) -> { + if (String.valueOf(target).contains(oldContainerPath)) + { + Path newTarget = Path.of(target.toString().replace(oldContainerPath, newContainerPath)); + try + { + Files.delete(link); + Files.createSymbolicLink(link, newTarget); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + }); + } + } + + public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container container) + { + if (PanoramaPublicManager.canBeSymlinkTarget(container)) // Only files in the Panorama Public project can be symlink targets. + { + Set containers = getSymlinkContainers(container); + + handleAllSymlinks(containers, (link, target) -> { + if (!target.equals(oldTarget)) + return; + + try + { + Files.delete(link); + Files.createSymbolicLink(link, newTarget); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + }); + } + } + + public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, File target, boolean createSourceSymLinks, @Nullable Logger log) throws IOException + { + if (null == log) + { + log = _log; + } + FileContentService fcs = FileContentService.get(); + if (null == fcs) + throw new RuntimeException("Unable to access FileContentService"); + + File[] files = source.listFiles(); + if (null != files) + { + for (File file : files) + { + if (file.isDirectory()) + { + Path targetPath = Path.of(target.getPath(), file.getName()); + if (!Files.exists(targetPath)) + { + Files.createDirectory(targetPath); + log.debug("Directory created: " + targetPath); + } + + moveAndSymLinkDirectory(job, file, targetPath.toFile(), createSourceSymLinks, log); + } + else + { + Path targetPath = Path.of(target.getPath(), file.getName()); + Path filePath = file.toPath(); + + // If this has already been copied, don't copy the symlink + if (Files.isSymbolicLink(filePath) && filePath.compareTo(targetPath) == 0) + continue; + + // Don't move over logs + if (FilenameUtils.getExtension(file.getPath()).equals("log")) + continue; + + // Check if move and symlink is manually disabled from the copy form. + // If on Windows (not the production server use-case), Windows cannot do symlink without admin permissions so + // just copy over the files. Also, if the file is a clib file, special handling is required so just copy it over. + if (!job.isMoveAndSymlink() || (!DEBUG_SYMLINKS_ON_WINDOWS && SystemUtils.IS_OS_WINDOWS) + || FilenameUtils.getExtension(file.getPath()).equals("clib")) + { + // If the file is a symlink, copy the target file over the symlink in the source project. + if (Files.isSymbolicLink(filePath)) + { + Files.copy(Files.readSymbolicLink(filePath), filePath, REPLACE_EXISTING); + log.debug("Copy file over symlink: " + filePath); + } + + // Copy the file to panorama public + Files.copy(filePath, targetPath, REPLACE_EXISTING); + fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer()); + + continue; + } + + // Symbolic link should move the target file over. This would be for a re-copy to public. + if (Files.isSymbolicLink(filePath)) + { + Path oldPath = Files.readSymbolicLink(filePath); + Files.move(oldPath, targetPath, REPLACE_EXISTING); + fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer()); + + fireSymlinkUpdate(oldPath, targetPath, job.getContainer()); // job container is the target container on Panorama Public + log.debug("File moved from " + oldPath + " to " + targetPath); + + Path symlink = Files.createSymbolicLink(oldPath, targetPath); + log.debug("Symlink created: " + symlink); + } + else + { + Files.move(filePath, targetPath, REPLACE_EXISTING); + fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer()); + + Files.createSymbolicLink(filePath, targetPath); + // We don't need to update any symlinks here since the source container should not have any symlink targets. + } + + if (createSourceSymLinks) + { + Path symlink = Files.createSymbolicLink(filePath, targetPath); + log.debug("Symlink created: " + symlink); + } + } + } + } + } + + private void verifyFileTreeSymlinks(File source, Map linkInvalidTarget, Map linkWithSymlinkTarget) throws IOException + { + for (File file : Objects.requireNonNull(source.listFiles())) + { + if (file.isDirectory()) + { + verifyFileTreeSymlinks(file, linkInvalidTarget, linkWithSymlinkTarget); + } + else + { + Path filePath = file.toPath(); + if (Files.isSymbolicLink(filePath)) + { + // Verify target file exists and is not a symbolic link + Path targetPath = Files.readSymbolicLink(filePath); + if (!FileUtil.isFileAndExists(targetPath)) + { + linkInvalidTarget.put(filePath.toString(), targetPath.toString()); + } + else if (Files.isSymbolicLink(targetPath)) + { + linkWithSymlinkTarget.put(filePath.toString(), targetPath.toString()); + } + } + } + } + } + + public boolean verifySymlinks() throws IOException + { + Map linkInvalidTarget = new HashMap<>(); + Map linkWithSymlinkTarget = new HashMap<>(); + Set containers = ContainerManager.getAllChildrenWithModule(ContainerManager.getRoot(), ModuleLoader.getInstance().getModule(PanoramaPublicModule.class)); + for (Container container : containers) + { + Set tree = ContainerManager.getAllChildren(container); + for (Container node : tree) + { + FileContentService fcs = FileContentService.get(); + if (null != fcs && null != fcs.getFileRoot(node)) + { + File root = fcs.getFileRoot(node); + if (null != root) + { + verifyFileTreeSymlinks(root, linkInvalidTarget, linkWithSymlinkTarget); + } + } + } + } + + if(linkInvalidTarget.size() > 0) + { + String linkInvalidTargets = linkInvalidTarget.entrySet().stream().map(String::valueOf).collect(Collectors.joining("\n")); + _log.error(linkInvalidTarget.size() + " Symlinks with invalid targets: \n" + linkInvalidTargets); + } + + if(linkWithSymlinkTarget.size() > 0) + { + String linkWithSymlinkTargets = linkWithSymlinkTarget.entrySet().stream().map(String::valueOf).collect(Collectors.joining("\n")); + _log.error(linkWithSymlinkTarget.size() + " Symlinks targeting symlinks: \n" + linkWithSymlinkTargets); + } + + return linkInvalidTarget.size() == 0 && linkWithSymlinkTarget.size() == 0; + } + + /** + * Returns a set of containers that can have symlinks that target files in targetContainer. This includes the source container, + * and containers with previous versions of the experiment. Only the container with the current version of the experiment + * can have symlink targets. + * @param targetContainer + * @return A set of containers that can have symlinks that target files in targetContainer + */ + private static Set getSymlinkContainers(Container targetContainer) + { + if (PanoramaPublicManager.canBeSymlinkTarget(targetContainer)) + { + ExperimentAnnotations expAnnotations = ExperimentAnnotationsManager.getExperimentIncludesContainer(targetContainer); + + if (expAnnotations != null && expAnnotations.getSourceExperimentId() != null) + { + Set symlinkContainers = new HashSet<>(); + Container sourceContainer = ExperimentAnnotationsManager.getSourceExperimentContainer(expAnnotations); + if (sourceContainer != null) + { + symlinkContainers.add(sourceContainer); + } + List exptVersions = ExperimentAnnotationsManager.getPublishedVersionsOfExperiment(expAnnotations.getSourceExperimentId()); + exptVersions.stream().map(ExperimentAnnotations::getContainer) + .filter(c -> !c.equals(targetContainer)) + .forEach(symlinkContainers::add); // Add containers for other versions + return symlinkContainers; + } + } + return Collections.emptySet(); + } +} diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java index 5b7cfd78..9257af61 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentFinalTask.java @@ -24,8 +24,6 @@ import org.labkey.api.data.DbScope; import org.labkey.api.data.PropertyManager; import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpExperiment; -import org.labkey.api.exp.api.ExpRun; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.files.FileContentService; import org.labkey.api.files.view.FilesWebPart; @@ -35,7 +33,6 @@ import org.labkey.api.pipeline.PipelineJobException; import org.labkey.api.pipeline.RecordedActionSet; import org.labkey.api.portal.ProjectUrls; -import org.labkey.api.query.BatchValidationException; import org.labkey.api.query.ValidationException; import org.labkey.api.security.Group; import org.labkey.api.security.InvalidGroupMembershipException; @@ -52,7 +49,6 @@ import org.labkey.api.security.roles.ReaderRole; import org.labkey.api.security.roles.Role; import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.targetedms.ITargetedMSRun; import org.labkey.api.targetedms.TargetedMSService; import org.labkey.api.util.FileType; import org.labkey.api.util.FileUtil; @@ -65,6 +61,7 @@ import org.labkey.panoramapublic.PanoramaPublicController; import org.labkey.panoramapublic.PanoramaPublicManager; import org.labkey.panoramapublic.PanoramaPublicNotification; +import org.labkey.panoramapublic.PanoramaPublicSymlinkManager; import org.labkey.panoramapublic.datacite.DataCiteException; import org.labkey.panoramapublic.datacite.DataCiteService; import org.labkey.panoramapublic.datacite.Doi; @@ -72,23 +69,18 @@ import org.labkey.panoramapublic.model.Journal; import org.labkey.panoramapublic.model.JournalSubmission; import org.labkey.panoramapublic.model.Submission; -import org.labkey.panoramapublic.model.speclib.SpecLibInfo; import org.labkey.panoramapublic.proteomexchange.ProteomeXchangeService; import org.labkey.panoramapublic.proteomexchange.ProteomeXchangeServiceException; import org.labkey.panoramapublic.query.CatalogEntryManager; import org.labkey.panoramapublic.query.ExperimentAnnotationsManager; import org.labkey.panoramapublic.query.JournalManager; -import org.labkey.panoramapublic.query.ModificationInfoManager; -import org.labkey.panoramapublic.query.SpecLibInfoManager; import org.labkey.panoramapublic.query.SubmissionManager; -import org.labkey.panoramapublic.query.modification.ExperimentIsotopeModInfo; -import org.labkey.panoramapublic.query.modification.ExperimentStructuralModInfo; import org.labkey.panoramapublic.security.PanoramaPublicSubmitterRole; +import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -131,35 +123,12 @@ public RecordedActionSet run() throws PipelineJobException private void finishUp(PipelineJob job, CopyExperimentJobSupport jobSupport) throws Exception { - // Get the experiment that was just created in the target folder as part of the folder import. Container container = job.getContainer(); User user = job.getUser(); - List experiments = ExperimentService.get().getExperiments(container, user, false, false); - if(experiments.size() == 0) - { - throw new PipelineJobException("No experiments found in the folder " + container.getPath()); - } - else if (experiments.size() > 1) - { - throw new PipelineJobException("More than one experiment found in the folder " + container.getPath()); - } - ExpExperiment experiment = experiments.get(0); - - // Get a list of all the ExpRuns imported to subfolders of this folder. - int[] runRowIdsInSubfolders = getAllExpRunRowIdsInSubfolders(container); Logger log = job.getLogger(); try(DbScope.Transaction transaction = PanoramaPublicManager.getSchema().getScope().ensureTransaction()) { - if(runRowIdsInSubfolders.length > 0) - { - // The folder export and import process, creates a new experiment in exp.experiment. - // However, only runs in the top-level folder are added to this experiment. - // We will add to the experiment all the runs imported to subfolders. - log.info("Adding runs imported in subfolders."); - ExperimentAnnotationsManager.addSelectedRunsToExperiment(experiment, runRowIdsInSubfolders, user); - } - // Get the ExperimentAnnotations in the source container ExperimentAnnotations sourceExperiment = jobSupport.getExpAnnotations(); // Get the submission request @@ -174,8 +143,9 @@ else if (experiments.size() > 1) // can be assigned to the new copy. ExperimentAnnotations previousCopy = getPreviousCopyRemoveShortUrl(latestCopiedSubmission, user); - // Create a new row in panoramapublic.ExperimentAnnotations and link it to the new experiment created during folder import. - ExperimentAnnotations targetExperiment = createNewExperimentAnnotations(experiment, sourceExperiment, js, previousCopy, jobSupport, user, log); + // Update the row in panoramapublic.ExperimentAnnotations - set the shortURL and version + ExperimentAnnotations targetExperiment = updateExperimentAnnotations(container, sourceExperiment, js, previousCopy, jobSupport, user, log); + // If there is a Panorama Public data catalog entry associated with the previous copy of the experiment, move it to the // new container. @@ -194,12 +164,6 @@ else if (experiments.size() > 1) updateDataPathsAndRawDataTab(targetExperiment, user, log); - // Copy any Spectral library information provided by the user in the source container - copySpecLibInfos(sourceExperiment, targetExperiment, user); - - // Copy any modifications related information provided by the user in the source container - copyModificationInfos(sourceExperiment, targetExperiment, user); - // Assign a reviewer account if one was requested Pair reviewer = assignReviewer(js, targetExperiment, previousCopy, jobSupport, currentSubmission.isKeepPrivate(), user, log); @@ -209,6 +173,16 @@ else if (experiments.size() > 1) js = updateSubmissionAndDeletePreviousCopy(js, currentSubmission, latestCopiedSubmission, targetExperiment, previousCopy, jobSupport, user, log); + alignSymlinks(job, jobSupport); + + cleanupExportDirectory(user, jobSupport.getExportDir()); + + verifySymlinks(sourceExperiment.getContainer(), container, true); + if (null != previousCopy) + { + verifySymlinks(previousCopy.getContainer(), container, false); + } + // Create notifications. Do this at the end after everything else is done. PanoramaPublicNotification.notifyCopied(sourceExperiment, targetExperiment, jobSupport.getJournal(), js.getJournalExperiment(), currentSubmission, reviewer.first, reviewer.second, user, previousCopy != null /*This is a re-copy if previousCopy exists*/); @@ -217,50 +191,94 @@ else if (experiments.size() > 1) } } - private void moveCatalogEntry(ExperimentAnnotations previousCopy, ExperimentAnnotations targetExperiment, User user) throws PipelineJobException + /** + * Verify symlinks created during the copy process are valid symlinks and point to the target container. + * @param source The copied experiment source container + * @param target The target container on Panorma Public + * @param matchingPath If true the target files will have the same relative path in the target container + */ + private void verifySymlinks(Container source, Container target, boolean matchingPath) { - if (previousCopy == null) - { - return; - } - try + FileContentService fcs = FileContentService.get(); + if (fcs != null) { - CatalogEntryManager.moveEntry(previousCopy, targetExperiment, user); + Path targetRoot = fcs.getFileRootPath(target); + if (null != targetRoot) + { + Path targetFileRoot = Path.of(targetRoot.toString(), File.separator); + PanoramaPublicSymlinkManager.get().handleContainerSymlinks(source, (sourceFile, targetFile) -> { + + // valid path + if (!FileUtil.isFileAndExists(targetFile)) + { + throw new IllegalStateException("Symlink " + sourceFile + " points to an invalid target " + targetFile); + } + + // we don't want symlinks pointing at other symlinks + if (Files.isSymbolicLink(targetFile)) + { + throw new IllegalStateException("Symlink " + sourceFile + " points to another symlink " + targetFile); + } + + // Check if symlink target is in the target container + if (!targetFile.startsWith(targetFileRoot)) + { + throw new IllegalStateException("Symlink " + sourceFile + " points to " + targetFile + " which is outside the target container " + target.getPath()); + } + + // Symlink targets should have exact same relative path as the source file. Previous versions will not + // necessarily have same path so don't check. + if (matchingPath) + { + String targetFileRelative = targetFile.toString().substring(targetFileRoot.toString().length()); + if (!sourceFile.toString().endsWith(targetFileRelative)) + { + throw new IllegalStateException("Symlink " + sourceFile + " points to " + targetFile + " which is not at the same path as the source file " + sourceFile); + } + } + }); + } } - catch (IOException e) + } + + private void cleanupExportDirectory(User user, File directory) throws IOException + { + List datas = ExperimentService.get().getExpDatasUnderPath(directory.toPath(), null, true); + for (ExpData data : datas) { - throw new PipelineJobException(String.format("Could not move Panorama Public catalog entry from the previous copy of the data in folder '%s' to the new folder '%s'.", - previousCopy.getContainer().getPath(), targetExperiment.getContainer().getPath()), e); + data.delete(user); } + FileUtil.deleteDir(directory); } - private void copySpecLibInfos(ExperimentAnnotations sourceExperiment, ExperimentAnnotations targetExperiment, User user) + // After a full file copy is done re-align the experiment project's symlinks to the newest version of the public project + private void alignSymlinks(PipelineJob job, CopyExperimentJobSupport jobSupport) { - List specLibInfos = SpecLibInfoManager.getForExperiment(sourceExperiment.getId(), sourceExperiment.getContainer()); - for (SpecLibInfo info: specLibInfos) + if (jobSupport.getPreviousVersionName() != null) { - info.setId(0); - info.setExperimentAnnotationsId(targetExperiment.getId()); - SpecLibInfoManager.save(info, user); + FileContentService fcs = FileContentService.get(); + if (fcs != null) + { + PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(jobSupport.getPreviousVersionName(), + fcs.getFileRoot(job.getContainer()).getPath(), job.getContainer()); + } } } - private void copyModificationInfos(ExperimentAnnotations sourceExperiment, ExperimentAnnotations targetExperiment, User user) + private void moveCatalogEntry(ExperimentAnnotations previousCopy, ExperimentAnnotations targetExperiment, User user) throws PipelineJobException { - List strModInfos = ModificationInfoManager.getStructuralModInfosForExperiment(sourceExperiment.getId(), sourceExperiment.getContainer()); - for (ExperimentStructuralModInfo info: strModInfos) + if (previousCopy == null) { - info.setId(0); - info.setExperimentAnnotationsId(targetExperiment.getId()); - ModificationInfoManager.saveStructuralModInfo(info, user); + return; } - - List isotopeModInfos = ModificationInfoManager.getIsotopeModInfosForExperiment(sourceExperiment.getId(), sourceExperiment.getContainer()); - for (ExperimentIsotopeModInfo info: isotopeModInfos) + try { - info.setId(0); - info.setExperimentAnnotationsId(targetExperiment.getId()); - ModificationInfoManager.saveIsotopeModInfo(info, user); + CatalogEntryManager.moveEntry(previousCopy, targetExperiment, user); + } + catch (IOException e) + { + throw new PipelineJobException(String.format("Could not move Panorama Public catalog entry from the previous copy of the data in folder '%s' to the new folder '%s'.", + previousCopy.getContainer().getPath(), targetExperiment.getContainer().getPath()), e); } } @@ -349,7 +367,7 @@ private Pair assignReviewer(JournalSubmission js, ExperimentAnnota return new Pair<>(null, null); } - private void updateDataPathsAndRawDataTab(ExperimentAnnotations targetExperiment, User user, Logger log) throws BatchValidationException, PipelineJobException + private void updateDataPathsAndRawDataTab(ExperimentAnnotations targetExperiment, User user, Logger log) { FileContentService service = FileContentService.get(); if (service != null) @@ -360,14 +378,6 @@ private void updateDataPathsAndRawDataTab(ExperimentAnnotations targetExperiment // we had to update the file root property on the Files webpart in the Raw Data tab in the Panorama Public copy. log.info("Updating the 'Raw Data' tab configuration"); updateRawDataTabConfig(targetExperiment.getContainer(), service, user); - - // DataFileUrl in exp.data and FilePathRoot in exp.experimentRun point to locations in the 'export' directory. - // We are now copying all files from the source container to the target container file root. Update the paths - // to point to locations in the target container file root, and delete the 'export' directory - if (!updateDataPaths(targetExperiment.getContainer(), service, user, log)) - { - throw new PipelineJobException("Unable to update all data file paths."); - } } } @@ -441,16 +451,17 @@ private ExperimentAnnotations getPreviousCopyRemoveShortUrl(Submission latestCop } @NotNull - private ExperimentAnnotations createNewExperimentAnnotations(ExpExperiment experiment, ExperimentAnnotations sourceExperiment, JournalSubmission js, - ExperimentAnnotations previousCopy, CopyExperimentJobSupport jobSupport, - User user, Logger log) throws PipelineJobException + private ExperimentAnnotations updateExperimentAnnotations(Container targetContainer, ExperimentAnnotations sourceExperiment, JournalSubmission js, + ExperimentAnnotations previousCopy, CopyExperimentJobSupport jobSupport, + User user, Logger log) throws PipelineJobException { - log.info("Creating a new TargetedMS experiment entry in panoramapublic.ExperimentAnnotations."); - ExperimentAnnotations targetExperiment = createExperimentCopy(sourceExperiment); - targetExperiment.setExperimentId(experiment.getRowId()); - targetExperiment.setContainer(experiment.getContainer()); - targetExperiment.setSourceExperimentId(sourceExperiment.getId()); - targetExperiment.setSourceExperimentPath(sourceExperiment.getContainer().getPath()); + + log.info("Updating TargetedMS experiment entry in target folder " + targetContainer.getPath()); + ExperimentAnnotations targetExperiment = ExperimentAnnotationsManager.getExperimentInContainer(targetContainer); + if (targetExperiment == null) + { + throw new PipelineJobException("ExperimentAnnotations row does not exist in target folder: '" + targetContainer.getPath() + "'"); + } targetExperiment.setShortUrl(js.getShortAccessUrl()); Integer currentVersion = ExperimentAnnotationsManager.getMaxVersionForExperiment(sourceExperiment.getId()); int version = currentVersion == null ? 1 : currentVersion + 1; @@ -477,28 +488,6 @@ private ExperimentAnnotations createNewExperimentAnnotations(ExpExperiment exper return targetExperiment; } - private ExperimentAnnotations createExperimentCopy(ExperimentAnnotations source) - { - ExperimentAnnotations copy = new ExperimentAnnotations(); - copy.setTitle(source.getTitle()); - copy.setExperimentDescription(source.getExperimentDescription()); - copy.setSampleDescription(source.getSampleDescription()); - copy.setOrganism(source.getOrganism()); - copy.setInstrument(source.getInstrument()); - copy.setSpikeIn(source.getSpikeIn()); - copy.setCitation(source.getCitation()); - copy.setAbstract(source.getAbstract()); - copy.setPublicationLink(source.getPublicationLink()); - copy.setIncludeSubfolders(source.isIncludeSubfolders()); - copy.setKeywords(source.getKeywords()); - copy.setLabHead(source.getLabHead()); - copy.setSubmitter(source.getSubmitter()); - copy.setLabHeadAffiliation(source.getLabHeadAffiliation()); - copy.setSubmitterAffiliation(source.getSubmitterAffiliation()); - copy.setPubmedId(source.getPubmedId()); - return copy; - } - private void updatePermissions(User formSubmitter, ExperimentAnnotations targetExperiment, ExperimentAnnotations sourceExperiment, ExperimentAnnotations previousCopy, Journal journal, User pipelineJobUser, Logger log) { @@ -565,7 +554,6 @@ private void addToGroup(User user, String groupName, Container project, Logger l if (group != null) { addToGroup(user, group, log); - return; } } @@ -673,121 +661,6 @@ private void assignDoi(ExperimentAnnotations targetExpt, boolean useTestDb) thro targetExpt.setDoi(doi.getDoi()); } - private boolean updateDataPaths(Container target, FileContentService service, User user, Logger logger) throws BatchValidationException - { - List allRuns = getAllExpRuns(target); - - for(ExpRun run: allRuns) - { - Path fileRootPath = service.getFileRootPath(run.getContainer(), FileContentService.ContentType.files); - if(fileRootPath == null || !Files.exists(fileRootPath)) - { - logger.error("File root path for container " + run.getContainer().getPath() + " does not exist: " + fileRootPath); - return false; - } - - List allData = new ArrayList<>(run.getAllDataUsedByRun()); - - ITargetedMSRun tmsRun = PanoramaPublicManager.getRunByLsid(run.getLSID(), run.getContainer()); - if(tmsRun == null) - { - logger.error("Could not find a targetedms run for exprun: " + run.getLSID() + " in container " + run.getContainer()); - return false; - } - logger.info("Updating dataFileUrls for run " + tmsRun.getFileName() + "; targetedms run ID " + tmsRun.getId()); - - // list returned by run.getAllDataUsedByRun() may not include rows in exp.data that do not have a corresponding row in exp.dataInput. - // This is known to happen for entries for .skyd datas. - if(!addMissingDatas(allData, tmsRun, logger)) - { - return false; - } - - for(ExpData data: allData) - { - String fileName = FileUtil.getFileName(data.getFilePath()); - Path newDataPath = fileRootPath.resolve(fileName); - if(!Files.exists(newDataPath)) - { - // This may be the .skyd file which is inside the exploded parent directory - String parentDir = tmsRun.getBaseName(); - newDataPath = fileRootPath.resolve(parentDir) // Name of the exploded directory - .resolve(fileName); // Name of the file - } - if(Files.exists(newDataPath)) - { - logger.info("Updating dataFileUrl..."); - logger.info("from: " + data.getDataFileURI().toString()); - logger.info("to: " + newDataPath.toUri()); - data.setDataFileURI(newDataPath.toUri()); - data.save(user); - } - else - { - logger.error("Data path does not exist: " + newDataPath); - return false; - } - } - - run.setFilePathRootPath(fileRootPath); - run.save(user); - } - return true; - } - - private boolean addMissingDatas(List allData, ITargetedMSRun tmsRun, Logger logger) - { - Integer skyZipDataId = tmsRun.getDataId(); - boolean skyZipDataIncluded = false; - - Integer skydDataId = tmsRun.getSkydDataId(); - boolean skydDataIncluded = false; - - if(skydDataId == null) - { - // Document may not have a skydDataId, for example, if it does not have any imported replicates. So no .skyd file - logger.info("Targetedms run " + tmsRun.getId() + " in container " + tmsRun.getContainer() + " does not have a skydDataId." ); - skydDataIncluded = true; // Set to true so we don't try to add it later - } - - for(ExpData data: allData) - { - if(data.getRowId() == skyZipDataId) - { - skyZipDataIncluded = true; - } - else if((skydDataId != null) && (data.getRowId() == skydDataId)) - { - skydDataIncluded = true; - } - } - - if(!skyZipDataIncluded) - { - ExpData skyZipData = ExperimentService.get().getExpData(skyZipDataId); - if(skyZipData == null) - { - logger.error("Could not find expdata for dataId (.sky.zip): " + tmsRun.getDataId() +" for runId" + tmsRun.getId() + " in container " + tmsRun.getContainer()); - return false; - } - logger.info("Adding ExpData for .sky.zip file"); - allData.add(skyZipData); - } - - if(!skydDataIncluded) - { - ExpData skydData = ExperimentService.get().getExpData(tmsRun.getSkydDataId()); - if (skydData == null) - { - logger.error("Could not find expdata for skydDataId (.skyd): " + tmsRun.getDataId() + " for runId " + tmsRun.getId() + " in container " + tmsRun.getContainer()); - return false; - } - logger.info("Adding ExpData for .skyd file"); - allData.add(skydData); - } - return true; - } - private void updateRawDataTabConfig(Container c, FileContentService service, User user) { Set children = ContainerManager.getAllChildren(c); // Includes parent @@ -823,45 +696,6 @@ private void updateRawDataTab(Container c, FileContentService service, User user } } - private List getAllExpRuns(Container container) - { - Set children = ContainerManager.getAllChildren(container); - ExperimentService expService = ExperimentService.get(); - List allRuns = new ArrayList<>(); - - for(Container child: children) - { - List runs = expService.getExpRuns(child, null, null); - allRuns.addAll(runs); - } - return allRuns; - } - - private static int[] getAllExpRunRowIdsInSubfolders(Container container) - { - Set children = ContainerManager.getAllChildren(container); - ExperimentService expService = ExperimentService.get(); - List expRunRowIds = new ArrayList<>(); - for(Container child: children) - { - if(container.equals(child)) - { - continue; - } - List runs = expService.getExpRuns(child, null, null); - for(ExpRun run: runs) - { - expRunRowIds.add(run.getRowId()); - } - } - int[] intIds = new int[expRunRowIds.size()]; - for(int i = 0; i < expRunRowIds.size(); i++) - { - intIds[i] = expRunRowIds.get(i); - } - return intIds; - } - private void updateLastCopiedExperiment(ExperimentAnnotations previousCopy, JournalSubmission js, Journal journal, User user, Logger log) throws PipelineJobException { if (previousCopy.getDataVersion() == null) diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentJobSupport.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentJobSupport.java index 819508eb..9ce69da1 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentJobSupport.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentJobSupport.java @@ -19,6 +19,7 @@ import org.labkey.panoramapublic.model.Journal; import java.io.File; +import java.nio.file.Path; /** * User: vsharma @@ -41,5 +42,11 @@ public interface CopyExperimentJobSupport boolean assignDoi(); boolean useDataCiteTestApi(); + boolean isMoveAndSymlink(); + boolean deletePreviousCopy(); + + String getPreviousVersionName(); + + void setExportTargetPath(Path exportTargetPath); } diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentPipelineJob.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentPipelineJob.java index f460c9c9..9c40977f 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentPipelineJob.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/CopyExperimentPipelineJob.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.labkey.api.data.Container; import org.labkey.api.pipeline.LocalDirectory; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineJob; @@ -35,6 +36,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Path; /** * User: vsharma @@ -55,9 +57,16 @@ public class CopyExperimentPipelineJob extends PipelineJob implements CopyExperi private boolean _assignDoi; private boolean _useDataCiteTestApi; + private boolean _moveAndSymlink; private boolean _deletePreviousCopy; + private String _previousVersionName; + + private Path _exportTargetPath; + + private Container _exportSourceContainer; + @JsonCreator protected CopyExperimentPipelineJob(@JsonProperty("_experimentAnnotations") ExperimentAnnotations experiment, @JsonProperty("_journal") Journal journal, @@ -153,7 +162,7 @@ public Journal getJournal() @Override public File getExportDir() { - return new File(getLocalDirectory().getLocalDirectoryFile(), PipelineService.EXPORT_DIR); + return _exportTargetPath.toFile(); } @Override @@ -186,6 +195,12 @@ public boolean useDataCiteTestApi() return _useDataCiteTestApi; } + @Override + public boolean isMoveAndSymlink() + { + return _moveAndSymlink; + } + @Override public boolean deletePreviousCopy() { @@ -217,8 +232,39 @@ public void setUseDataCiteTestApi(boolean useDataCiteTestApi) _useDataCiteTestApi = useDataCiteTestApi; } + public void setMoveAndSymlink(boolean moveAndSymlink) + { + _moveAndSymlink = moveAndSymlink; + } + public void setDeletePreviousCopy(boolean deletePreviousCopy) { _deletePreviousCopy = deletePreviousCopy; } + + public String getPreviousVersionName() + { + return _previousVersionName; + } + + public void setPreviousVersionName(String previousVersionName) + { + _previousVersionName = previousVersionName; + } + + @Override + public void setExportTargetPath(Path exportTargetPath) + { + _exportTargetPath = exportTargetPath; + } + + public Container getExportSourceContainer() + { + return _exportSourceContainer; + } + + public void setExportSourceContainer(Container exportSourceContainer) + { + _exportSourceContainer = exportSourceContainer; + } } diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentExportTask.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentExportTask.java index 93379034..c5288778 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentExportTask.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentExportTask.java @@ -16,10 +16,11 @@ package org.labkey.panoramapublic.pipeline; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; +import org.labkey.api.admin.FolderArchiveDataTypes; import org.labkey.api.admin.FolderExportContext; import org.labkey.api.admin.FolderWriterImpl; -import org.labkey.api.admin.FolderArchiveDataTypes; import org.labkey.api.admin.StaticLoggerGetter; import org.labkey.api.data.Container; import org.labkey.api.data.PHI; @@ -34,11 +35,13 @@ import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.writer.FileSystemFile; +import org.labkey.panoramapublic.PanoramaPublicManager; import org.labkey.panoramapublic.model.ExperimentAnnotations; import java.io.File; import java.util.Collections; import java.util.List; +import java.util.Set; /** * User: vsharma @@ -47,6 +50,8 @@ */ public class ExperimentExportTask extends PipelineJob.Task { + private static final Logger _log = LogManager.getLogger(ExperimentExportTask.class); + private ExperimentExportTask(Factory factory, PipelineJob job) { super(factory, job); @@ -97,16 +102,16 @@ public void writeExperiment(CopyExperimentJobSupport support, ExperimentAnnotati FolderArchiveDataTypes.CONTAINER_SPECIFIC_MODULE_PROPERTIES, // "Container specific module properties", FolderArchiveDataTypes.EXPERIMENTS_AND_RUNS, // "Experiments and runs" FolderArchiveDataTypes.LISTS, // "Lists" - FolderArchiveDataTypes.FILES, // "Files" TargetedMSService.QC_FOLDER_DATA_TYPE - }; + }; + Set templateWriterSet = PageFlowUtil.set(templateWriterTypes); boolean includeSubfolders = exptAnnotations.isIncludeSubfolders(); Container source = exptAnnotations.getContainer(); FolderWriterImpl writer = new FolderWriterImpl(); - FolderExportContext ctx = new FolderExportContext(user, source, PageFlowUtil.set(templateWriterTypes), + FolderExportContext ctx = new FolderExportContext(user, source, templateWriterSet, null, includeSubfolders, PHI.NotPHI, false, false, false, new StaticLoggerGetter(LogManager.getLogger(FolderWriterImpl.class))); @@ -126,6 +131,7 @@ public void writeExperiment(CopyExperimentJobSupport support, ExperimentAnnotati writer.write(source, ctx, vf); FilesMetadataWriter filesMetadataWriter = new FilesMetadataWriter(); filesMetadataWriter.write(exptAnnotations.getContainer(), exptAnnotations.isIncludeSubfolders(), vf, user, getJob().getLogger()); + } public static class Factory extends AbstractTaskFactory diff --git a/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentImportTask.java b/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentImportTask.java index f46b5a86..75411602 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentImportTask.java +++ b/panoramapublic/src/org/labkey/panoramapublic/pipeline/ExperimentImportTask.java @@ -82,7 +82,8 @@ public static void doImport(PipelineJob job, CopyExperimentJobSupport jobSupport } File folderXml = new File(importDir, "folder.xml"); - if(!folderXml.exists()){ + if(!folderXml.exists()) + { throw new Exception("This directory doesn't contain an appropriate xml: " + importDir.getAbsolutePath()); } diff --git a/panoramapublic/src/org/labkey/panoramapublic/query/ExperimentAnnotationsManager.java b/panoramapublic/src/org/labkey/panoramapublic/query/ExperimentAnnotationsManager.java index 907b3dc7..c144c5ea 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/query/ExperimentAnnotationsManager.java +++ b/panoramapublic/src/org/labkey/panoramapublic/query/ExperimentAnnotationsManager.java @@ -84,7 +84,7 @@ public static ExperimentAnnotations get(int experimentAnnotationsId, Container c * @param experimentId FK -> exp.experiment.rowId * @return ExperimentAnnotations object with the given experimentId */ - private static ExperimentAnnotations getForExperimentId(int experimentId) + public static ExperimentAnnotations getForExperimentId(int experimentId) { return new TableSelector(PanoramaPublicManager.getTableInfoExperimentAnnotations(), new SimpleFilter(FieldKey.fromParts("ExperimentId"), experimentId), null).getObject(ExperimentAnnotations.class); @@ -552,7 +552,7 @@ public static boolean isCurrentVersion(ExperimentAnnotations experimentAnnotatio if (experimentAnnotations != null && experimentAnnotations.isJournalCopy()) { Integer maxVersion = ExperimentAnnotationsManager.getMaxVersionForExperiment(experimentAnnotations.getSourceExperimentId()); - return experimentAnnotations.getDataVersion().equals(maxVersion); + return experimentAnnotations.getDataVersion() != null && experimentAnnotations.getDataVersion().equals(maxVersion); } return false; } @@ -568,6 +568,12 @@ public static List getPublishedVersionsOfExperiment(int s return new TableSelector(PanoramaPublicManager.getTableInfoExperimentAnnotations(), filter, sort).getArrayList(ExperimentAnnotations.class); } + public static @Nullable Container getSourceExperimentContainer(ExperimentAnnotations expAnnotations) + { + ExperimentAnnotations sourceExpt = ExperimentAnnotationsManager.get(expAnnotations.getSourceExperimentId()); + return sourceExpt != null ? sourceExpt.getContainer() : null; + } + /** * @param container * @param user diff --git a/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp b/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp index 8166e488..c5b3c5e7 100644 --- a/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp +++ b/panoramapublic/src/org/labkey/panoramapublic/view/publish/copyExperimentForm.jsp @@ -207,6 +207,13 @@ name: 'useDataCiteTestApi', boxLabel: 'Check this box for tests so that we get a DOI with the DataCite test API.' }, + { + xtype: 'checkbox', + fieldLabel: "Move and Symlink Files", + checked: <%=form.isMoveAndSymlink()%>, + name: 'moveAndSymlink', + boxLabel: 'Check this box to perform a move and symlink of the files instead of a full copy.' + }, { xtype: 'textfield', hidden: <%=!currentSubmission.isKeepPrivate() || isRecopy%>, diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java index 19d57ec1..2f55e345 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicBaseTest.java @@ -3,8 +3,13 @@ import org.apache.commons.collections4.CollectionUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; +import org.labkey.remoteapi.CommandException; +import org.labkey.remoteapi.CommandResponse; +import org.labkey.remoteapi.Connection; +import org.labkey.remoteapi.SimpleGetCommand; import org.labkey.test.Locator; import org.labkey.test.TestFileUtils; import org.labkey.test.TestTimeoutException; @@ -27,6 +32,7 @@ import org.openqa.selenium.support.ui.WebDriverWait; import java.io.File; +import java.io.IOException; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; @@ -47,6 +53,25 @@ public class PanoramaPublicBaseTest extends TargetedMSTest implements PostgresOn static final String REVIEWER_PREFIX = "panoramapublictest_reviewer"; + static final String SKY_FILE_1 = "Study9S_Site52_v1.sky.zip"; + static final String SKY_FOLDER_NAME = "Study9S_Site52_v1"; + static final String RAW_FILE_WIFF = "Site52_041009_Study9S_Phase-I.wiff"; + static final String RAW_FILE_WIFF_RENAMED = RAW_FILE_WIFF + ".RENAMED"; + static final String RAW_FILE_WIFF_SCAN = RAW_FILE_WIFF + ".scan"; + static final String QC_1 = "QC_1"; + static final String QC_1_SKY = QC_1 + ".sky"; + static final String QC_1_SKY_ZIP = QC_1_SKY + ".zip"; + static final String QC_1_SKY_VIEW = QC_1_SKY + ".view"; + static final String QC_1_SKYD = QC_1_SKY + "d"; + static final String SMALL_MOL_FILES = "SmallMoleculeFiles"; + static final String SMALLMOL_PLUS_PEPTIDES = "smallmol_plus_peptides"; + static final String SMALLMOL_PLUS_PEPTIDES_SKY = SMALLMOL_PLUS_PEPTIDES + ".sky"; + static final String SMALLMOL_PLUS_PEPTIDES_SKYD = SMALLMOL_PLUS_PEPTIDES_SKY + "d"; + static final String SMALLMOL_PLUS_PEPTIDES_SKY_VIEW = SMALLMOL_PLUS_PEPTIDES_SKY + ".view"; + static final String SMALLMOL_PLUS_PEPTIDES_SKY_ZIP = SMALLMOL_PLUS_PEPTIDES_SKY + ".zip"; + + static final String SAMPLEDATA_FOLDER = "panoramapublic/"; + PortalHelper portalHelper = new PortalHelper(this); @Override @@ -75,6 +100,16 @@ public static void initProject() init.setupFolder(FolderType.Experiment); } + @After + public void afterTest() throws IOException, CommandException + { + if (isImpersonating()) + { + stopImpersonating(); + } + verifySymlinks(); + } + private void createPanoramaPublicJournalProject() { // Create a "Panorama Public" project where we will copy data. @@ -102,6 +137,16 @@ private void createPanoramaPublicJournalProject() _permissionsHelper.createProjectGroup(PANORAMA_PUBLIC_SUBMITTERS, PANORAMA_PUBLIC); } + boolean verifySymlinks() throws IOException, CommandException + { + Connection connection = createDefaultConnection(); + SimpleGetCommand command = new SimpleGetCommand("PanoramaPublic", "verifySymlinks"); + CommandResponse verifyResponse = command.execute(connection, "/"); + + // Failure will throw exception and put results in log + return verifyResponse.getProperty("success") != null; + } + @NotNull TargetedMsExperimentWebPart createExperiment(String experimentTitle) { @@ -202,6 +247,27 @@ String submitIncompletePxButton() return submitFormAndGetAccessLink(); } + void resubmitWithoutPxd(boolean fromDataValidationPage, boolean keepPrivate) + { + if (fromDataValidationPage) + { + clickButton("Submit without a ProteomeXchange ID"); + } + else + { + clickAndWait(Locator.linkContainingText("Submit without a ProteomeXchange ID")); + } + waitForText("Resubmit Request to "); + if (!keepPrivate) + { + uncheck("Keep Private:"); + } + click(Ext4Helper.Locators.ext4Button(("Resubmit"))); + waitForText("Confirm resubmission request to"); + click(Locator.lkButton("OK")); // Confirm to proceed with the submission. + waitAndClickAndWait(Locator.linkWithText("Back to Experiment Details")); // Navigate to the experiment details page. + } + private String submitFormAndGetAccessLink() { submitForm(); @@ -237,22 +303,30 @@ void makeCopy(String shortAccessUrl, String experimentTitle, String destinationF { stopImpersonating(); } - makeCopy(shortAccessUrl, experimentTitle, recopy, deleteOldCopy, destinationFolder); + makeCopy(shortAccessUrl, experimentTitle, recopy, deleteOldCopy, destinationFolder, true); } void copyExperimentAndVerify(String projectName, String folderName, @Nullable List subfolders, String experimentTitle, @Nullable Integer version, boolean recopy, boolean deleteOldCopy, String destinationFolder, String shortAccessUrl) + { + copyExperimentAndVerify(projectName, folderName, subfolders, experimentTitle, version, recopy, deleteOldCopy, + destinationFolder, shortAccessUrl, true); + } + + void copyExperimentAndVerify(String projectName, String folderName, @Nullable List subfolders, String experimentTitle, + @Nullable Integer version, boolean recopy, boolean deleteOldCopy, String destinationFolder, + String shortAccessUrl, boolean symlinks) { if(isImpersonating()) { stopImpersonating(); } - makeCopy(shortAccessUrl, experimentTitle, recopy, deleteOldCopy, destinationFolder); + makeCopy(shortAccessUrl, experimentTitle, recopy, deleteOldCopy, destinationFolder, symlinks); verifyCopy(shortAccessUrl, experimentTitle, version, projectName, folderName, subfolders, recopy); } - private void makeCopy(String shortAccessUrl, String experimentTitle, boolean recopy, boolean deleteOldCopy, String destinationFolder) + private void makeCopy(String shortAccessUrl, String experimentTitle, boolean recopy, boolean deleteOldCopy, String destinationFolder, boolean symlinks) { goToProjectHome(PANORAMA_PUBLIC); impersonateGroup(PANORAMA_PUBLIC_GROUP, false); @@ -268,6 +342,10 @@ private void makeCopy(String shortAccessUrl, String experimentTitle, boolean rec setFormElement(Locator.tagWithName("input", "destContainerName"), destinationFolder); uncheck("Assign ProteomeXchange ID:"); uncheck("Assign Digital Object Identifier:"); + if (!symlinks) + { + uncheck("Move and Symlink Files:"); + } if(recopy) { diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicMakePublicTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicMakePublicTest.java index 684e14c3..60454ed1 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicMakePublicTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicMakePublicTest.java @@ -4,6 +4,7 @@ import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.labkey.remoteapi.CommandException; import org.labkey.test.BaseWebDriverTest; import org.labkey.test.Locator; import org.labkey.test.TestFileUtils; @@ -20,6 +21,7 @@ import org.openqa.selenium.WebElement; import java.io.File; +import java.io.IOException; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -38,7 +40,7 @@ public class PanoramaPublicMakePublicTest extends PanoramaPublicBaseTest private static final String IMAGE_PATH = "TargetedMS/panoramapublic/" + IMAGE_FILE; @Test - public void testExperimentCopy() + public void testExperimentCopy() throws IOException, CommandException { // Set up our source folder. We will create an experiment and submit it to our "Panorama Public" project. String projectName = getProjectName(); @@ -225,7 +227,7 @@ private void resubmitFolder(String projectName, String folderName, String submit goToDashboard(); expWebPart = new TargetedMsExperimentWebPart(this); expWebPart.clickResubmit(); - resubmitWithoutPxd(keepPrivate); + resubmitWithoutPxd(false, keepPrivate); goToDashboard(); assertTextPresent("Copy Pending!"); } @@ -280,21 +282,6 @@ private String getReviewerEmail(String panoramaPublicProject, String panoramaPub return null; } - private void resubmitWithoutPxd(boolean keepPrivate) - { - clickAndWait(Locator.linkContainingText("Submit without a ProteomeXchange ID")); - waitForText("Resubmit Request to "); - if (!keepPrivate) - { - uncheck("Keep Private:"); - } - click(Ext4Helper.Locators.ext4Button(("Resubmit"))); - waitForText("Confirm resubmission request to"); - click(Locator.lkButton("OK")); // Confirm to proceed with the submission. - waitForText("Request resubmitted to"); - click(Locator.linkWithText("Back to Experiment Details")); // Navigate to the experiment details page. - } - private void verifyMakePublic(String projectName, String folderName, String user, boolean isSubmitter) { verifyMakePublic(projectName, folderName, user, isSubmitter, false); diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicSymlinkTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicSymlinkTest.java new file mode 100644 index 00000000..b9577594 --- /dev/null +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicSymlinkTest.java @@ -0,0 +1,422 @@ +package org.labkey.test.tests.panoramapublic; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.labkey.api.util.FileUtil; +import org.labkey.test.BaseWebDriverTest; +import org.labkey.test.Locator; +import org.labkey.test.TestFileUtils; +import org.labkey.test.categories.External; +import org.labkey.test.categories.MacCossLabModules; +import org.labkey.test.components.panoramapublic.TargetedMsExperimentWebPart; +import org.labkey.test.components.targetedms.TargetedMSRunsTable; +import org.labkey.test.util.APIContainerHelper; +import org.openqa.selenium.WebElement; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +@Category({External.class, MacCossLabModules.class}) +@BaseWebDriverTest.ClassTimeout(minutes = 10) +public class PanoramaPublicSymlinkTest extends PanoramaPublicBaseTest +{ + + @Override + public String getSampleDataFolder() + { + return SAMPLEDATA_FOLDER; + } + + private void fileCheck(String parent, String file, String target, boolean symlink) throws IOException + { + File f = new File(parent, file); + assertTrue(file + " in " + parent + " is missing", f.exists()); + if (symlink) + { + assertTrue(file + " in " + parent + " is not sym link", + Files.isSymbolicLink(f.toPath())); + assertTrue(file + " in " + parent + " is not pointing to correct target. Expected: " + target + + " Actual: " + Files.readSymbolicLink(f.toPath()).toString(), + Files.readSymbolicLink(f.toPath()).toString().startsWith(target)); + } + else + { + assertFalse(file + " in " + parent + " is sym link", + Files.isSymbolicLink(f.toPath())); + } + } + + private void verifySmallMolFiles(String filesLoc, String filesTarget, boolean symlinks) throws IOException + { + fileCheck(filesLoc, SMALLMOL_PLUS_PEPTIDES_SKY_ZIP, filesTarget, symlinks); + + // directory should never be symlink + fileCheck(filesLoc, SMALLMOL_PLUS_PEPTIDES, filesTarget, false); + + fileCheck(filesLoc + SMALLMOL_PLUS_PEPTIDES, SMALLMOL_PLUS_PEPTIDES_SKY, filesTarget, symlinks); + fileCheck(filesLoc + SMALLMOL_PLUS_PEPTIDES, SMALLMOL_PLUS_PEPTIDES_SKYD, filesTarget, symlinks); + fileCheck(filesLoc + SMALLMOL_PLUS_PEPTIDES, SMALLMOL_PLUS_PEPTIDES_SKY_VIEW, filesTarget, symlinks); + } + + private void verifyQC1Files(String filesLoc, String filesTarget, boolean symlinks) throws IOException + { + fileCheck(filesLoc, QC_1_SKY_ZIP, filesTarget, symlinks); + + // directory should never be symlink + fileCheck(filesLoc, QC_1, filesTarget, false); + + fileCheck(filesLoc + QC_1, QC_1_SKY, filesTarget, symlinks); + fileCheck(filesLoc + QC_1, QC_1_SKY_VIEW, filesTarget, symlinks); + fileCheck(filesLoc + QC_1, QC_1_SKYD, filesTarget, symlinks); + } + + private void verifyCopyFiles(String sourcePath, String targetPath, boolean symlinks) throws IOException + { + String filesLoc = TestFileUtils.getDefaultFileRoot(sourcePath).getPath() + File.separator; + String filesTarget = TestFileUtils.getDefaultFileRoot(targetPath).getPath() + File.separator; + + fileCheck(filesLoc, SKY_FILE_1, filesTarget, symlinks); + + // directory should never be symlink + fileCheck(filesLoc, SKY_FOLDER_NAME, filesTarget, false); + + fileCheck(filesLoc + SKY_FOLDER_NAME, SKY_FOLDER_NAME + ".skyd", filesTarget, symlinks); + fileCheck(filesLoc + SKY_FOLDER_NAME, SKY_FOLDER_NAME + ".sky", filesTarget, symlinks); + fileCheck(filesLoc + SKY_FOLDER_NAME, SKY_FOLDER_NAME + ".sky.view", filesTarget, symlinks); + + // directory should never be symlink + fileCheck(filesLoc, "RawFiles", filesTarget, false); + + fileCheck(filesLoc + "RawFiles", RAW_FILE_WIFF_SCAN, filesTarget, symlinks); + fileCheck(filesLoc + "RawFiles", RAW_FILE_WIFF, filesTarget, symlinks); + + verifyQC1Files(filesLoc, filesTarget, symlinks); + + // directory should never be symlink + fileCheck(filesLoc, SMALL_MOL_FILES, filesTarget, false); + filesLoc = TestFileUtils.getDefaultFileRoot(sourcePath).getPath() + File.separator + SMALL_MOL_FILES + File.separator; + + verifySmallMolFiles(filesLoc, filesTarget, symlinks); + } + + private void verifyCopySubfolderFiles(String sourcePath, String targetPath, boolean symlinks) throws IOException + { + String filesLoc = TestFileUtils.getDefaultFileRoot(sourcePath).getPath() + File.separator; + String filesTarget = TestFileUtils.getDefaultFileRoot(targetPath).getParent() + File.separator; + + fileCheck(filesLoc, SKY_FILE_1, filesTarget, symlinks); + + // directory should never be symlink + fileCheck(filesLoc, SKY_FOLDER_NAME, filesTarget, false); + + fileCheck(filesLoc + SKY_FOLDER_NAME, SKY_FOLDER_NAME + ".skyd", filesTarget, symlinks); + fileCheck(filesLoc + SKY_FOLDER_NAME, SKY_FOLDER_NAME + ".sky", filesTarget, symlinks); + fileCheck(filesLoc + SKY_FOLDER_NAME, SKY_FOLDER_NAME + ".sky.view", filesTarget, symlinks); + + // directory should never be symlink + fileCheck(filesLoc, "RawFiles", filesTarget, false); + + fileCheck(filesLoc + "RawFiles", RAW_FILE_WIFF_SCAN, filesTarget, symlinks); + fileCheck(filesLoc + "RawFiles", RAW_FILE_WIFF_RENAMED, filesTarget, symlinks); + + verifyQC1Files(filesLoc, filesTarget, symlinks); + + filesLoc = TestFileUtils.getDefaultFileRoot(sourcePath + File.separator + "ExperimentalData").getPath() + File.separator; + verifySmallMolFiles(filesLoc, filesTarget, symlinks); + + filesLoc = TestFileUtils.getDefaultFileRoot(sourcePath + File.separator + "SystemSuitability").getPath() + File.separator; + verifyQC1Files(filesLoc, filesTarget, symlinks); + } + + @Test + public void testSymLinks() throws IOException + { + testCopyOrSymlink(true); + } + + @Test + public void testFileCopy() throws IOException + { + testCopyOrSymlink(false); + } + + private void testCopyOrSymlink(boolean useSymlinks) throws IOException + { + String projectName = getProjectName(); + String sourceFolder = "Folder 3" + (useSymlinks ? " Symlinks" : " Copy"); + + String targetFolder = "Test Copy 3" + (useSymlinks ? " Symlinks" : " Copy"); + String experimentTitle = "This is a test for Panorama Public file " + (useSymlinks ? "symlinks" : "copy") + " functionality"; + + setupSourceFolder(projectName, sourceFolder, SUBMITTER); + impersonate(SUBMITTER); + updateSubmitterAccountInfo("One"); + + // Import Skyline documents to the folder + importData(SKY_FILE_1, 1); + importData(QC_1_FILE, 2, false); + importDataInSubfolder(getSampleDataFolder() + SKY_FILE_SMALLMOL_PEP, "SmallMoleculeFiles", 3); + + // Upload some raw files + portalHelper.click(Locator.folderTab("Raw Data")); + _fileBrowserHelper.uploadFile(getSampleDataPath(RAW_FILE_WIFF)); + _fileBrowserHelper.uploadFile(getSampleDataPath(RAW_FILE_WIFF_SCAN)); + + // Add the "Targeted MS Experiment" webpart + TargetedMsExperimentWebPart expWebPart = createExperimentCompleteMetadata(experimentTitle); + expWebPart.clickSubmit(); + String shortAccessUrl = submitWithoutPXId(); + + // Copy the experiment to the Panorama Public project + copyExperimentAndVerify(projectName, sourceFolder, null, experimentTitle, targetFolder, shortAccessUrl); + + // Verify files symlinked or copied correctly + String targetPath = PANORAMA_PUBLIC + File.separator + targetFolder + File.separator; + verifyCopyFiles(projectName + File.separator + sourceFolder + File.separator, targetPath, true); + verifyCopyFiles(targetPath, targetPath, false); + + + // Prepare to resubmit + goToProjectFolder(projectName, sourceFolder); + impersonate(SUBMITTER); + + // Reorganize data in subfolders. + String subfolder1 = "SystemSuitability"; + String subfolder2 = "ExperimentalData"; + setupSubfolder(projectName, sourceFolder, subfolder1, FolderType.QC); + setupSubfolder(projectName, sourceFolder, subfolder2, FolderType.Experiment); + + // Add QC document to the "SystemSuitability" subfolder + goToProjectFolder(projectName, sourceFolder + "/" + subfolder1); + importData(QC_1_FILE, 1, false); + + // Add the small molecule document to "ExperimentalData" subfolder, and remove it from the parent folder + goToProjectFolder(projectName, sourceFolder + "/" + subfolder2); + importData(SKY_FILE_SMALLMOL_PEP, 1); + goToProjectFolder(projectName, sourceFolder); + TargetedMSRunsTable runsTable = new TargetedMSRunsTable(this); + runsTable.deleteRun(SKY_FILE_SMALLMOL_PEP); // delete the run + goToModule("FileContent"); + _fileBrowserHelper.deleteFile("SmallMoleculeFiles/" + SKY_FILE_SMALLMOL_PEP); // delete the .sky.zip + _fileBrowserHelper.deleteFile("SmallMoleculeFiles/" + FileUtil.getBaseName(SKY_FILE_SMALLMOL_PEP, 2)); // delete the exploded folder + + // Rename one of the raw files + portalHelper.click(Locator.folderTab("Raw Data")); + _fileBrowserHelper.renameFile(RAW_FILE_WIFF, RAW_FILE_WIFF + ".RENAMED"); + + + // Include subfolders in the experiment + goToDashboard(); + expWebPart = new TargetedMsExperimentWebPart(this); + expWebPart.clickMoreDetails(); + clickButton("Include Subfolders"); + + // Resubmit + goToDashboard(); + expWebPart.clickResubmit(); + resubmitWithoutPxd(false, true); + goToDashboard(); + assertTextPresent("Copy Pending!"); + + // Copy, and keep the previous copy + copyExperimentAndVerify(projectName, sourceFolder, List.of(subfolder1, subfolder2), experimentTitle, + 2, // We are not deleting the first copy so this is version 2 + true, + false, // Do not delete old copy + targetFolder, + shortAccessUrl, + useSymlinks); + + verifyCopySubfolderFiles(projectName + File.separator + sourceFolder + File.separator, targetPath, useSymlinks); + verifyCopySubfolderFiles(targetPath, targetPath, false); + + // Verify files that were deleted from the submitted folder should not be moved from the old copy to the new one + String filesTarget = TestFileUtils.getDefaultFileRoot(targetPath).getPath() + File.separator; + String filesTargetParent = filesTarget + "SmallMoleculeFiles" + File.separator; + filesTarget = filesTargetParent + SKY_FILE_SMALLMOL_PEP; + File f = new File(filesTarget); + assertFalse(filesTarget + " should not exist in " + filesTargetParent + ".", f.exists()); + + filesTarget = filesTargetParent + FileUtil.getBaseName(SKY_FILE_SMALLMOL_PEP, 2); + f = new File(filesTarget); + assertFalse(filesTarget + " should not exist in " + filesTargetParent + ".", f.exists()); + } + + @Test + public void testDeletePanoramaPublicFolder() + { + String projectName = getProjectName(); + String sourceFolder = "Folder 4"; + + String targetFolder = "Test Copy 4"; + String experimentTitle = "This is a test for moving symlinked files back after the Panorama Public copy is deleted"; + + setupSourceFolder(projectName, sourceFolder, SUBMITTER); + impersonate(SUBMITTER); + updateSubmitterAccountInfo("Rollback"); + + // Import Skyline documents to the folder + importData(SKY_FILE_1, 1); + // Upload some raw files + portalHelper.click(Locator.folderTab("Raw Data")); + _fileBrowserHelper.uploadFile(getSampleDataPath(RAW_FILE_WIFF)); + + // Add the "Targeted MS Experiment" webpart + TargetedMsExperimentWebPart expWebPart = createExperimentCompleteMetadata(experimentTitle); + expWebPart.clickSubmit(); + String shortAccessUrl = submitWithoutPXId(); + + // Copy the experiment to the Panorama Public project + copyExperimentAndVerify(projectName, sourceFolder, null, experimentTitle, targetFolder, shortAccessUrl); + + // Prepare to resubmit + goToProjectFolder(projectName, sourceFolder); + impersonate(SUBMITTER); + + // Upload another raw file + portalHelper.click(Locator.folderTab("Raw Data")); + _fileBrowserHelper.uploadFile(getSampleDataPath(RAW_FILE_WIFF_SCAN)); + // Rename the raw file uploaded earlier + portalHelper.click(Locator.folderTab("Raw Data")); + _fileBrowserHelper.renameFile(RAW_FILE_WIFF, RAW_FILE_WIFF + ".RENAMED"); + + // Resubmit + goToDashboard(); + expWebPart.clickResubmit(); + resubmitWithoutPxd(false, true); + assertTextPresent("Copy Pending!"); + + // Copy, and keep the previous copy + copyExperimentAndVerify(projectName, sourceFolder, null, experimentTitle, + 2, // We are not deleting the first copy so this is version 2 + true, + false, // Do not delete old copy + targetFolder, + shortAccessUrl); + + // The folder containing the older copy should have a "V.1" suffix added to the name. + String v1Folder = targetFolder + " V.1"; + assertTrue("Expected the container for the previous copy to have been renamed with a suffix 'V.1'", + _containerHelper.doesContainerExist(PANORAMA_PUBLIC + "/" + v1Folder)); + + // Source container path (sourceFolder): getProjectName() + "/" + sourceFolder + // Container path for current copy on Panorama Public (current_copy): PANORAMA_PUBLIC + "/" + targetFolder + // Container path for the previous copy (V.1) on Panorama Public (v1_folder): PANORAMA_PUBLIC + "/" + v1Folder + // State of symlinks should be (TODO: @Sweta) + // In the source folder: + // sourceFolder/@files/Study9S_Site52_v1.sky.zip -> current_copy/@files/Study9S_Site52_v1.sky.zip + // sourceFolder/@files/Study9S_Site52_v1/Study9S_Site52_v1.sky -> current_copy/@files/Study9S_Site52_v1/Study9S_Site52_v1.sky + // sourceFolder/@files/Study9S_Site52_v1/Study9S_Site52_v1.skyd -> current_copy/@files/Study9S_Site52_v1/Study9S_Site52_v1.skyd + // sourceFolder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.RENAMED -> current_copy/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.RENAMED + // sourceFolder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.scan -> current_copy/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.scan + // In V.1 folder: + // v1_folder/@files/Study9S_Site52_v1.sky.zip -> current_copy/@files/Study9S_Site52_v1.sky.zip + // v1_folder/@files/Study9S_Site52_v1/Study9S_Site52_v1.sky -> current_copy/@files/Study9S_Site52_v1/Study9S_Site52_v1.sky + // v1_folder/@files/Study9S_Site52_v1/Study9S_Site52_v1.skyd -> current_copy/@files/Study9S_Site52_v1/Study9S_Site52_v1.skyd + // v1_folder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff -> current_copy/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.RENAMED + // v1_folder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.scan -- FILE SHOULD NOT EXISTS IN V.1 copy + + log("Verifying files in source folder"); + File filesLoc = TestFileUtils.getDefaultFileRoot(projectName + "/" + sourceFolder); + assertTrue("sky.zip should be sym link", Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FILE_1).toPath())); + assertTrue("file(.sky) in the " + SKY_FOLDER_NAME + " is not sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".sky").toPath())); + assertTrue("file(.skyd) in the " + SKY_FOLDER_NAME + " is not sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".skyd").toPath())); + assertTrue("Rename raw file missing", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF + ".RENAMED").exists()); + assertTrue("Additional raw file missing", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF_SCAN).exists()); + + log("Verifying files in copied folder"); + filesLoc = TestFileUtils.getDefaultFileRoot(PANORAMA_PUBLIC + "/" + v1Folder); + assertTrue(v1Folder + " sky.zip should be sym link", Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FILE_1).toPath())); + assertTrue(v1Folder + " file(.sky) in the " + SKY_FOLDER_NAME + " should be a sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".sky").toPath())); + assertTrue(v1Folder + " file(.skyd) in the " + SKY_FOLDER_NAME + " should be a sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".skyd").toPath())); + assertFalse("Rename raw file should not be present", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF + ".RENAMED").exists()); + assertFalse("Additional new raw file should not be present", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF_SCAN).exists()); + + APIContainerHelper apiContainerHelper = new APIContainerHelper(this); + + // Delete the current copy. Symlinked files should be moved to the previous copy + apiContainerHelper.deleteFolder(PANORAMA_PUBLIC, targetFolder); + assertFalse("Expected the container for the current copy to have been deleted", + _containerHelper.doesContainerExist(PANORAMA_PUBLIC + "/" + targetFolder)); + + // State of symlinks should be (TODO: @Sweta) + // In the source folder: + // sourceFolder/@files/Study9S_Site52_v1.sky.zip -> v1_folder/@files/Study9S_Site52_v1.sky.zip + // sourceFolder/@files/Study9S_Site52_v1/Study9S_Site52_v1.sky -> v1_folder/@files/Study9S_Site52_v1/Study9S_Site52_v1.sky + // sourceFolder/@files/Study9S_Site52_v1/Study9S_Site52_v1.skyd -> v1_folder/@files/Study9S_Site52_v1/Study9S_Site52_v1.skyd + // sourceFolder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.RENAMED -> v1_folder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff + // sourceFolder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.scan -- Not a symlink. Did not exist in V.1 copy so should have been moved back to the sourceFolder. + // In V.1 version folder: + // NONE OF THE FILES SHOULD BE SYMLINKS + + log("Verifying files in target folder after delete"); + filesLoc = TestFileUtils.getDefaultFileRoot(PANORAMA_PUBLIC + "/" + v1Folder); + assertFalse("sky.zip should not be sym link", Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FILE_1).toPath())); + assertFalse(v1Folder + " file(.sky) in the " + SKY_FOLDER_NAME + " should be a sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".sky").toPath())); + assertFalse(v1Folder + " file(.skyd) in the " + SKY_FOLDER_NAME + " should be a sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".skyd").toPath())); + assertTrue("Original raw file should be present", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF).exists()); + assertFalse("Rename raw file should not be present", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF + ".RENAMED").exists()); + assertFalse("Additional new raw file should not be present", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF_SCAN).exists()); + + // Delete the older copy as well. Symlinked files should be moved back to the source folder + apiContainerHelper.deleteFolder(PANORAMA_PUBLIC, v1Folder); + assertFalse("Expected the container for the current copy to have been deleted", + _containerHelper.doesContainerExist(PANORAMA_PUBLIC + "/" + v1Folder)); + + // State of symlinks should be (TODO: @Sweta) + // In the source folder: + // ALL FILES SHOULD HAVE BEEN MOVED BACK TO sourceFolder. NONE OF THE FILES SHOULD BE SYMLINKS + // sourceFolder/@files/Study9S_Site52_v1.sky.zip + // sourceFolder/@files/Study9S_Site52_v1/Study9S_Site52_v1.sky + // sourceFolder/@files/Study9S_Site52_v1/Study9S_Site52_v1.skyd + // sourceFolder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.RENAMED + // sourceFolder/@files/RawFiles/Site52_041009_Study9S_Phase-I.wiff.scan + + log("Verifying files in source folder after deleting target folder"); + filesLoc = TestFileUtils.getDefaultFileRoot(projectName + "/" + sourceFolder); + assertFalse("sky.zip should be sym link", Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FILE_1).toPath())); + assertFalse("file(.sky) in the " + SKY_FOLDER_NAME + " is not sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".sky").toPath())); + assertFalse("file(.skyd) in the " + SKY_FOLDER_NAME + " is not sym link", + Files.isSymbolicLink(new File(filesLoc, "/" + SKY_FOLDER_NAME + "/" + SKY_FOLDER_NAME + ".skyd").toPath())); + assertTrue("Rename raw file missing", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF + ".RENAMED").exists()); + assertTrue("Additional raw file missing", new File(filesLoc + "/RawFiles/" + RAW_FILE_WIFF_SCAN).exists()); + } + + private void importDataInSubfolder(String file, String subfolder, int jobCount) + { + Locator.XPathLocator importButtonLoc = Locator.lkButton("Process and Import Data"); + WebElement importButton = importButtonLoc.findElementOrNull(getDriver()); + if (null == importButton) + { + goToModule("Pipeline"); + importButton = importButtonLoc.findElement(getDriver()); + } + clickAndWait(importButton); + String fileName = Paths.get(file).getFileName().toString(); + if (!_fileBrowserHelper.fileIsPresent(subfolder)) + { + _fileBrowserHelper.createFolder(subfolder); + } + _fileBrowserHelper.selectFileBrowserItem("/" + subfolder + "/"); + if (!_fileBrowserHelper.fileIsPresent(fileName)) + { + _fileBrowserHelper.uploadFile(TestFileUtils.getSampleData("TargetedMS/" + file)); + } + _fileBrowserHelper.importFile(fileName, "Import Skyline Results"); + waitForText("Skyline document import"); + waitForPipelineJobsToComplete(jobCount, file, false); + } +} diff --git a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicTest.java b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicTest.java index 701e358b..5fb5308f 100644 --- a/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicTest.java +++ b/panoramapublic/test/src/org/labkey/test/tests/panoramapublic/PanoramaPublicTest.java @@ -14,7 +14,6 @@ import org.labkey.test.util.APIContainerHelper; import org.labkey.test.util.ApiPermissionsHelper; import org.labkey.test.util.DataRegionTable; -import org.labkey.test.util.Ext4Helper; import org.labkey.test.util.PermissionsHelper; import org.labkey.test.util.PipelineStatusTable; import org.labkey.test.util.PortalHelper; @@ -31,8 +30,6 @@ @BaseWebDriverTest.ClassTimeout(minutes = 7) public class PanoramaPublicTest extends PanoramaPublicBaseTest { - public static final String SAMPLEDATA_FOLDER = "panoramapublic/"; - private static final String SKY_FILE_1 = "Study9S_Site52_v1.sky.zip"; private static final String RAW_FILE_WIFF = "Site52_041009_Study9S_Phase-I.wiff"; private static final String RAW_FILE_WIFF_SCAN = RAW_FILE_WIFF + ".scan"; @@ -96,7 +93,7 @@ public void testExperimentCopy() impersonate(SUBMITTER); goToDashboard(); expWebPart.clickResubmit(); - resubmitWithoutPxd(); + resubmitWithoutPxd(true, true); goToDashboard(); assertTextPresent("Copy Pending!"); @@ -134,7 +131,7 @@ public void testExperimentCopy() verifySubmissionsAndPublishedVersions(projectName, folderName, 2, 2, List.of(Boolean.TRUE, Boolean.TRUE), List.of("", experimentTitle), List.of("", "2"), List.of("", shortAccessLink)); // Submitter should be able to delete their folder after it has been copied to Panorama Public - goToProjectFolder(projectName, folderName); + goToHome(); impersonate(SUBMITTER); apiContainerHelper.deleteFolder(projectName, folderName); assertFalse("Expected the submitter's container to have been deleted", @@ -287,6 +284,8 @@ public void testCopyExperimentWithSubfolder() assertTrue("Copy Job's log file not set to pipeline root", pipelineStatusDetailsPage.getFilePath().contains("@files")); //Proxy check to see if Job's log file is set to the pipeline root. } + + @Override protected void setupSourceFolder(String projectName, String folderName, String ... adminUsers) { @@ -374,17 +373,6 @@ private String testSubmitWithSubfolders(TargetedMsExperimentWebPart expWebPart) return submitWithoutPXId(); } - private void resubmitWithoutPxd() - { - clickButton("Submit without a ProteomeXchange ID"); - waitForText("Resubmit Request to "); - click(Ext4Helper.Locators.ext4Button(("Resubmit"))); - waitForText("Confirm resubmission request to"); - click(Locator.lkButton("OK")); // Confirm to proceed with the submission. - waitForText("Request resubmitted to"); - click(Locator.linkWithText("Back to Experiment Details")); // Navigate to the experiment details page. - } - private void verifyVersionCount(String experimentTitle, int count) { goToProjectHome(PANORAMA_PUBLIC);