Merge discvr-23.11 to discvr-24.3

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractCommandWrapper.java

@@ -289,7 +289,7 @@ protected boolean isThrowNonZeroExits()
         return _throwNonZeroExits;
     }
 
-    protected static File resolveFileInPath(String exe, @Nullable String packageName, boolean throwIfNotFound)
+    public static File resolveFileInPath(String exe, @Nullable String packageName, boolean throwIfNotFound)
     {
         File fn;
         String path;

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/run/AbstractGatk4Wrapper.java

@@ -90,6 +90,7 @@ public List<String> getBaseArgs(@Nullable String toolName)
         List<String> args = new ArrayList<>();
         args.add(SequencePipelineService.get().getJavaFilepath());
         args.addAll(SequencePipelineService.get().getJavaOpts(_maxRamOverride));
+        args.add("-DGATK_STACKTRACE_ON_USER_EXCEPTION=true");
         args.add("-jar");
         args.add(getJAR().getPath());
 
@@ -98,6 +99,8 @@ public List<String> getBaseArgs(@Nullable String toolName)
             args.add(toolName);
         }
 
+
+
         return args;
     }
 

SequenceAnalysis/pipeline_code/sequence_tools_install.sh

@@ -486,10 +486,10 @@ then
     rm -Rf bcftools*
     rm -Rf $LKTOOLS_DIR/bcftools
 
-    wget $WGET_OPTS https://github.com/samtools/bcftools/releases/download/1.18/bcftools-1.18.tar.bz2
-    tar xjvf bcftools-1.18.tar.bz2
-    chmod 755 bcftools-1.18
-    cd bcftools-1.18
+    wget $WGET_OPTS https://github.com/samtools/bcftools/releases/download/1.20/bcftools-1.20.tar.bz2
+    tar xjvf bcftools-1.20.tar.bz2
+    chmod 755 bcftools-1.20
+    cd bcftools-1.20
     rm -f plugins/liftover.c
     wget $WGET_OPTS -P plugins https://raw.githubusercontent.com/freeseek/score/master/liftover.c
 

SequenceAnalysis/resources/queries/sequenceanalysis/chain_files/.qview.xml

@@ -0,0 +1,16 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView">
+   <columns>
+       <!--<column name="rowid" />-->
+       <column name="genomeId1" />
+       <column name="genomeId2" />
+       <column name="source" />
+       <column name="version" />
+       <column name="datedisabled" />
+       <column name="chainFile" />
+   </columns>
+   <sorts>
+       <sort column="genomeId1/name"/>
+       <sort column="genomeId2/name"/>
+       <sort column="version"/>
+   </sorts>
+</customView>

SequenceAnalysis/resources/queries/sequenceanalysis/chain_files/With Filepath.qview.xml

@@ -0,0 +1,17 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView">
+   <columns>
+       <!--<column name="rowid" />-->
+       <column name="genomeId1" />
+       <column name="genomeId2" />
+       <column name="source" />
+       <column name="version" />
+       <column name="dateDisabled" />
+       <column name="chainFile" />
+       <column name="chainFile/DataFileUrl" />
+   </columns>
+   <sorts>
+       <sort column="genomeId1/name"/>
+       <sort column="genomeId2/name"/>
+       <sort column="version"/>
+   </sorts>
+</customView>

SequenceAnalysis/resources/web/SequenceAnalysis/window/LiftoverWindow.js

@@ -117,6 +117,11 @@ Ext4.define('SequenceAnalysis.window.LiftoverWindow', {
                 itemId: 'useBcfTools',
                 checked: false,
                 fieldLabel: 'Use bcftools'
+            },{
+                xtype: 'checkbox',
+                itemId: 'doNotRetainUnmapped',
+                checked: false,
+                fieldLabel: 'Do Not Retain Unmapped'
             }].concat(SequenceAnalysis.window.OutputHandlerWindow.getCfgForToolParameters(this.toolParameters)),
             buttons: [{
                 text: 'Submit',
@@ -157,6 +162,14 @@ Ext4.define('SequenceAnalysis.window.LiftoverWindow', {
             params.dropGenotypes = this.down('#dropGenotypes').getValue();
         }
 
+        if (this.down('#useBcfTools').getValue()){
+            params.useBcfTools = this.down('#useBcfTools').getValue();
+        }
+
+        if (this.down('#doNotRetainUnmapped').getValue()){
+            params.doNotRetainUnmapped = this.down('#doNotRetainUnmapped').getValue();
+        }
+
         Ext4.Msg.wait('Saving...');
         LABKEY.Ajax.request({
             url: LABKEY.ActionURL.buildURL('sequenceanalysis', 'runSequenceHandler', this.containerPath),

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -59,6 +59,7 @@
 import org.labkey.sequenceanalysis.analysis.RnaSeqcHandler;
 import org.labkey.sequenceanalysis.analysis.SbtGeneCountHandler;
 import org.labkey.sequenceanalysis.analysis.UnmappedSequenceBasedGenotypeHandler;
+import org.labkey.sequenceanalysis.analysis.UpdateReadsetFilesHandler;
 import org.labkey.sequenceanalysis.button.AddSraRunButton;
 import org.labkey.sequenceanalysis.button.ArchiveReadsetsButton;
 import org.labkey.sequenceanalysis.button.ChangeReadsetStatusButton;
@@ -338,6 +339,7 @@ public static void registerPipelineSteps()
         SequenceAnalysisService.get().registerFileHandler(new DeepVariantHandler());
         SequenceAnalysisService.get().registerFileHandler(new GLNexusHandler());
         SequenceAnalysisService.get().registerFileHandler(new ParagraphStep());
+        SequenceAnalysisService.get().registerFileHandler(new UpdateReadsetFilesHandler());
 
         SequenceAnalysisService.get().registerReadsetHandler(new MultiQCHandler());
         SequenceAnalysisService.get().registerReadsetHandler(new RestoreSraDataHandler());

SequenceAnalysis/src/org/labkey/sequenceanalysis/analysis/LiftoverHandler.java

@@ -28,6 +28,7 @@
 import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceAnalysisJobSupport;
 import org.labkey.api.sequenceanalysis.pipeline.SequenceOutputHandler;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
 import org.labkey.api.sequenceanalysis.pipeline.VariantProcessingStep;
 import org.labkey.api.sequenceanalysis.run.SelectVariantsWrapper;
 import org.labkey.api.util.FileType;
@@ -176,6 +177,7 @@ public void processFilesRemote(List<SequenceOutputFile> inputFiles, JobContext c
 
             boolean dropGenotypes = params.optBoolean("dropGenotypes", false);
             boolean useBcfTools = params.optBoolean("useBcfTools", false);
+            boolean doNotRetainUnmapped = params.optBoolean("doNotRetainUnmapped", false);
 
             int chainFileId = params.getInt("chainFileId");
             File chainFile = ctx.getSequenceSupport().getCachedData(chainFileId);
@@ -214,7 +216,7 @@ else if (_vcfFileType.isType(f.getFile()))
                 }
 
                 File lifted = new File(outDir, baseName + ".lifted-" + targetGenomeId + ext);
-                File unmappedOutput = new File(outDir, baseName + ".unmapped-" + targetGenomeId + ext);
+                File unmappedOutput = doNotRetainUnmapped ? null : new File(outDir, baseName + ".unmapped-" + targetGenomeId + ext);
 
                 try
                 {
@@ -260,7 +262,11 @@ else if (_vcfFileType.isType(f.getFile()))
                     ctx.addSequenceOutput(so1);
                 }
 
-                if (!unmappedOutput.exists())
+                if (unmappedOutput == null)
+                {
+                    // skip
+                }
+                else if (!unmappedOutput.exists())
                 {
                     job.getLogger().info("no unmapped intervals");
                 }
@@ -328,6 +334,8 @@ public void liftOverVcf(JobContext ctx, ReferenceGenome targetGenome, ReferenceG
         {
             LiftoverBcfToolsWrapper wrapper = new LiftoverBcfToolsWrapper(job.getLogger());
             wrapper.doLiftover(currentVCF, chain, sourceGenome.getWorkingFastaFile(), targetGenome.getWorkingFastaFile(), unmappedOutput, output);
+
+            SequencePipelineService.get().sortVcf(output, null, targetGenome.getSequenceDictionary(), ctx.getLogger());
         }
         else
         {