multicam input pipelines (#1346)

* multicam input pipelines * only use multicam pipeline inputs when the dataset is multicam * add TODO reminders for refactoring cycle dependencies * add default single cam args to multicam pipelines
Kitware · Aug 16, 2023 · 4ae53d9 · 4ae53d9
1 parent 18b861c
commit 4ae53d9
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 17 deletions.
diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts
@@ -26,6 +26,8 @@ import * as viameSerializers from 'platform/desktop/backend/serializers/viame';
 import * as nistSerializers from 'platform/desktop/backend/serializers/nist';
 import * as dive from 'platform/desktop/backend/serializers/dive';
 import kpf from 'platform/desktop/backend/serializers/kpf';
+// TODO:  Check to Refactor this
+// eslint-disable-next-line import/no-cycle
 import { checkMedia, convertMedia } from 'platform/desktop/backend/native/mediaJobs';
 import {
   websafeImageTypes, websafeVideoTypes, otherImageTypes, otherVideoTypes, MultiType, JsonMetaRegEx,
@@ -41,6 +43,8 @@ import {
 
 import processTrackAttributes from './attributeProcessor';
 import { upgrade } from './migrations';
+// TODO:  Check to Refactor this
+// eslint-disable-next-line import/no-cycle
 import { getMultiCamUrls, transcodeMultiCam } from './multiCamUtils';
 import { splitExt } from './utils';
 

diff --git a/client/platform/desktop/backend/native/mediaJobs.ts b/client/platform/desktop/backend/native/mediaJobs.ts
@@ -14,6 +14,8 @@ import { observeChild } from 'platform/desktop/backend/native/processManager';
 import {
   jobFileEchoMiddleware, spawnResult, createWorkingDirectory, getBinaryPath,
 } from './utils';
+// TODO:  Check to Refactor this
+// eslint-disable-next-line import/no-cycle
 import {
   getTranscodedMultiCamType,
 } from './multiCamUtils';

diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts
@@ -1,11 +1,15 @@
+/* eslint-disable no-await-in-loop */
 import npath from 'path';
 import fs from 'fs-extra';
 import {
   FrameImage,
   MultiCamMedia,
 } from 'dive-common/apispec';
 
-import { JsonMeta } from 'platform/desktop/constants';
+import { JsonMeta, Settings } from 'platform/desktop/constants';
+// eslint-disable-next-line import/no-cycle
+import { loadAnnotationFile, loadJsonMetadata, getValidatedProjectDir } from 'platform/desktop/backend/native/common';
+import { serialize } from 'platform/desktop/backend/serializers/viame';
 
 /**
  * Figure out the destination location
@@ -71,12 +75,15 @@ function getTranscodedMultiCamType(imageListFile: string, jsonMeta: JsonMeta) {
   throw new Error(`No associate type for ${imageListFile} in multiCam data`);
 }
 
-function writeMultiCamStereoPipelineArgs(jobWorkDir: string, meta: JsonMeta) {
+async function writeMultiCamStereoPipelineArgs(
+  jobWorkDir: string, meta: JsonMeta, settings: Settings, utility = false,
+) {
   const argFilePair: Record<string, string> = {};
   const outFiles: Record<string, string> = {};
   if (meta.multiCam && meta.multiCam.cameras) {
-    let i = 0;
-    Object.entries(meta.multiCam.cameras).forEach(([key, list]) => {
+    const cameraList = Object.entries(meta.multiCam.cameras);
+    for (let i = 0; i < cameraList.length; i += 1) {
+      const [key, list] = cameraList[i];
       const { originalBasePath } = list;
       const outputFileName = `computed_tracks_${key}.csv`;
       const outputArg = `detector_writer${i + 1}:file_name`;
@@ -85,22 +92,50 @@ function writeMultiCamStereoPipelineArgs(jobWorkDir: string, meta: JsonMeta) {
       argFilePair[outputArgWriteTracks] = outputFileName;
       outFiles[key] = outputFileName;
       const inputArg = `input${i + 1}:video_filename`;
+      if (i === 0) {
+        argFilePair['detector_writer:file_name'] = outputFileName;
+        argFilePair['track_writer:file_name'] = outputFileName;
+      }
       if (list.type === 'image-sequence') {
         const inputFileName = npath.join(jobWorkDir, `input${i + 1}_images.txt`);
         const inputFile = fs.createWriteStream(inputFileName);
         list.originalImageFiles.forEach((image) => inputFile.write(`${npath.join(originalBasePath, image)}\n`));
         inputFile.end();
         argFilePair[inputArg] = inputFileName;
+        if (i === 0) {
+          argFilePair['input:video_filename'] = inputFileName;
+        }
       } else if (list.originalVideoFile) {
         const vidFile = list.transcodedVideoFile
           ? list.transcodedVideoFile : list.originalVideoFile;
         const vidTypeArg = `input${i + 1}:video_reader:type`;
         const vidType = 'vidl_ffmpeg';
         argFilePair[vidTypeArg] = vidType;
-        argFilePair[inputArg] = npath.join(originalBasePath, vidFile);
+        const videoFileName = npath.join(originalBasePath, vidFile);
+        argFilePair[inputArg] = videoFileName;
+        if (i === 0) {
+          argFilePair['input:video_filename'] = videoFileName;
+        }
       }
-      i += 1;
-    });
+      if (utility) {
+        const inputArgDetection = `detection_reader${i + 1}:file_name`;
+        const inputArgTrack = `track_reader${i + 1}:file_name`;
+        const groundTruthFileName = npath.join(jobWorkDir, `detections${i + 1}.csv`);
+        // We need to download these files with the proper names
+        const projectDirInfo = await getValidatedProjectDir(settings, `${meta.id}/${key}`);
+        const groundTruthFileStream = fs.createWriteStream(groundTruthFileName);
+        argFilePair[inputArgTrack] = groundTruthFileName;
+        argFilePair[inputArgDetection] = groundTruthFileName;
+        if (i === 0) {
+          argFilePair['detection_reader:file_name'] = groundTruthFileName;
+          argFilePair['track_reader:file_name'] = groundTruthFileName;
+        }
+        const subMeta = await loadJsonMetadata(projectDirInfo.metaFileAbsPath);
+        const inputData = await loadAnnotationFile(projectDirInfo.trackFileAbsPath);
+        await serialize(groundTruthFileStream, inputData, subMeta);
+        groundTruthFileStream.end();
+      }
+    }
   }
   return { argFilePair, outFiles };
 }

diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts
@@ -82,6 +82,9 @@ async function runPipeline(
   }
 
   let command: string[] = [];
+  const stereoOrMultiCam = (pipeline.type === stereoPipelineMarker
+    || multiCamPipelineMarkers.includes(pipeline.type));
+
   if (metaType === 'video') {
     let videoAbsPath = npath.join(meta.originalBasePath, meta.originalVideoFile);
     if (meta.type === MultiType) {
@@ -94,11 +97,13 @@ async function runPipeline(
       `"${viameConstants.kwiverExe}" runner`,
       '-s "input:video_reader:type=vidl_ffmpeg"',
       `-p "${pipelinePath}"`,
-      `-s input:video_filename="${videoAbsPath}"`,
       `-s downsampler:target_frame_rate=${meta.fps}`,
-      `-s detector_writer:file_name="${detectorOutput}"`,
-      `-s track_writer:file_name="${trackOutput}"`,
     ];
+    if (!stereoOrMultiCam) {
+      command.push(`-s input:video_filename="${videoAbsPath}"`);
+      command.push(`-s detector_writer:file_name="${detectorOutput}"`);
+      command.push(`-s track_writer:file_name="${trackOutput}"`);
+    }
   } else if (metaType === 'image-sequence') {
     // Create frame image manifest
     const manifestFile = npath.join(jobWorkDir, 'image-manifest.txt');
@@ -115,21 +120,22 @@ async function runPipeline(
       `${viameConstants.setupScriptAbs} &&`,
       `"${viameConstants.kwiverExe}" runner`,
       `-p "${pipelinePath}"`,
-      `-s input:video_filename="${manifestFile}"`,
-      `-s detector_writer:file_name="${detectorOutput}"`,
-      `-s track_writer:file_name="${trackOutput}"`,
     ];
+    if (!stereoOrMultiCam) {
+      command.push(`-s input:video_filename="${manifestFile}"`);
+      command.push(`-s detector_writer:file_name="${detectorOutput}"`);
+      command.push(`-s track_writer:file_name="${trackOutput}"`);
+    }
   }
-  if (requiresInput) {
+  if (requiresInput && !stereoOrMultiCam) {
     command.push(`-s detection_reader:file_name="${groundTruthFileName}"`);
     command.push(`-s track_reader:file_name="${groundTruthFileName}"`);
   }
 
   let multiOutFiles: Record<string, string>;
-  const stereoOrMultiCam = (pipeline.type === stereoPipelineMarker
-    || multiCamPipelineMarkers.includes(pipeline.type));
   if (meta.multiCam && stereoOrMultiCam) {
-    const { argFilePair, outFiles } = writeMultiCamStereoPipelineArgs(jobWorkDir, meta);
+    // eslint-disable-next-line max-len
+    const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput);
     Object.entries(argFilePair).forEach(([arg, file]) => {
       command.push(`-s ${arg}="${file}"`);
     });