Add icav2 copy manager stack

ICAv2 sync fix

Author: alexiswl

config/config.ts

@@ -76,6 +76,10 @@ import {
   getFastqSyncManagerStackProps,
   getFastqSyncManagerTableStackProps,
 } from './stacks/fastqSyncManager';
+import {
+  getIcav2DataCopyManagerStackProps,
+  getIcav2DataCopyManagerTableStackProps,
+} from './stacks/icav2DataCopyManager';
 
 interface EnvironmentConfig {
   name: string;
@@ -120,6 +124,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       fastqManagerTableStackProps: getFastqManagerTableStackProps(stage),
       fastqUnarchivingManagerTableStackProps: getFastqUnarchivingManagerTableStackProps(),
       fastqSyncManagerTableStackProps: getFastqSyncManagerTableStackProps(),
+      icav2DataCopyTableStackProps: getIcav2DataCopyManagerTableStackProps(),
     },
     statelessConfig: {
       metadataManagerStackProps: getMetadataManagerStackProps(stage),
@@ -153,6 +158,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       fastqManagerStackProps: getFastqManagerStackProps(stage),
       fastqUnarchivingManagerStackProps: getFastqUnarchivingManagerStackProps(stage),
       fastqSyncManagerStackProps: getFastqSyncManagerStackProps(stage),
+      icav2DataCopyManagerStackProps: getIcav2DataCopyManagerStackProps(stage),
     },
   };
 

config/constants.ts

@@ -971,3 +971,10 @@ export const fastqUnarchivingManagerEventSource = 'orcabus.fastqunarchivingmanag
 Fastq sync service
 */
 export const fastqSyncEventDetailType = 'fastqSync';
+
+/*
+ICAv2 ProjectData Copy Manager Stack
+*/
+export const icav2DataCopyManagerDynamodbTableName = 'icav2DataCopyManagerDynamoDBTable';
+export const icav2DataCopyEventSource = 'orcabus.icav2datacopymanager';
+export const icav2DataCopySyncDetailType = 'ICAv2DataCopySync';

config/stacks/icav2DataCopyManager.ts

@@ -0,0 +1,53 @@
+import {
+  AppStage,
+  eventBusName,
+  icaEventPipeStackName,
+  icav2AccessTokenSecretName,
+  icav2DataCopyEventSource,
+  icav2DataCopyManagerDynamodbTableName,
+  icav2DataCopySyncDetailType,
+} from '../constants';
+import { Icav2DataCopyManagerTableConfig } from '../../lib/workload/stateful/stacks/icav2-data-copy-manager-dynamo-db/deploy';
+import { Icav2DataCopyManagerConfig } from '../../lib/workload/stateless/stacks/icav2-data-copy-manager/deploy/interfaces';
+
+/*
+Internal constants
+*/
+const icav2DataCopyInternalDetailType = 'ICAv2DataCopyInternalSync';
+
+// Stateful
+export const getIcav2DataCopyManagerTableStackProps = (): Icav2DataCopyManagerTableConfig => {
+  return {
+    dynamodbTableName: icav2DataCopyManagerDynamodbTableName,
+  };
+};
+
+// Stateless
+export const getIcav2DataCopyManagerStackProps = (stage: AppStage): Icav2DataCopyManagerConfig => {
+  return {
+    /*
+    Tables
+    */
+    dynamodbTableName: icav2DataCopyManagerDynamodbTableName,
+
+    /*
+    Event handling
+    */
+    eventBusName: eventBusName,
+    icaEventPipeName: icaEventPipeStackName,
+    eventSource: icav2DataCopyEventSource,
+    eventExternalDetailType: icav2DataCopySyncDetailType,
+    eventInternalDetailType: icav2DataCopyInternalDetailType,
+
+    /*
+    Names for things
+    */
+    stateMachinePrefix: 'icav2-data-copy',
+    ruleNamePrefix: 'icav2-data-copy',
+
+    /*
+    Secrets
+    */
+    icav2AccessTokenSecretId: icav2AccessTokenSecretName[stage], // "/icav2/umccr-prod/service-production-jwt-token-secret-arn"
+  };
+};

lib/workload/stateful/stacks/icav2-data-copy-manager-dynamo-db/deploy/index.ts

@@ -0,0 +1,22 @@
+import * as cdk from 'aws-cdk-lib';
+import { Construct } from 'constructs';
+import { DynamodbPartitionedPipelineConstruct } from '../../../../components/dynamodb-partitioned-table';
+
+export interface Icav2DataCopyManagerTableConfig {
+  dynamodbTableName: string;
+}
+
+export type Icav2DataCopyManagerTableStackProps = Icav2DataCopyManagerTableConfig & cdk.StackProps;
+
+export class Icav2DataCopyManagerTable extends cdk.Stack {
+  constructor(scope: Construct, id: string, props: Icav2DataCopyManagerTableStackProps) {
+    super(scope, id, props);
+
+    /*
+    Initialise dynamodb table with id and sort keys
+    */
+    new DynamodbPartitionedPipelineConstruct(this, props.dynamodbTableName, {
+      tableName: props.dynamodbTableName,
+    });
+  }
+}

lib/workload/stateful/statefulStackCollectionClass.ts

@@ -78,6 +78,10 @@ import {
   FastqSyncManagerTable,
   FastqSyncManagerTableStackProps,
 } from './stacks/fastq-sync-dynamodb/deploy/stack';
+import {
+  Icav2DataCopyManagerTable,
+  Icav2DataCopyManagerTableStackProps,
+} from './stacks/icav2-data-copy-manager-dynamo-db/deploy';
 
 export interface StatefulStackCollectionProps {
   dataBucketStackProps: DataBucketStackProps;
@@ -103,6 +107,7 @@ export interface StatefulStackCollectionProps {
   fastqManagerTableStackProps: FastqManagerTableStackProps;
   fastqUnarchivingManagerTableStackProps: FastqUnarchivingManagerTableStackProps;
   fastqSyncManagerTableStackProps: FastqSyncManagerTableStackProps;
+  icav2DataCopyTableStackProps: Icav2DataCopyManagerTableStackProps;
 }
 
 export class StatefulStackCollection {
@@ -131,6 +136,7 @@ export class StatefulStackCollection {
   readonly fastqManagerTableStack: Stack;
   readonly fastqUnarchivingManagerTableStack: Stack;
   readonly fastqSyncManagerTableStack: Stack;
+  readonly icav2DataCopyTableStack: Stack;
 
   constructor(
     scope: Construct,
@@ -307,6 +313,14 @@ export class StatefulStackCollection {
         ...statefulConfiguration.fastqSyncManagerTableStackProps,
       }
     );
+    this.icav2DataCopyTableStack = new Icav2DataCopyManagerTable(
+      scope,
+      'Icav2DataCopyManagerTableStack',
+      {
+        ...this.createTemplateProps(env, 'Icav2DataCopyManagerTableStack'),
+        ...statefulConfiguration.icav2DataCopyTableStackProps,
+      }
+    );
   }
 
   /**

lib/workload/stateless/stacks/icav2-data-copy-manager/Readme.md

@@ -0,0 +1,41 @@
+# ICAv2 Data Copy Manager
+
+Service for copying data from one location to another.  
+
+Supports a TaskToken object that can be used to allow processes to hang until the copy is complete for other services
+that may call this service.  
+
+## Usage
+
+Given a list of source uris, and a destination uri, the service will copy the data from the source to the destination.
+
+The destination uri does not need to exist prior to the copy, but the source uris must exist.
+
+If the source uri is a folder, it will be a subfolder in the destination.
+
+If a file size is under 8 Mb, we will use the `requests` library to download the file and upload it to the destination,
+since single-part uploads often fail when the source object is tagged.  
+
+AWS S3 URIs in the sourceUriList and destinationUri are also supported.
+
+The Task Token is optional and can be used to allow the calling service to wait for the copy to complete.
+
+## Event Example
+
+```json
+{
+  "EventBusName": "OrcaBusMain",
+  "Source": "Whatever",
+  "DetailType": "ICAv2DataCopySync",
+  "Detail": {
+    "payload": {
+      "sourceUriList": [
+        "icav2://project-id-or-name/path-to-data.txt",
+        "icav2://project-id-or-name/path-to-folder/"
+      ],
+      "destinationUri": "icav2://project-id-or-name/path-to-destination/"
+    },
+    "taskToken": "your-task-token"
+  }
+}
+```