Skip to content

Commit a570966

Browse files
alexiswlalexiswl
committed
Add fastq manager
1 parent 5242426 commit a570966

File tree

72 files changed

+10029
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+10029
-0
lines changed

‎config/config.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ import { getDataMigrateStackProps } from './stacks/dataMigrate';
6767
import { getHtsgetProps } from './stacks/htsget';
6868
import { getSampleSheetCheckerProps } from './stacks/sampleSheetChecker';
6969
import { getAccessKeySecretStackProps } from './stacks/accessKeySecret';
70+
import { getFastqManagerStackProps, getFastqManagerTableStackProps } from './stacks/fastqManager';
7071

7172
interface EnvironmentConfig {
7273
name: string;
@@ -108,6 +109,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
108109
oncoanalyserPipelineTableStackProps: getOncoanalyserPipelineTableStackProps(),
109110
sashPipelineTableStackProps: getSashPipelineTableStackProps(),
110111
accessKeySecretStackProps: getAccessKeySecretStackProps(stage),
112+
fastqManagerTableStackProps: getFastqManagerTableStackProps(stage),
111113
},
112114
statelessConfig: {
113115
metadataManagerStackProps: getMetadataManagerStackProps(stage),
@@ -138,6 +140,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
138140
htsgetProps: getHtsgetProps(stage),
139141
sampleSheetCheckerProps: getSampleSheetCheckerProps(stage),
140142
pgDDProps: getPgDDProps(stage),
143+
fastqManagerStackProps: getFastqManagerStackProps(stage),
141144
},
142145
};
143146

‎config/constants.ts

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,11 @@ export const icav2PipelineCacheBucket: Record<AppStage, string> = {
8888
[AppStage.GAMMA]: 'pipeline-stg-cache-503977275616-ap-southeast-2',
8989
[AppStage.PROD]: 'pipeline-prod-cache-503977275616-ap-southeast-2',
9090
};
91+
export const icav2PipelineCachePrefix: Record<AppStage, string> = {
92+
[AppStage.BETA]: 'byob-icav2/development/',
93+
[AppStage.GAMMA]: 'byob-icav2/staging/',
94+
[AppStage.PROD]: 'byob-icav2/production/',
95+
};
9196

9297
// The test inventory bucket for dev.
9398
export const fileManagerInventoryBucket: Record<AppStage.BETA, string> = {
@@ -892,3 +897,42 @@ export const oraDecompressionIcav2ReadyEventSource = 'orcabus.workflowmanager';
892897
export const oraDecompressionIcav2EventSource = 'orcabus.oradecompression';
893898
export const oraDecompressionIcav2EventDetailType = 'FastqListRowDecompressed';
894899
export const oraDecompressionStateMachinePrefix = 'oraDecompressionSfn';
900+
901+
/*
902+
Fastq Manager
903+
*/
904+
905+
// Tables
906+
export const fastqListRowTableName = 'fastqManagerDynamoDBTable';
907+
export const fastqSetTableName = 'fastqSetDynamoDBTable';
908+
export const fastqJobTableName = 'fastqJobDynamoDBTable';
909+
910+
// Table indexes
911+
export const fastqListRowManagerIndexes = [
912+
'rgid_ext',
913+
'instrument_run_id',
914+
'library_orcabus_id',
915+
'fastq_set_id',
916+
];
917+
export const fastqSetManagerIndexes = ['rgid_ext', 'instrument_run_id', 'library_orcabus_id'];
918+
export const fastqJobManagerIndexes = ['fastq_id', 'job_type', 'status'];
919+
920+
// S3 Buckets
921+
export const fastqManagerCacheBucket: Record<AppStage, string> = {
922+
[AppStage.BETA]: `fastq-manager-cache-${accountIdAlias.beta}-ap-southeast-2`,
923+
[AppStage.GAMMA]: `fastq-manager-cache-${accountIdAlias.gamma}-ap-southeast-2`,
924+
[AppStage.PROD]: `fastq-manager-cache-${accountIdAlias.prod}-ap-southeast-2`,
925+
};
926+
927+
export const ntsmBucket: Record<AppStage, string> = {
928+
[AppStage.BETA]: `ntsm-fingerprints-${accountIdAlias.beta}-ap-southeast-2`,
929+
[AppStage.GAMMA]: `ntsm-fingerprints-${accountIdAlias.gamma}-ap-southeast-2`,
930+
[AppStage.PROD]: `ntsm-fingerprints-${accountIdAlias.prod}-ap-southeast-2`,
931+
};
932+
933+
// Events
934+
export const fastqManagerEventSource = 'orcabus.fastqmanager';
935+
export const fastqManagerEventDetails = {
936+
updateFastqListRow: 'FastqListRowStateChange',
937+
updateFastqSet: 'FastqSetStateChange',
938+
};

‎config/stacks/fastqManager.ts

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import {
2+
AppStage,
3+
fastqListRowTableName,
4+
cognitoApiGatewayConfig,
5+
corsAllowOrigins,
6+
logsApiGatewayConfig,
7+
jwtSecretName,
8+
hostedZoneNameParameterPath,
9+
fastqListRowManagerIndexes,
10+
fastqSetTableName,
11+
fastqSetManagerIndexes,
12+
fastqJobTableName,
13+
fastqJobManagerIndexes,
14+
fastqManagerCacheBucket,
15+
ntsmBucket,
16+
fastqManagerEventSource,
17+
fastqManagerEventDetails,
18+
icav2PipelineCacheBucket,
19+
icav2PipelineCachePrefix,
20+
eventBusName,
21+
} from '../constants';
22+
import { FastqManagerTableConfig } from '../../lib/workload/stateful/stacks/fastq-manager-db/deploy/stack';
23+
import { FastqManagerStackConfig } from '../../lib/workload/stateless/stacks/fastq-manager/deploy/interfaces';
24+
25+
// Stateful
26+
export const getFastqManagerTableStackProps = (stage: AppStage): FastqManagerTableConfig => {
27+
return {
28+
/* DynamoDB table for fastq list rows */
29+
fastqListRowDynamodbTableName: fastqListRowTableName,
30+
fastqSetDynamodbTableName: fastqSetTableName,
31+
fastqJobDynamodbTableName: fastqJobTableName,
32+
/* Buckets */
33+
fastqManagerCacheBucketName: fastqManagerCacheBucket[stage],
34+
ntsmBucketName: ntsmBucket[stage],
35+
};
36+
};
37+
38+
// Stateless
39+
export const getFastqManagerStackProps = (stage: AppStage): FastqManagerStackConfig => {
40+
return {
41+
/*
42+
API Gateway props
43+
*/
44+
apiGatewayCognitoProps: {
45+
...cognitoApiGatewayConfig,
46+
corsAllowOrigins: corsAllowOrigins[stage],
47+
apiGwLogsConfig: logsApiGatewayConfig[stage],
48+
apiName: 'FastqManager',
49+
customDomainNamePrefix: 'fastq',
50+
},
51+
52+
/*
53+
Orcabus token and zone name for external lambda functions
54+
*/
55+
orcabusTokenSecretsManagerPath: jwtSecretName,
56+
hostedZoneNameSsmParameterPath: hostedZoneNameParameterPath,
57+
58+
/*
59+
Data tables
60+
*/
61+
fastqListRowDynamodbTableName: fastqListRowTableName,
62+
fastqSetDynamodbTableName: fastqSetTableName,
63+
fastqJobsDynamodbTableName: fastqJobTableName,
64+
/* Indexes - need permissions to query indexes */
65+
fastqListRowDynamodbIndexes: fastqListRowManagerIndexes,
66+
fastqSetDynamodbIndexes: fastqSetManagerIndexes,
67+
fastqJobsDynamodbIndexes: fastqJobManagerIndexes,
68+
69+
/*
70+
Buckets stuff
71+
*/
72+
pipelineCacheBucketName: icav2PipelineCacheBucket[stage],
73+
pipelineCachePrefix: icav2PipelineCachePrefix[stage],
74+
fastqManagerCacheBucketName: fastqManagerCacheBucket[stage],
75+
ntsmBucketName: ntsmBucket[stage],
76+
77+
/*
78+
Event bus stuff
79+
*/
80+
eventBusName: eventBusName,
81+
eventSource: fastqManagerEventSource,
82+
eventDetailType: fastqManagerEventDetails,
83+
};
84+
};

‎config/stacks/fileManager.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ import {
1818
oncoanalyserBucket,
1919
region,
2020
vpcProps,
21+
ntsmBucket,
22+
dataSharingCacheBucket,
2123
} from '../constants';
2224

2325
export const fileManagerBuckets = (stage: AppStage): string[] => {
@@ -27,6 +29,9 @@ export const fileManagerBuckets = (stage: AppStage): string[] => {
2729
eventSourceBuckets.push(icav2ArchiveAnalysisBucket[stage]);
2830
eventSourceBuckets.push(icav2ArchiveFastqBucket[stage]);
2931
}
32+
eventSourceBuckets.push(ntsmBucket[stage]);
33+
eventSourceBuckets.push(dataSharingCacheBucket[stage]);
34+
3035
return eventSourceBuckets;
3136
};
3237

‎config/stacks/shared.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
computeSecurityGroupName,
88
databasePort,
99
dataSchemaRegistryName,
10+
dataSharingCacheBucket,
1011
dbClusterEndpointHostParameterName,
1112
dbClusterIdentifier,
1213
dbClusterResourceIdParameterName,
@@ -17,6 +18,7 @@ import {
1718
icav2ArchiveAnalysisBucket,
1819
icav2ArchiveFastqBucket,
1920
icav2PipelineCacheBucket,
21+
ntsmBucket,
2022
oncoanalyserBucket,
2123
rdsMasterSecretName,
2224
vpcProps,
@@ -160,6 +162,19 @@ export const getEventSourceConstructProps = (stage: AppStage): EventSourceProps
160162
});
161163
}
162164

165+
// Add the ntsm bucket rule
166+
props.rules.push({
167+
bucket: ntsmBucket[stage],
168+
eventTypes,
169+
patterns: eventSourcePattern(),
170+
});
171+
172+
props.rules.push({
173+
bucket: dataSharingCacheBucket[stage],
174+
eventTypes,
175+
patterns: eventSourcePattern(),
176+
});
177+
163178
return props;
164179
};
165180

‎lib/workload/stateful/stacks/fastq-manager-db/Readme.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Fastq Manager DB
2+
3+
This comprises three dynamodb table that stores the metadata of fastq files.
4+
5+
The tables are:
6+
* fastq list row
7+
* fastq set
8+
* jobs
9+
10+
The fastq list row table is indexed by the id with the following global secondary indexes:
11+
* rgid_ext (the read group id, plus the instrument run id)
12+
* instrument_run_id (the instrument run id)
13+
* library_id (the library orcabus id)
14+
* fastq_set_id (easier to find the corresponding set object)
15+
16+
The fastq set table is indexed by the id with the following global secondary indexes:
17+
* library_id (the library orcabus id)
18+
19+
The jobs table is indexed by the id with the following global secondary indexes:
20+
* fastq_set_id (the fastq set id)
21+
* status (the status of the job)
22+

0 commit comments

Comments
 (0)