Merge branch 'main' into B-22726-PPM-Disbursement

transcom · Feb 24, 2025 · 96a1078 · 96a1078
2 parents ecddfd0 + 08877ed
commit 96a1078
Show file tree

Hide file tree

Showing 225 changed files with 17,311 additions and 1,935 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -945,6 +945,7 @@ commands:
             FEATURE_FLAG_UNACCOMPANIED_BAGGAGE: 'false'
             FEATURE_FLAG_ENABLE_ALASKA: 'false'
             FEATURE_FLAG_BULK_ASSIGNMENT: 'false'
+
           command: |
             SHARD=$((${CIRCLE_NODE_INDEX}+1))
             PLAYWRIGHT_JUNIT_OUTPUT_NAME=playwright-results.xml \

diff --git a/.envrc b/.envrc
@@ -100,7 +100,23 @@ export APPLICATION=app
 
 # Migration Path
 export MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/schema;file://${MYMOVE_DIR}/migrations/app/secure"
-export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt"
+export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt" ##deprecated
+export DML_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/dml_migrations_manifest.txt"
+
+# DDL Migrations
+export DDL_TYPES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_types"
+export DDL_TYPES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_types_manifest.txt"
+
+export DDL_TABLES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_tables"
+export DDL_TABLES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_tables_manifest.txt"
+
+export DDL_VIEWS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_views"
+export DDL_VIEWS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_views_manifest.txt"
+
+export DDL_FUNCTIONS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_functions"
+export DDL_FUNCTIONS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_functions_manifest.txt"
+
+
 
 # Default DB configuration
 export DB_PASSWORD=mysecretpassword
@@ -232,19 +248,22 @@ export TZ="UTC"
 
 # AWS development access
 #
-# To use S3/SES for local builds, you'll need to uncomment the following.
+# To use S3/SES or SNS & SQS for local builds, you'll need to uncomment the following.
 # Do not commit the change:
 #
 #   export STORAGE_BACKEND=s3
 #   export EMAIL_BACKEND=ses
+#   export RECEIVER_BACKEND=sns_sqs
 #
 # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally
 # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally
+# Instructions for using SNS&SQS backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/2793242625/How+to+test+notifications+receiver+locally
 #
 # The default and equivalent to not being set is:
 #
 #   export STORAGE_BACKEND=local
 #   export EMAIL_BACKEND=local
+#   export RECEIVER_BACKEND=local
 #
 # Setting region and profile conditionally while we migrate from com to govcloud.
 if [ "$STORAGE_BACKEND" == "s3" ]; then
@@ -258,6 +277,13 @@ export AWS_S3_KEY_NAMESPACE=$USER
 export AWS_SES_DOMAIN="devlocal.dp3.us"
 export AWS_SES_REGION="us-gov-west-1"
 
+if [ "$RECEIVER_BACKEND" == "sns_sqs" ]; then
+  export SNS_TAGS_UPDATED_TOPIC="app_s3_tag_events"
+  export SNS_REGION="us-gov-west-1"
+# cleanup flag false by default, only used at server startup to wipe receiver artifacts from previous runs
+# export RECEIVER_CLEANUP_ON_START=false
+fi
+
 # To use s3 links aws-bucketname/xx/user/ for local builds,
 # you'll need to add the following to your .envrc.local:
 #
@@ -444,4 +470,4 @@ then
 fi
 
 # Check that all required environment variables are set
-check_required_variables
+check_required_variables
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -220,7 +220,16 @@ stages:
       export DB_HOST=localhost
       export DB_PORT=5432
       export MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+      export DML_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+      export DDL_TYPES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_TABLES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_VIEWS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_FUNCTIONS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
       export MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+      export DDL_TYPES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+      export DDL_TABLES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+      export DDL_VIEWS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+      export DDL_FUNCTIONS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
       export EIA_KEY=db2522a43820268a41a802a16ae9fd26
 
 .setup_devseed_env_variables: &setup_devseed_env_variables
@@ -789,7 +798,16 @@ server_test:
     DB_NAME_TEST: test_db
     DTOD_USE_MOCK: 'true'
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENV: test
     ENVIRONMENT: test
@@ -948,7 +966,16 @@ integration_test_devseed:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1023,7 +1050,16 @@ integration_test_mtls:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1076,7 +1112,16 @@ integration_test_admin:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1134,7 +1179,16 @@ integration_test_my:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1193,7 +1247,16 @@ integration_test_office:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -159,18 +159,10 @@ repos:
     rev: v1.1.1
     hooks:
       - id: gen-docs
-        args: ['docs/adr']
+        args: ["docs/adr"]
       - id: markdown-toc
       - id: hadolint
 
-  - repo: local
-    hooks:
-      - id: migrations-manifest
-        name: migrations manifest
-        entry: scripts/update-migrations-manifest
-        language: script
-        pass_filenames: false
-
   - repo: local
     hooks:
       - id: scripts-docs

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt

diff --git a/Dockerfile.dp3 b/Dockerfile.dp3
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 # hadolint ignore=DL3007
 FROM gcr.io/distroless/base-debian12@sha256:74ddbf52d93fafbdd21b399271b0b4aac1babf8fa98cab59e5692e01169a1348

diff --git a/Dockerfile.migrations b/Dockerfile.migrations
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt
@@ -20,6 +20,15 @@ COPY bin/rds-ca-2019-root.pem /bin/rds-ca-2019-root.pem
 COPY bin/milmove /bin/milmove
 
 COPY migrations/app/schema /migrate/schema
+COPY migrations/app/ddl_migrations/ddl_types /migrate/ddl_migrations/ddl_types
+COPY migrations/app/ddl_migrations/ddl_tables /migrate/ddl_migrations/ddl_tables
+COPY migrations/app/ddl_migrations/ddl_views /migrate/ddl_migrations/ddl_views
+COPY migrations/app/ddl_migrations/ddl_functions /migrate/ddl_migrations/ddl_functions
 COPY migrations/app/migrations_manifest.txt /migrate/migrations_manifest.txt
-
-ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt"]
+COPY migrations/app/dml_migrations_manifest.txt /migrate/dml_migrations_manifest.txt
+COPY migrations/app/ddl_types_manifest.txt /migrate/ddl_types_manifest.txt
+COPY migrations/app/ddl_tables_manifest.txt /migrate/ddl_tables_manifest.txt
+COPY migrations/app/ddl_views_manifest.txt /migrate/ddl_views_manifest.txt
+COPY migrations/app/ddl_functions_manifest.txt /migrate/ddl_functions_manifest.txt
+# hadolint ignore=DL3025
+ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt", '-d', '/migrate/dml_migrations_manifest.txt', '-t', '/migrate/ddl_types_manifest.txt', '-T', '/migrate/ddl_tables_manifest.txt', '-V', '/migrate/ddl_views_manifest.txt', '-F', '/migrate/ddl_functions_manifest.txt']
diff --git a/Dockerfile.tasks b/Dockerfile.tasks
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt

diff --git a/cmd/milmove/gen_migration.go b/cmd/milmove/gen_migration.go
@@ -69,7 +69,7 @@ func genMigrationFunction(cmd *cobra.Command, args []string) error {
 	}
 
 	migrationPath := v.GetString(cli.MigrationGenPathFlag)
-	migrationManifest := v.GetString(cli.MigrationManifestFlag)
+	migrationManifest := v.GetString(cli.DMLMigrationManifestFlag)
 	migrationVersion := v.GetString(cli.MigrationVersionFlag)
 	migrationName := v.GetString(cli.MigrationNameFlag)
 	migrationType := v.GetString(cli.MigrationTypeFlag)