initial merge of main with new migration process

transcom · Feb 21, 2025 · 93499f5 · 93499f5
2 parents 6019d16 + ec359eb
commit 93499f5
Show file tree

Hide file tree

Showing 278 changed files with 21,387 additions and 2,183 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -945,6 +945,7 @@ commands:
             FEATURE_FLAG_UNACCOMPANIED_BAGGAGE: 'false'
             FEATURE_FLAG_ENABLE_ALASKA: 'false'
             FEATURE_FLAG_BULK_ASSIGNMENT: 'false'
+
           command: |
             SHARD=$((${CIRCLE_NODE_INDEX}+1))
             PLAYWRIGHT_JUNIT_OUTPUT_NAME=playwright-results.xml \

diff --git a/.envrc b/.envrc
@@ -100,7 +100,23 @@ export APPLICATION=app
 
 # Migration Path
 export MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/schema;file://${MYMOVE_DIR}/migrations/app/secure"
-export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt"
+export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt" ##deprecated
+export DML_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/dml_migrations_manifest.txt"
+
+# DDL Migrations
+export DDL_TYPES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_types"
+export DDL_TYPES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_types_manifest.txt"
+
+export DDL_TABLES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_tables"
+export DDL_TABLES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_tables_manifest.txt"
+
+export DDL_VIEWS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_views"
+export DDL_VIEWS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_views_manifest.txt"
+
+export DDL_FUNCTIONS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_functions"
+export DDL_FUNCTIONS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_functions_manifest.txt"
+
+
 
 # Default DB configuration
 export DB_PASSWORD=mysecretpassword

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -152,7 +152,7 @@ stages:
   #build off prd variables
   - export ECR_REPOSITORY_URI=${PRD_ACCOUNT_ID}.dkr.ecr.${PRD_REGION}.amazonaws.com
   - export APP_DOCKER_FILE=Dockerfile
-  - export TASK_DOCKER_FILE=Dockerfile
+  - export TASK_DOCKER_FILE=Dockerfile.tasks
   #TODO: update exp to prod
   - export APP_ENVIRONMENT=prd
 
@@ -220,7 +220,16 @@ stages:
       export DB_HOST=localhost
       export DB_PORT=5432
       export MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+      export DML_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+      export DDL_TYPES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_TABLES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_VIEWS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_FUNCTIONS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
       export MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+      export DDL_TYPES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+      export DDL_TABLES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+      export DDL_VIEWS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+      export DDL_FUNCTIONS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
       export EIA_KEY=db2522a43820268a41a802a16ae9fd26
 
 .setup_devseed_env_variables: &setup_devseed_env_variables
@@ -789,7 +798,16 @@ server_test:
     DB_NAME_TEST: test_db
     DTOD_USE_MOCK: 'true'
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENV: test
     ENVIRONMENT: test
@@ -948,7 +966,16 @@ integration_test_devseed:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1023,7 +1050,16 @@ integration_test_mtls:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1076,7 +1112,16 @@ integration_test_admin:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1134,7 +1179,16 @@ integration_test_my:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1193,7 +1247,16 @@ integration_test_office:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -159,18 +159,10 @@ repos:
     rev: v1.1.1
     hooks:
       - id: gen-docs
-        args: ['docs/adr']
+        args: ["docs/adr"]
       - id: markdown-toc
       - id: hadolint
 
-  - repo: local
-    hooks:
-      - id: migrations-manifest
-        name: migrations manifest
-        entry: scripts/update-migrations-manifest
-        language: script
-        pass_filenames: false
-
   - repo: local
     hooks:
       - id: scripts-docs

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt

diff --git a/Dockerfile.dp3 b/Dockerfile.dp3
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 # hadolint ignore=DL3007
 FROM gcr.io/distroless/base-debian12@sha256:74ddbf52d93fafbdd21b399271b0b4aac1babf8fa98cab59e5692e01169a1348

diff --git a/Dockerfile.migrations b/Dockerfile.migrations
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt
@@ -20,6 +20,15 @@ COPY bin/rds-ca-2019-root.pem /bin/rds-ca-2019-root.pem
 COPY bin/milmove /bin/milmove
 
 COPY migrations/app/schema /migrate/schema
+COPY migrations/app/ddl_migrations/ddl_types /migrate/ddl_migrations/ddl_types
+COPY migrations/app/ddl_migrations/ddl_tables /migrate/ddl_migrations/ddl_tables
+COPY migrations/app/ddl_migrations/ddl_views /migrate/ddl_migrations/ddl_views
+COPY migrations/app/ddl_migrations/ddl_functions /migrate/ddl_migrations/ddl_functions
 COPY migrations/app/migrations_manifest.txt /migrate/migrations_manifest.txt
-
-ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt"]
+COPY migrations/app/dml_migrations_manifest.txt /migrate/dml_migrations_manifest.txt
+COPY migrations/app/ddl_types_manifest.txt /migrate/ddl_types_manifest.txt
+COPY migrations/app/ddl_tables_manifest.txt /migrate/ddl_tables_manifest.txt
+COPY migrations/app/ddl_views_manifest.txt /migrate/ddl_views_manifest.txt
+COPY migrations/app/ddl_functions_manifest.txt /migrate/ddl_functions_manifest.txt
+# hadolint ignore=DL3025
+ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt", '-d', '/migrate/dml_migrations_manifest.txt', '-t', '/migrate/ddl_types_manifest.txt', '-T', '/migrate/ddl_tables_manifest.txt', '-V', '/migrate/ddl_views_manifest.txt', '-F', '/migrate/ddl_functions_manifest.txt']
diff --git a/Dockerfile.tasks b/Dockerfile.tasks
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt

diff --git a/cmd/milmove/gen_migration.go b/cmd/milmove/gen_migration.go
@@ -69,7 +69,7 @@ func genMigrationFunction(cmd *cobra.Command, args []string) error {
 	}
 
 	migrationPath := v.GetString(cli.MigrationGenPathFlag)
-	migrationManifest := v.GetString(cli.MigrationManifestFlag)
+	migrationManifest := v.GetString(cli.DMLMigrationManifestFlag)
 	migrationVersion := v.GetString(cli.MigrationVersionFlag)
 	migrationName := v.GetString(cli.MigrationNameFlag)
 	migrationType := v.GetString(cli.MigrationTypeFlag)