Merge branch 'main' of github.com:transcom/mymove into MAIN-B-22056_s…

…ns_sqs_deps_w_endpoint
transcom · Feb 21, 2025 · 34d9b75 · 34d9b75
2 parents 0c3bfd7 + b668ce3
commit 34d9b75
Show file tree

Hide file tree

Showing 235 changed files with 18,368 additions and 1,968 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -945,6 +945,7 @@ commands:
             FEATURE_FLAG_UNACCOMPANIED_BAGGAGE: 'false'
             FEATURE_FLAG_ENABLE_ALASKA: 'false'
             FEATURE_FLAG_BULK_ASSIGNMENT: 'false'
+
           command: |
             SHARD=$((${CIRCLE_NODE_INDEX}+1))
             PLAYWRIGHT_JUNIT_OUTPUT_NAME=playwright-results.xml \

diff --git a/.envrc b/.envrc
@@ -100,7 +100,23 @@ export APPLICATION=app
 
 # Migration Path
 export MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/schema;file://${MYMOVE_DIR}/migrations/app/secure"
-export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt"
+export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt" ##deprecated
+export DML_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/dml_migrations_manifest.txt"
+
+# DDL Migrations
+export DDL_TYPES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_types"
+export DDL_TYPES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_types_manifest.txt"
+
+export DDL_TABLES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_tables"
+export DDL_TABLES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_tables_manifest.txt"
+
+export DDL_VIEWS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_views"
+export DDL_VIEWS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_views_manifest.txt"
+
+export DDL_FUNCTIONS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_functions"
+export DDL_FUNCTIONS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_functions_manifest.txt"
+
+
 
 # Default DB configuration
 export DB_PASSWORD=mysecretpassword

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -220,7 +220,16 @@ stages:
       export DB_HOST=localhost
       export DB_PORT=5432
       export MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+      export DML_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+      export DDL_TYPES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_TABLES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_VIEWS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_FUNCTIONS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
       export MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+      export DDL_TYPES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+      export DDL_TABLES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+      export DDL_VIEWS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+      export DDL_FUNCTIONS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
       export EIA_KEY=db2522a43820268a41a802a16ae9fd26
 
 .setup_devseed_env_variables: &setup_devseed_env_variables
@@ -789,7 +798,16 @@ server_test:
     DB_NAME_TEST: test_db
     DTOD_USE_MOCK: 'true'
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENV: test
     ENVIRONMENT: test
@@ -948,7 +966,16 @@ integration_test_devseed:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1023,7 +1050,16 @@ integration_test_mtls:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1076,7 +1112,16 @@ integration_test_admin:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1134,7 +1179,16 @@ integration_test_my:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1193,7 +1247,16 @@ integration_test_office:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -159,18 +159,10 @@ repos:
     rev: v1.1.1
     hooks:
       - id: gen-docs
-        args: ['docs/adr']
+        args: ["docs/adr"]
       - id: markdown-toc
       - id: hadolint
 
-  - repo: local
-    hooks:
-      - id: migrations-manifest
-        name: migrations manifest
-        entry: scripts/update-migrations-manifest
-        language: script
-        pass_filenames: false
-
   - repo: local
     hooks:
       - id: scripts-docs

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt

diff --git a/Dockerfile.dp3 b/Dockerfile.dp3
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 # hadolint ignore=DL3007
 FROM gcr.io/distroless/base-debian12@sha256:74ddbf52d93fafbdd21b399271b0b4aac1babf8fa98cab59e5692e01169a1348

diff --git a/Dockerfile.migrations b/Dockerfile.migrations
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt
@@ -20,6 +20,15 @@ COPY bin/rds-ca-2019-root.pem /bin/rds-ca-2019-root.pem
 COPY bin/milmove /bin/milmove
 
 COPY migrations/app/schema /migrate/schema
+COPY migrations/app/ddl_migrations/ddl_types /migrate/ddl_migrations/ddl_types
+COPY migrations/app/ddl_migrations/ddl_tables /migrate/ddl_migrations/ddl_tables
+COPY migrations/app/ddl_migrations/ddl_views /migrate/ddl_migrations/ddl_views
+COPY migrations/app/ddl_migrations/ddl_functions /migrate/ddl_migrations/ddl_functions
 COPY migrations/app/migrations_manifest.txt /migrate/migrations_manifest.txt
-
-ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt"]
+COPY migrations/app/dml_migrations_manifest.txt /migrate/dml_migrations_manifest.txt
+COPY migrations/app/ddl_types_manifest.txt /migrate/ddl_types_manifest.txt
+COPY migrations/app/ddl_tables_manifest.txt /migrate/ddl_tables_manifest.txt
+COPY migrations/app/ddl_views_manifest.txt /migrate/ddl_views_manifest.txt
+COPY migrations/app/ddl_functions_manifest.txt /migrate/ddl_functions_manifest.txt
+# hadolint ignore=DL3025
+ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt", '-d', '/migrate/dml_migrations_manifest.txt', '-t', '/migrate/ddl_types_manifest.txt', '-T', '/migrate/ddl_tables_manifest.txt', '-V', '/migrate/ddl_views_manifest.txt', '-F', '/migrate/ddl_functions_manifest.txt']
diff --git a/Dockerfile.tasks b/Dockerfile.tasks
@@ -1,4 +1,4 @@
-FROM harbor.csde.caci.com/docker.io/debian:stable AS build-env
+FROM harbor.csde.caci.com/docker.io/library/debian:stable AS build-env
 
 COPY config/tls/dod-wcf-root-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-root-ca-1.pem.crt
 COPY config/tls/dod-wcf-intermediate-ca-1.pem /usr/local/share/ca-certificates/dod-wcf-intermediate-ca-1.pem.crt

diff --git a/cmd/milmove/gen_migration.go b/cmd/milmove/gen_migration.go
@@ -69,7 +69,7 @@ func genMigrationFunction(cmd *cobra.Command, args []string) error {
 	}
 
 	migrationPath := v.GetString(cli.MigrationGenPathFlag)
-	migrationManifest := v.GetString(cli.MigrationManifestFlag)
+	migrationManifest := v.GetString(cli.DMLMigrationManifestFlag)
 	migrationVersion := v.GetString(cli.MigrationVersionFlag)
 	migrationName := v.GetString(cli.MigrationNameFlag)
 	migrationType := v.GetString(cli.MigrationTypeFlag)

diff --git a/cmd/milmove/migrate.go b/cmd/milmove/migrate.go
@@ -174,7 +174,6 @@ func migrateFunction(cmd *cobra.Command, args []string) error {
 	// Remove any extra quotes around path
 	trimmedMigrationPaths := strings.Trim(v.GetString(cli.MigrationPathFlag), "\"")
 	migrationPaths := expandPaths(strings.Split(trimmedMigrationPaths, ";"))
-	logger.Info(fmt.Sprintf("using migration paths %q", migrationPaths))
 
 	logger.Info("migration Path from s3")
 
@@ -308,5 +307,136 @@ func migrateFunction(cmd *cobra.Command, args []string) error {
 		return errors.Wrap(errUp, "error running migrations")
 	}
 
+	// Begin DDL migrations
+	ddlTypesManifest := expandPath(v.GetString(cli.DDLTypesMigrationManifestFlag))
+	ddlTypesManifestPath := expandPath(v.GetString(cli.DDLTypesMigrationPathFlag))
+
+	ddlTablesManifest := expandPath(v.GetString(cli.DDLTablesMigrationManifestFlag))
+	ddlTablesPath := expandPath(v.GetString(cli.DDLTablesMigrationPathFlag))
+
+	ddlViewsManifest := expandPath(v.GetString(cli.DDLViewsMigrationManifestFlag))
+	ddlViewsPath := expandPath(v.GetString(cli.DDLViewsMigrationPathFlag))
+
+	ddlFunctionsManifest := expandPath(v.GetString(cli.DDLFunctionsMigrationManifestFlag))
+	ddlFunctionsPath := expandPath(v.GetString(cli.DDLFunctionsMigrationPathFlag))
+
+	ddlObjects := []struct {
+		name     string
+		manifest string
+		path     string
+	}{
+		{"DDL Types", ddlTypesManifest, ddlTypesManifestPath},
+		{"DDL Tables", ddlTablesManifest, ddlTablesPath},
+		{"DDL Views", ddlViewsManifest, ddlViewsPath},
+		{"DDL Functions", ddlFunctionsManifest, ddlFunctionsPath},
+	}
+
+	for _, ddlObj := range ddlObjects {
+		logger.Info(fmt.Sprintf("=== Processing %s ===", ddlObj.name))
+		logger.Info(fmt.Sprintf("Using manifest %q", ddlObj.manifest))
+		filenames, errListFiles := fileHelper.ListFiles(ddlObj.path, s3Client)
+		if errListFiles != nil {
+			logger.Fatal(fmt.Sprintf("Error listing %s directory %s", ddlObj.name, ddlObj.path), zap.Error(errListFiles))
+		}
+
+		ddlMigrationFiles := map[string][]string{
+			ddlObj.path: filenames,
+		}
+
+		manifest, err := os.Open(ddlObj.manifest[len("file://"):])
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("error reading %s manifest", ddlObj.name))
+		}
+
+		scanner := bufio.NewScanner(manifest)
+		for scanner.Scan() {
+			target := scanner.Text()
+			if strings.HasPrefix(target, "#") {
+				continue
+			}
+
+			uri := ""
+			for dir, files := range ddlMigrationFiles {
+				for _, filename := range files {
+					if target == filename {
+						uri = fmt.Sprintf("%s/%s", dir, filename)
+						break
+					}
+				}
+			}
+
+			if len(uri) == 0 {
+				return errors.Errorf("Error finding %s migration for filename %q", ddlObj.name, target)
+			}
+
+			m, err := pop.ParseMigrationFilename(target)
+			if err != nil {
+				return errors.Wrapf(err, "error parsing %s migration filename %q", ddlObj.name, uri)
+			}
+
+			b := &migrate.Builder{Match: m, Path: uri}
+			migration, errCompile := b.Compile(s3Client, wait, logger)
+			if errCompile != nil {
+				return errors.Wrap(errCompile, fmt.Sprintf("Error compiling %s migration", ddlObj.name))
+			}
+
+			if err := migration.Run(dbConnection); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("error executing %s migration", ddlObj.name))
+			}
+
+			logger.Info(fmt.Sprintf("Successfully executed %s: %s", ddlObj.name, target))
+		}
+		manifest.Close()
+	}
+
+	// After DDL migrations, process DML migrations
+	dmlManifest := expandPath(v.GetString(cli.DMLMigrationManifestFlag))
+	logger.Info(fmt.Sprintf("using DML migration manifest %q", dmlManifest))
+
+	// Create a new migrator for DML migrations
+	dmlMigrator := pop.NewMigrator(dbConnection)
+
+	manifest, err = os.Open(dmlManifest[len("file://"):])
+	if err != nil {
+		return errors.Wrap(err, "error reading DML manifest")
+	}
+
+	scanner = bufio.NewScanner(manifest)
+	for scanner.Scan() {
+		target := scanner.Text()
+		if strings.HasPrefix(target, "#") {
+			continue
+		}
+		uri := ""
+		for dir, filenames := range migrationFiles {
+			for _, filename := range filenames {
+				if target == filename {
+					uri = fmt.Sprintf("%s/%s", dir, filename)
+					break
+				}
+			}
+		}
+		if len(uri) == 0 {
+			return errors.Errorf("Error finding DML migration for filename %q", target)
+		}
+		m, err := pop.ParseMigrationFilename(target)
+		if err != nil {
+			return errors.Wrapf(err, "error parsing DML migration filename %q", uri)
+		}
+		b := &migrate.Builder{Match: m, Path: uri}
+		migration, errCompile := b.Compile(s3Client, wait, logger)
+		if errCompile != nil {
+			return errors.Wrap(errCompile, "Error compiling DML migration")
+		}
+
+		dmlMigrator.UpMigrations.Migrations = append(dmlMigrator.UpMigrations.Migrations, *migration)
+	}
+
+	// Run DML migrations and track versions
+	errUp = dmlMigrator.Up()
+	if errUp != nil {
+		return errors.Wrap(errUp, "error running DML migrations")
+	}
+
 	return nil
 }