Merge branch 'main' of github.com:transcom/mymove into MAIN-B-22039-r…

…emove_lat_lon
transcom · Feb 25, 2025 · 7ad4c72 · 7ad4c72
2 parents 6e533cf + 810fb35
commit 7ad4c72
Show file tree

Hide file tree

Showing 115 changed files with 4,322 additions and 293 deletions.
diff --git a/.envrc b/.envrc
@@ -100,7 +100,23 @@ export APPLICATION=app
 
 # Migration Path
 export MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/schema;file://${MYMOVE_DIR}/migrations/app/secure"
-export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt"
+export MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/migrations_manifest.txt" ##deprecated
+export DML_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/dml_migrations_manifest.txt"
+
+# DDL Migrations
+export DDL_TYPES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_types"
+export DDL_TYPES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_types_manifest.txt"
+
+export DDL_TABLES_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_tables"
+export DDL_TABLES_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_tables_manifest.txt"
+
+export DDL_VIEWS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_views"
+export DDL_VIEWS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_views_manifest.txt"
+
+export DDL_FUNCTIONS_MIGRATION_PATH="file://${MYMOVE_DIR}/migrations/app/ddl_migrations/ddl_functions"
+export DDL_FUNCTIONS_MIGRATION_MANIFEST="${MYMOVE_DIR}/migrations/app/ddl_functions_manifest.txt"
+
+
 
 # Default DB configuration
 export DB_PASSWORD=mysecretpassword
@@ -232,19 +248,22 @@ export TZ="UTC"
 
 # AWS development access
 #
-# To use S3/SES for local builds, you'll need to uncomment the following.
+# To use S3/SES or SNS & SQS for local builds, you'll need to uncomment the following.
 # Do not commit the change:
 #
 #   export STORAGE_BACKEND=s3
 #   export EMAIL_BACKEND=ses
+#   export RECEIVER_BACKEND=sns_sqs
 #
 # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally
 # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally
+# Instructions for using SNS&SQS backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/2793242625/How+to+test+notifications+receiver+locally
 #
 # The default and equivalent to not being set is:
 #
 #   export STORAGE_BACKEND=local
 #   export EMAIL_BACKEND=local
+#   export RECEIVER_BACKEND=local
 #
 # Setting region and profile conditionally while we migrate from com to govcloud.
 if [ "$STORAGE_BACKEND" == "s3" ]; then
@@ -258,6 +277,13 @@ export AWS_S3_KEY_NAMESPACE=$USER
 export AWS_SES_DOMAIN="devlocal.dp3.us"
 export AWS_SES_REGION="us-gov-west-1"
 
+if [ "$RECEIVER_BACKEND" == "sns_sqs" ]; then
+  export SNS_TAGS_UPDATED_TOPIC="app_s3_tag_events"
+  export SNS_REGION="us-gov-west-1"
+# cleanup flag false by default, only used at server startup to wipe receiver artifacts from previous runs
+# export RECEIVER_CLEANUP_ON_START=false
+fi
+
 # To use s3 links aws-bucketname/xx/user/ for local builds,
 # you'll need to add the following to your .envrc.local:
 #
@@ -444,4 +470,4 @@ then
 fi
 
 # Check that all required environment variables are set
-check_required_variables
+check_required_variables
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -220,7 +220,16 @@ stages:
       export DB_HOST=localhost
       export DB_PORT=5432
       export MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+      export DML_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+      export DDL_TYPES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_TABLES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_VIEWS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_FUNCTIONS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
       export MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+      export DDL_TYPES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+      export DDL_TABLES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+      export DDL_VIEWS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+      export DDL_FUNCTIONS_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
       export EIA_KEY=db2522a43820268a41a802a16ae9fd26
 
 .setup_devseed_env_variables: &setup_devseed_env_variables
@@ -789,7 +798,16 @@ server_test:
     DB_NAME_TEST: test_db
     DTOD_USE_MOCK: 'true'
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENV: test
     ENVIRONMENT: test
@@ -948,7 +966,16 @@ integration_test_devseed:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1023,7 +1050,16 @@ integration_test_mtls:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1076,7 +1112,16 @@ integration_test_admin:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1134,7 +1179,16 @@ integration_test_my:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b
@@ -1193,7 +1247,16 @@ integration_test_office:
     DB_NAME: dev_db
     DB_NAME_DEV: dev_db
     MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
+    DML_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
+    DDL_TYPES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+    DDL_TABLES_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+    DDL_VIEWS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+    DDL_FUNCTIONS_MIGRATION_MANIFEST: '/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
     MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
+    DDL_TYPES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
+    DDL_TABLES_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
+    DDL_VIEWS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_views'
+    DDL_FUNCTIONS_MIGRATION_PATH: 'file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_functions'
     EIA_KEY: db2522a43820268a41a802a16ae9fd26 # dummy key generated with openssl rand -hex 16
     ENVIRONMENT: development
     DOD_CA_PACKAGE: /builds/milmove/mymove/config/tls/milmove-cert-bundle.p7b

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -159,18 +159,10 @@ repos:
     rev: v1.1.1
     hooks:
       - id: gen-docs
-        args: ['docs/adr']
+        args: ["docs/adr"]
       - id: markdown-toc
       - id: hadolint
 
-  - repo: local
-    hooks:
-      - id: migrations-manifest
-        name: migrations manifest
-        entry: scripts/update-migrations-manifest
-        language: script
-        pass_filenames: false
-
   - repo: local
     hooks:
       - id: scripts-docs

diff --git a/Dockerfile.migrations b/Dockerfile.migrations
@@ -20,6 +20,15 @@ COPY bin/rds-ca-2019-root.pem /bin/rds-ca-2019-root.pem
 COPY bin/milmove /bin/milmove
 
 COPY migrations/app/schema /migrate/schema
+COPY migrations/app/ddl_migrations/ddl_types /migrate/ddl_migrations/ddl_types
+COPY migrations/app/ddl_migrations/ddl_tables /migrate/ddl_migrations/ddl_tables
+COPY migrations/app/ddl_migrations/ddl_views /migrate/ddl_migrations/ddl_views
+COPY migrations/app/ddl_migrations/ddl_functions /migrate/ddl_migrations/ddl_functions
 COPY migrations/app/migrations_manifest.txt /migrate/migrations_manifest.txt
-
-ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt"]
+COPY migrations/app/dml_migrations_manifest.txt /migrate/dml_migrations_manifest.txt
+COPY migrations/app/ddl_types_manifest.txt /migrate/ddl_types_manifest.txt
+COPY migrations/app/ddl_tables_manifest.txt /migrate/ddl_tables_manifest.txt
+COPY migrations/app/ddl_views_manifest.txt /migrate/ddl_views_manifest.txt
+COPY migrations/app/ddl_functions_manifest.txt /migrate/ddl_functions_manifest.txt
+# hadolint ignore=DL3025
+ENTRYPOINT ["/bin/milmove", "migrate", "-p", "file:///migrate/migrations", "-m", "/migrate/migrations_manifest.txt", '-d', '/migrate/dml_migrations_manifest.txt', '-t', '/migrate/ddl_types_manifest.txt', '-T', '/migrate/ddl_tables_manifest.txt', '-V', '/migrate/ddl_views_manifest.txt', '-F', '/migrate/ddl_functions_manifest.txt']
diff --git a/cmd/milmove/gen_migration.go b/cmd/milmove/gen_migration.go
@@ -69,7 +69,7 @@ func genMigrationFunction(cmd *cobra.Command, args []string) error {
 	}
 
 	migrationPath := v.GetString(cli.MigrationGenPathFlag)
-	migrationManifest := v.GetString(cli.MigrationManifestFlag)
+	migrationManifest := v.GetString(cli.DMLMigrationManifestFlag)
 	migrationVersion := v.GetString(cli.MigrationVersionFlag)
 	migrationName := v.GetString(cli.MigrationNameFlag)
 	migrationType := v.GetString(cli.MigrationTypeFlag)

diff --git a/cmd/milmove/migrate.go b/cmd/milmove/migrate.go
@@ -174,7 +174,6 @@ func migrateFunction(cmd *cobra.Command, args []string) error {
 	// Remove any extra quotes around path
 	trimmedMigrationPaths := strings.Trim(v.GetString(cli.MigrationPathFlag), "\"")
 	migrationPaths := expandPaths(strings.Split(trimmedMigrationPaths, ";"))
-	logger.Info(fmt.Sprintf("using migration paths %q", migrationPaths))
 
 	logger.Info("migration Path from s3")
 
@@ -308,5 +307,136 @@ func migrateFunction(cmd *cobra.Command, args []string) error {
 		return errors.Wrap(errUp, "error running migrations")
 	}
 
+	// Begin DDL migrations
+	ddlTypesManifest := expandPath(v.GetString(cli.DDLTypesMigrationManifestFlag))
+	ddlTypesManifestPath := expandPath(v.GetString(cli.DDLTypesMigrationPathFlag))
+
+	ddlTablesManifest := expandPath(v.GetString(cli.DDLTablesMigrationManifestFlag))
+	ddlTablesPath := expandPath(v.GetString(cli.DDLTablesMigrationPathFlag))
+
+	ddlViewsManifest := expandPath(v.GetString(cli.DDLViewsMigrationManifestFlag))
+	ddlViewsPath := expandPath(v.GetString(cli.DDLViewsMigrationPathFlag))
+
+	ddlFunctionsManifest := expandPath(v.GetString(cli.DDLFunctionsMigrationManifestFlag))
+	ddlFunctionsPath := expandPath(v.GetString(cli.DDLFunctionsMigrationPathFlag))
+
+	ddlObjects := []struct {
+		name     string
+		manifest string
+		path     string
+	}{
+		{"DDL Types", ddlTypesManifest, ddlTypesManifestPath},
+		{"DDL Tables", ddlTablesManifest, ddlTablesPath},
+		{"DDL Views", ddlViewsManifest, ddlViewsPath},
+		{"DDL Functions", ddlFunctionsManifest, ddlFunctionsPath},
+	}
+
+	for _, ddlObj := range ddlObjects {
+		logger.Info(fmt.Sprintf("=== Processing %s ===", ddlObj.name))
+		logger.Info(fmt.Sprintf("Using manifest %q", ddlObj.manifest))
+		filenames, errListFiles := fileHelper.ListFiles(ddlObj.path, s3Client)
+		if errListFiles != nil {
+			logger.Fatal(fmt.Sprintf("Error listing %s directory %s", ddlObj.name, ddlObj.path), zap.Error(errListFiles))
+		}
+
+		ddlMigrationFiles := map[string][]string{
+			ddlObj.path: filenames,
+		}
+
+		manifest, err := os.Open(ddlObj.manifest[len("file://"):])
+		if err != nil {
+			return errors.Wrap(err, fmt.Sprintf("error reading %s manifest", ddlObj.name))
+		}
+
+		scanner := bufio.NewScanner(manifest)
+		for scanner.Scan() {
+			target := scanner.Text()
+			if strings.HasPrefix(target, "#") {
+				continue
+			}
+
+			uri := ""
+			for dir, files := range ddlMigrationFiles {
+				for _, filename := range files {
+					if target == filename {
+						uri = fmt.Sprintf("%s/%s", dir, filename)
+						break
+					}
+				}
+			}
+
+			if len(uri) == 0 {
+				return errors.Errorf("Error finding %s migration for filename %q", ddlObj.name, target)
+			}
+
+			m, err := pop.ParseMigrationFilename(target)
+			if err != nil {
+				return errors.Wrapf(err, "error parsing %s migration filename %q", ddlObj.name, uri)
+			}
+
+			b := &migrate.Builder{Match: m, Path: uri}
+			migration, errCompile := b.Compile(s3Client, wait, logger)
+			if errCompile != nil {
+				return errors.Wrap(errCompile, fmt.Sprintf("Error compiling %s migration", ddlObj.name))
+			}
+
+			if err := migration.Run(dbConnection); err != nil {
+				return errors.Wrap(err, fmt.Sprintf("error executing %s migration", ddlObj.name))
+			}
+
+			logger.Info(fmt.Sprintf("Successfully executed %s: %s", ddlObj.name, target))
+		}
+		manifest.Close()
+	}
+
+	// After DDL migrations, process DML migrations
+	dmlManifest := expandPath(v.GetString(cli.DMLMigrationManifestFlag))
+	logger.Info(fmt.Sprintf("using DML migration manifest %q", dmlManifest))
+
+	// Create a new migrator for DML migrations
+	dmlMigrator := pop.NewMigrator(dbConnection)
+
+	manifest, err = os.Open(dmlManifest[len("file://"):])
+	if err != nil {
+		return errors.Wrap(err, "error reading DML manifest")
+	}
+
+	scanner = bufio.NewScanner(manifest)
+	for scanner.Scan() {
+		target := scanner.Text()
+		if strings.HasPrefix(target, "#") {
+			continue
+		}
+		uri := ""
+		for dir, filenames := range migrationFiles {
+			for _, filename := range filenames {
+				if target == filename {
+					uri = fmt.Sprintf("%s/%s", dir, filename)
+					break
+				}
+			}
+		}
+		if len(uri) == 0 {
+			return errors.Errorf("Error finding DML migration for filename %q", target)
+		}
+		m, err := pop.ParseMigrationFilename(target)
+		if err != nil {
+			return errors.Wrapf(err, "error parsing DML migration filename %q", uri)
+		}
+		b := &migrate.Builder{Match: m, Path: uri}
+		migration, errCompile := b.Compile(s3Client, wait, logger)
+		if errCompile != nil {
+			return errors.Wrap(errCompile, "Error compiling DML migration")
+		}
+
+		dmlMigrator.UpMigrations.Migrations = append(dmlMigrator.UpMigrations.Migrations, *migration)
+	}
+
+	// Run DML migrations and track versions
+	errUp = dmlMigrator.Up()
+	if errUp != nil {
+		return errors.Wrap(errUp, "error running DML migrations")
+	}
+
 	return nil
 }