Merge branch 'main' into MAIN-B-22769

.envrc

@@ -248,19 +248,22 @@ export TZ="UTC"
 
 # AWS development access
 #
-# To use S3/SES for local builds, you'll need to uncomment the following.
+# To use S3/SES or SNS & SQS for local builds, you'll need to uncomment the following.
 # Do not commit the change:
 #
 #   export STORAGE_BACKEND=s3
 #   export EMAIL_BACKEND=ses
+#   export RECEIVER_BACKEND=sns_sqs
 #
 # Instructions for using S3 storage backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1470955567/How+to+test+storing+data+in+S3+locally
 # Instructions for using SES email backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/1467973894/How+to+test+sending+email+locally
+# Instructions for using SNS&SQS backend here: https://dp3.atlassian.net/wiki/spaces/MT/pages/2793242625/How+to+test+notifications+receiver+locally
 #
 # The default and equivalent to not being set is:
 #
 #   export STORAGE_BACKEND=local
 #   export EMAIL_BACKEND=local
+#   export RECEIVER_BACKEND=local
 #
 # Setting region and profile conditionally while we migrate from com to govcloud.
 if [ "$STORAGE_BACKEND" == "s3" ]; then
@@ -274,6 +277,13 @@ export AWS_S3_KEY_NAMESPACE=$USER
 export AWS_SES_DOMAIN="devlocal.dp3.us"
 export AWS_SES_REGION="us-gov-west-1"
 
+if [ "$RECEIVER_BACKEND" == "sns_sqs" ]; then
+  export SNS_TAGS_UPDATED_TOPIC="app_s3_tag_events"
+  export SNS_REGION="us-gov-west-1"
+# cleanup flag false by default, only used at server startup to wipe receiver artifacts from previous runs
+# export RECEIVER_CLEANUP_ON_START=false
+fi
+
 # To use s3 links aws-bucketname/xx/user/ for local builds,
 # you'll need to add the following to your .envrc.local:
 #
@@ -460,4 +470,4 @@ then
 fi
 
 # Check that all required environment variables are set
-check_required_variables
+check_required_variables

.gitlab-ci.yml

@@ -222,9 +222,9 @@ stages:
       export MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/migrations_manifest.txt'
       export DML_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/dml_migrations_manifest.txt'
       export DDL_TYPES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
-      export DDL_TABLES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
-      export DDL_VIEWS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
-      export DDL_FUNCTIONS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_types_manifest.txt'
+      export DDL_TABLES_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_tables_manifest.txt'
+      export DDL_VIEWS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_views_manifest.txt'
+      export DDL_FUNCTIONS_MIGRATION_MANIFEST='/builds/milmove/mymove/migrations/app/ddl_functions_manifest.txt'
       export MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/schema;file:///builds/milmove/mymove/migrations/app/secure'
       export DDL_TYPES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_types'
       export DDL_TABLES_MIGRATION_PATH='file:///builds/milmove/mymove/migrations/app/ddl_migrations/ddl_tables'
@@ -1457,6 +1457,8 @@ deploy_tasks_dp3:
     - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
     - echo "Deploying payment reminder email task service"
     - ./scripts/ecs-deploy-task-container send-payment-reminder "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
+    - echo "Deploying process TPPS task service"
+    - ./scripts/ecs-deploy-task-container process-tpps "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
   after_script:
     - *announce_failure
   rules:
@@ -1723,6 +1725,8 @@ deploy_tasks_stg:
     - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
     - echo "Deploying payment reminder email task service"
     - ./scripts/ecs-deploy-task-container send-payment-reminder "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
+    - echo "Deploying process TPPS task service"
+    - ./scripts/ecs-deploy-task-container process-tpps "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
   after_script:
     - *announce_failure
   rules:
@@ -2008,6 +2012,8 @@ deploy_tasks_prd:
     - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
     - echo "Deploying payment reminder email task service"
     - ./scripts/ecs-deploy-task-container send-payment-reminder "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
+    - echo "Deploying process TPPS task service"
+    - ./scripts/ecs-deploy-task-container process-tpps "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}"
   after_script:
     - *announce_failure
   rules:
@@ -2106,4 +2112,4 @@ deploy_app_prd:
   after_script:
     - *announce_failure
   rules:
-    - *check_main
+    - *check_main

Dockerfile.tasks

@@ -16,4 +16,8 @@ COPY bin/rds-ca-rsa4096-g1.pem /bin/rds-ca-rsa4096-g1.pem
 COPY bin/rds-ca-2019-root.pem /bin/rds-ca-2019-root.pem
 COPY bin/milmove-tasks /bin/milmove-tasks
 
+# Mount mutable tmp for process-tpps
+# hadolint ignore=DL3007
+VOLUME ["/tmp"]
+
 WORKDIR /bin

Dockerfile.tasks_dp3

@@ -15,4 +15,8 @@ COPY bin/rds-ca-rsa4096-g1.pem /bin/rds-ca-rsa4096-g1.pem
 
 COPY bin/milmove-tasks /bin/milmove-tasks
 
+# Mount mutable tmp for process-tpps
+# hadolint ignore=DL3007
+VOLUME ["/tmp"]
+
 WORKDIR /bin

Makefile

@@ -831,6 +831,22 @@ tasks_process_edis: tasks_build_linux_docker ## Run process-edis from inside doc
 		$(TASKS_DOCKER_CONTAINER):latest \
 		milmove-tasks process-edis
 
+.PHONY: tasks_process_tpps
+tasks_process_tpps: tasks_build_linux_docker ## Run process-tpps from inside docker container
+	@echo "Processing TPPS files with docker command..."
+	DB_NAME=$(DB_NAME_DEV) DB_DOCKER_CONTAINER=$(DB_DOCKER_CONTAINER_DEV) scripts/wait-for-db-docker
+	docker run \
+		-t \
+		-e DB_HOST="database" \
+		-e DB_NAME \
+		-e DB_PORT \
+		-e DB_USER \
+		-e DB_PASSWORD \
+		--link="$(DB_DOCKER_CONTAINER_DEV):database" \
+		--rm \
+		$(TASKS_DOCKER_CONTAINER):latest \
+		milmove-tasks process-tpps
+
 .PHONY: tasks_save_ghc_fuel_price_data
 tasks_save_ghc_fuel_price_data: tasks_build_linux_docker ## Run save-ghc-fuel-price-data from inside docker container
 	@echo "Saving the fuel price data to the ${DB_NAME_DEV} database with docker command..."

cmd/ecs-deploy/put_target.go

@@ -32,6 +32,7 @@ var names = []string{
 	"connect-to-gex-via-sftp",
 	"post-file-to-gex",
 	"process-edis",
+	"process-tpps",
 	"save-ghc-fuel-price-data",
 	"send-payment-reminder",
 }

cmd/ecs-deploy/task_def.go

@@ -59,6 +59,7 @@ var servicesToEntryPoints = map[string][]string{
 		fmt.Sprintf("%s connect-to-gex-via-sftp", binMilMoveTasks),
 		fmt.Sprintf("%s post-file-to-gex", binMilMoveTasks),
 		fmt.Sprintf("%s process-edis", binMilMoveTasks),
+		fmt.Sprintf("%s process-tpps", binMilMoveTasks),
 		fmt.Sprintf("%s save-ghc-fuel-price-data", binMilMoveTasks),
 		fmt.Sprintf("%s send-payment-reminder", binMilMoveTasks),
 	},

cmd/milmove-tasks/main.go

@@ -77,6 +77,16 @@ func main() {
 	initConnectToGEXViaSFTPFlags(processEDIsCommand.Flags())
 	root.AddCommand(processEDIsCommand)
 
+	processTPPSCommand := &cobra.Command{
+		Use:          "process-tpps",
+		Short:        "process TPPS files asynchrounously",
+		Long:         "process TPPS files asynchrounously",
+		RunE:         processTPPS,
+		SilenceUsage: true,
+	}
+	initProcessTPPSFlags(processTPPSCommand.Flags())
+	root.AddCommand(processTPPSCommand)
+
 	completionCommand := &cobra.Command{
 		Use:   "completion",
 		Short: "Generates bash completion scripts",

cmd/milmove-tasks/process_edis.go

@@ -244,16 +244,5 @@ func processEDIs(_ *cobra.Command, _ []string) error {
 		logger.Info("Successfully processed EDI824 application advice responses")
 	}
 
-	// Pending completion of B-20560, uncomment the code below
-	/*
-		// Process TPPS paid invoice report
-		pathTPPSPaidInvoiceReport := v.GetString(cli.SFTPTPPSPaidInvoiceReportPickupDirectory)
-		_, err = syncadaSFTPSession.FetchAndProcessSyncadaFiles(appCtx, pathTPPSPaidInvoiceReport, lastReadTime, invoice.NewTPPSPaidInvoiceReportProcessor())
-		if err != nil {
-			logger.Error("Error reading TPPS Paid Invoice Report application advice responses", zap.Error(err))
-		} else {
-			logger.Info("Successfully processed TPPS Paid Invoice Report application advice responses")
-		}
-	*/
 	return nil
 }