diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f0ad29c4c38..6b5f6f54fd1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1457,6 +1457,8 @@ deploy_tasks_dp3: - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" - echo "Deploying payment reminder email task service" - ./scripts/ecs-deploy-task-container send-payment-reminder "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" + - echo "Deploying process TPPS task service" + - ./scripts/ecs-deploy-task-container process-tpps "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" after_script: - *announce_failure rules: @@ -1723,6 +1725,8 @@ deploy_tasks_stg: - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" - echo "Deploying payment reminder email task service" - ./scripts/ecs-deploy-task-container send-payment-reminder "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" + - echo "Deploying process TPPS task service" + - ./scripts/ecs-deploy-task-container process-tpps "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" after_script: - *announce_failure rules: @@ -2008,6 +2012,8 @@ deploy_tasks_prd: - ./scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" - echo "Deploying payment reminder email task service" - ./scripts/ecs-deploy-task-container send-payment-reminder "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" + - echo "Deploying process TPPS task service" + - ./scripts/ecs-deploy-task-container process-tpps "${ECR_REPOSITORY_URI}/app-tasks@${ECR_DIGEST}" "${APP_ENVIRONMENT}" after_script: - *announce_failure rules: diff --git a/Dockerfile.tasks b/Dockerfile.tasks index 7546db7c4c8..e8da0d76a01 100644 --- a/Dockerfile.tasks +++ b/Dockerfile.tasks @@ -16,4 +16,8 @@ COPY bin/rds-ca-rsa4096-g1.pem /bin/rds-ca-rsa4096-g1.pem COPY bin/rds-ca-2019-root.pem /bin/rds-ca-2019-root.pem COPY bin/milmove-tasks /bin/milmove-tasks +# Mount mutable tmp for process-tpps +# hadolint ignore=DL3007 +VOLUME ["/tmp"] + WORKDIR /bin diff --git a/Dockerfile.tasks_dp3 b/Dockerfile.tasks_dp3 index 72f71bdb971..f4326ed162a 100644 --- a/Dockerfile.tasks_dp3 +++ b/Dockerfile.tasks_dp3 @@ -15,4 +15,8 @@ COPY bin/rds-ca-rsa4096-g1.pem /bin/rds-ca-rsa4096-g1.pem COPY bin/milmove-tasks /bin/milmove-tasks +# Mount mutable tmp for process-tpps +# hadolint ignore=DL3007 +VOLUME ["/tmp"] + WORKDIR /bin diff --git a/Makefile b/Makefile index e2f29bde50e..3b6b96ca3bc 100644 --- a/Makefile +++ b/Makefile @@ -831,6 +831,22 @@ tasks_process_edis: tasks_build_linux_docker ## Run process-edis from inside doc $(TASKS_DOCKER_CONTAINER):latest \ milmove-tasks process-edis +.PHONY: tasks_process_tpps +tasks_process_tpps: tasks_build_linux_docker ## Run process-tpps from inside docker container + @echo "Processing TPPS files with docker command..." + DB_NAME=$(DB_NAME_DEV) DB_DOCKER_CONTAINER=$(DB_DOCKER_CONTAINER_DEV) scripts/wait-for-db-docker + docker run \ + -t \ + -e DB_HOST="database" \ + -e DB_NAME \ + -e DB_PORT \ + -e DB_USER \ + -e DB_PASSWORD \ + --link="$(DB_DOCKER_CONTAINER_DEV):database" \ + --rm \ + $(TASKS_DOCKER_CONTAINER):latest \ + milmove-tasks process-tpps + .PHONY: tasks_save_ghc_fuel_price_data tasks_save_ghc_fuel_price_data: tasks_build_linux_docker ## Run save-ghc-fuel-price-data from inside docker container @echo "Saving the fuel price data to the ${DB_NAME_DEV} database with docker command..." diff --git a/cmd/ecs-deploy/put_target.go b/cmd/ecs-deploy/put_target.go index 099af5981ff..84bf759ed1f 100644 --- a/cmd/ecs-deploy/put_target.go +++ b/cmd/ecs-deploy/put_target.go @@ -32,6 +32,7 @@ var names = []string{ "connect-to-gex-via-sftp", "post-file-to-gex", "process-edis", + "process-tpps", "save-ghc-fuel-price-data", "send-payment-reminder", } diff --git a/cmd/ecs-deploy/task_def.go b/cmd/ecs-deploy/task_def.go index 82a1ae0b8c4..27ce20131b6 100644 --- a/cmd/ecs-deploy/task_def.go +++ b/cmd/ecs-deploy/task_def.go @@ -59,6 +59,7 @@ var servicesToEntryPoints = map[string][]string{ fmt.Sprintf("%s connect-to-gex-via-sftp", binMilMoveTasks), fmt.Sprintf("%s post-file-to-gex", binMilMoveTasks), fmt.Sprintf("%s process-edis", binMilMoveTasks), + fmt.Sprintf("%s process-tpps", binMilMoveTasks), fmt.Sprintf("%s save-ghc-fuel-price-data", binMilMoveTasks), fmt.Sprintf("%s send-payment-reminder", binMilMoveTasks), }, diff --git a/cmd/milmove-tasks/main.go b/cmd/milmove-tasks/main.go index dd4f689bd83..7953e4e04d6 100644 --- a/cmd/milmove-tasks/main.go +++ b/cmd/milmove-tasks/main.go @@ -77,6 +77,16 @@ func main() { initConnectToGEXViaSFTPFlags(processEDIsCommand.Flags()) root.AddCommand(processEDIsCommand) + processTPPSCommand := &cobra.Command{ + Use: "process-tpps", + Short: "process TPPS files asynchrounously", + Long: "process TPPS files asynchrounously", + RunE: processTPPS, + SilenceUsage: true, + } + initProcessTPPSFlags(processTPPSCommand.Flags()) + root.AddCommand(processTPPSCommand) + completionCommand := &cobra.Command{ Use: "completion", Short: "Generates bash completion scripts", diff --git a/cmd/milmove-tasks/process_edis.go b/cmd/milmove-tasks/process_edis.go index 31cf87d9c23..d026d3194da 100644 --- a/cmd/milmove-tasks/process_edis.go +++ b/cmd/milmove-tasks/process_edis.go @@ -244,16 +244,5 @@ func processEDIs(_ *cobra.Command, _ []string) error { logger.Info("Successfully processed EDI824 application advice responses") } - // Pending completion of B-20560, uncomment the code below - /* - // Process TPPS paid invoice report - pathTPPSPaidInvoiceReport := v.GetString(cli.SFTPTPPSPaidInvoiceReportPickupDirectory) - _, err = syncadaSFTPSession.FetchAndProcessSyncadaFiles(appCtx, pathTPPSPaidInvoiceReport, lastReadTime, invoice.NewTPPSPaidInvoiceReportProcessor()) - if err != nil { - logger.Error("Error reading TPPS Paid Invoice Report application advice responses", zap.Error(err)) - } else { - logger.Info("Successfully processed TPPS Paid Invoice Report application advice responses") - } - */ return nil } diff --git a/cmd/milmove-tasks/process_tpps.go b/cmd/milmove-tasks/process_tpps.go new file mode 100644 index 00000000000..2e68cb20f51 --- /dev/null +++ b/cmd/milmove-tasks/process_tpps.go @@ -0,0 +1,368 @@ +package main + +import ( + "context" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" + "go.uber.org/zap" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" + + "github.com/transcom/mymove/pkg/appcontext" + "github.com/transcom/mymove/pkg/cli" + "github.com/transcom/mymove/pkg/logging" + "github.com/transcom/mymove/pkg/services/invoice" +) + +// Call this from the command line with go run ./cmd/milmove-tasks process-tpps +func checkProcessTPPSConfig(v *viper.Viper, logger *zap.Logger) error { + + err := cli.CheckTPPSFlags(v) + if err != nil { + return err + } + + err = cli.CheckDatabase(v, logger) + if err != nil { + return err + } + + return nil +} + +// initProcessTPPSFlags initializes TPPS processing flags +func initProcessTPPSFlags(flag *pflag.FlagSet) { + + // TPPS Config + cli.InitTPPSFlags(flag) + + // DB Config + cli.InitDatabaseFlags(flag) + + // Logging Levels + cli.InitLoggingFlags(flag) + + // Don't sort flags + flag.SortFlags = false +} + +const ( + // AVStatusCLEAN string CLEAN + AVStatusCLEAN string = "CLEAN" + + // AVStatusUNKNOWN string UNKNOWN + // Placeholder for error when scanning, actual scan results from ClamAV are CLEAN or INFECTED + AVStatusUNKNOWN string = "UNKNOWN" + + // Default value for parameter store environment variable + tppsSFTPFileFormatNoCustomDate string = "MILMOVE-enYYYYMMDD.csv" +) + +type S3API interface { + GetObjectTagging(ctx context.Context, input *s3.GetObjectTaggingInput, optFns ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) + GetObject(ctx context.Context, input *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) +} + +var s3Client S3API + +func processTPPS(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + if flags.Lookup(cli.DbEnvFlag) == nil { + flag := pflag.CommandLine + cli.InitDatabaseFlags(flag) + } + err := cmd.ParseFlags(args) + if err != nil { + return fmt.Errorf("could not parse args: %w", err) + } + v := viper.New() + err = v.BindPFlags(flags) + if err != nil { + return fmt.Errorf("could not bind flags: %w", err) + } + v.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + v.AutomaticEnv() + + dbEnv := v.GetString(cli.DbEnvFlag) + + logger, _, err := logging.Config( + logging.WithEnvironment(dbEnv), + logging.WithLoggingLevel(v.GetString(cli.LoggingLevelFlag)), + logging.WithStacktraceLength(v.GetInt(cli.StacktraceLengthFlag)), + ) + if err != nil { + logger.Fatal("Failed to initialized Zap logging for process-tpps") + } + + zap.ReplaceGlobals(logger) + + startTime := time.Now() + defer func() { + elapsedTime := time.Since(startTime) + logger.Info(fmt.Sprintf("Duration of processTPPS task: %v", elapsedTime)) + }() + + err = checkProcessTPPSConfig(v, logger) + if err != nil { + logger.Fatal("invalid configuration", zap.Error(err)) + } + + // Create a connection to the DB + dbConnection, err := cli.InitDatabase(v, logger) + if err != nil { + logger.Fatal("Connecting to DB", zap.Error(err)) + } + + appCtx := appcontext.NewAppContext(dbConnection, logger, nil) + + tppsInvoiceProcessor := invoice.NewTPPSPaidInvoiceReportProcessor() + // Process TPPS paid invoice report + // The daily run of the task will process the previous day's payment file (matching the TPPS lambda schedule of working with the previous day's file). + // Example for running the task February 3, 2025 - we process February 2's payment file: MILMOVE-en20250202.csv + + // Should we need to process a filename from a specific day instead of the daily scheduled run: + // 1. Find the ProcessTPPSCustomDateFile in the AWS parameter store + // 2. Verify that it has default value of "MILMOVE-enYYYYMMDD.csv" + // 3. Fill in the YYYYMMDD with the desired date value of the file needing processed + // 4. Manually run the process-tpps task + // 5. *IMPORTANT*: Set the ProcessTPPSCustomDateFile value back to default value of "MILMOVE-enYYYYMMDD.csv" in the environment that it was modified in + + customFilePathToProcess := v.GetString(cli.ProcessTPPSCustomDateFile) + logger.Info(fmt.Sprintf("customFilePathToProcess: %s", customFilePathToProcess)) + + timezone, err := time.LoadLocation("America/Chicago") + if err != nil { + logger.Error("Error loading timezone for process-tpps ECS task", zap.Error(err)) + } + + tppsFilename := "" + if customFilePathToProcess == tppsSFTPFileFormatNoCustomDate || customFilePathToProcess == "" { + // Process the previous day's payment file + logger.Info("No custom filepath provided to process, processing payment file for yesterday's date.") + yesterday := time.Now().In(timezone).AddDate(0, 0, -1) + previousDay := yesterday.Format("20060102") + tppsFilename = fmt.Sprintf("MILMOVE-en%s.csv", previousDay) + previousDayFormatted := yesterday.Format("January 02, 2006") + logger.Info(fmt.Sprintf("Starting processing of TPPS data for %s: %s", previousDayFormatted, tppsFilename)) + } else { + // Process the custom date specified by the ProcessTPPSCustomDateFile AWS parameter store value + logger.Info("Custom filepath provided to process") + tppsFilename = customFilePathToProcess + logger.Info(fmt.Sprintf("Starting transfer of TPPS data file: %s", tppsFilename)) + } + + s3Region := v.GetString(cli.AWSS3RegionFlag) + if s3Client == nil { + cfg, errCfg := config.LoadDefaultConfig(context.Background(), + config.WithRegion(s3Region), + ) + if errCfg != nil { + logger.Error("error loading AWS config", zap.Error(errCfg)) + } + s3Client = s3.NewFromConfig(cfg) + } + + tppsS3Bucket := v.GetString(cli.TPPSS3Bucket) + tppsS3Folder := v.GetString(cli.TPPSS3Folder) + s3Key := tppsS3Folder + tppsFilename + + avStatus, s3ObjectTags, err := getS3ObjectTags(s3Client, tppsS3Bucket, s3Key) + if err != nil { + logger.Error("Failed to get S3 object tags", zap.Error(err)) + return fmt.Errorf("failed to get S3 object tags: %w", err) + } + + if avStatus == AVStatusCLEAN { + logger.Info(fmt.Sprintf("av-status is CLEAN for TPPS file: %s", tppsFilename)) + + // get the S3 object, download file to /tmp dir for processing if clean + localFilePath, err := downloadS3File(logger, s3Client, tppsS3Bucket, s3Key) + if err != nil { + logger.Error("Error with getting the S3 object data via GetObject", zap.Error(err)) + } + + err = tppsInvoiceProcessor.ProcessFile(appCtx, localFilePath, "") + + if err != nil { + logger.Error("Error processing TPPS Paid Invoice Report", zap.Error(err)) + } else { + logger.Info("Successfully processed TPPS Paid Invoice Report") + } + } else { + logger.Warn("Skipping unclean file", + zap.String("bucket", tppsS3Bucket), + zap.String("key", s3Key), + zap.Any("tags", s3ObjectTags)) + logger.Info("avStatus is not CLEAN, not attempting file download") + return nil + } + + return nil +} + +func getS3ObjectTags(s3Client S3API, bucket, key string) (string, map[string]string, error) { + tagResp, err := s3Client.GetObjectTagging(context.Background(), + &s3.GetObjectTaggingInput{ + Bucket: &bucket, + Key: &key, + }) + if err != nil { + return AVStatusUNKNOWN, nil, err + } + + tags := make(map[string]string) + avStatus := AVStatusUNKNOWN + + for _, tag := range tagResp.TagSet { + tags[*tag.Key] = *tag.Value + if *tag.Key == "av-status" { + avStatus = *tag.Value + } + } + + return avStatus, tags, nil +} + +func downloadS3File(logger *zap.Logger, s3Client S3API, bucket, key string) (string, error) { + response, err := s3Client.GetObject(context.Background(), + &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }) + + if err != nil { + logger.Error("Failed to get S3 object", + zap.String("bucket", bucket), + zap.String("key", key), + zap.Error(err)) + return "", err + } + defer response.Body.Close() + + // create a temp file in /tmp directory to store the CSV from the S3 bucket + // the /tmp directory will only exist for the duration of the task, so no cleanup is required + tempDir := os.TempDir() + if !isDirMutable(tempDir) { + return "", fmt.Errorf("tmp directory (%s) is not mutable, cannot write /tmp file for TPPS processing", tempDir) + } + + localFilePath := filepath.Join(tempDir, filepath.Base(key)) + + file, err := os.Create(localFilePath) + if err != nil { + logger.Error("Failed to create tmp file", zap.Error(err)) + return "", err + } + defer file.Close() + + _, err = io.Copy(file, response.Body) + if err != nil { + logger.Error("Failed to write S3 object to tmp file", zap.Error(err)) + return "", err + } + + _, err = os.ReadFile(localFilePath) + if err != nil { + logger.Error("Failed to read tmp file contents", zap.Error(err)) + return "", err + } + + logger.Info(fmt.Sprintf("Successfully wrote S3 file contents to local file: %s", localFilePath)) + + logFileContents(logger, localFilePath) + + return localFilePath, nil +} + +// convert to UTF-8 encoding +func convertToUTF8(data []byte) string { + if len(data) >= 2 { + if data[0] == 0xFF && data[1] == 0xFE { // UTF-16 LE + decoder := unicode.UTF16(unicode.LittleEndian, unicode.ExpectBOM).NewDecoder() + utf8Bytes, _, _ := transform.Bytes(decoder, data) + return string(utf8Bytes) + } else if data[0] == 0xFE && data[1] == 0xFF { // UTF-16 BE + decoder := unicode.UTF16(unicode.BigEndian, unicode.ExpectBOM).NewDecoder() + utf8Bytes, _, _ := transform.Bytes(decoder, data) + return string(utf8Bytes) + } + } + return string(data) +} + +// Identifies if a filepath directory is mutable +// This is needed in to write contents of S3 stream to +// local file so that we can open it with os.Open() in the parser +func isDirMutable(path string) bool { + testFile := filepath.Join(path, "tmp") + file, err := os.Create(testFile) + if err != nil { + log.Printf("isDirMutable: failed for %s: %v\n", path, err) + return false + } + file.Close() + os.Remove(testFile) // Cleanup the test file, it is mutable here + return true +} + +func logFileContents(logger *zap.Logger, filePath string) { + stat, err := os.Stat(filePath) + + if err != nil { + logger.Error("File does not exist or cannot be accessed", zap.String("filePath", filePath), zap.Error(err)) + return + } + + if stat.Size() == 0 { + logger.Warn("File is empty", zap.String("filePath", filePath)) + return + } + + file, err := os.Open(filePath) + if err != nil { + logger.Error("Failed to open file for logging", zap.String("filePath", filePath), zap.Error(err)) + return + } + defer file.Close() + + content, err := io.ReadAll(file) + if err != nil { + logger.Error("Failed to read file contents", zap.String("filePath", filePath), zap.Error(err)) + return + } + + const maxPreviewSize = 5000 + utf8Content := convertToUTF8(content) + + cleanedContent := cleanLogOutput(utf8Content) + + preview := cleanedContent + if len(cleanedContent) > maxPreviewSize { + preview = cleanedContent[:maxPreviewSize] + "..." + } + + logger.Info("File contents preview:", + zap.String("filePath", filePath), + zap.Int64("fileSize", stat.Size()), + zap.String("content-preview", preview), + ) +} + +func cleanLogOutput(input string) string { + cleaned := strings.ReplaceAll(input, "\t", ", ") + cleaned = strings.TrimSpace(cleaned) + cleaned = strings.Join(strings.Fields(cleaned), " ") + + return cleaned +} diff --git a/cmd/milmove-tasks/process_tpps_test.go b/cmd/milmove-tasks/process_tpps_test.go new file mode 100644 index 00000000000..1977353db6b --- /dev/null +++ b/cmd/milmove-tasks/process_tpps_test.go @@ -0,0 +1,258 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + "github.com/transcom/mymove/pkg/appcontext" + "github.com/transcom/mymove/pkg/cli" +) + +type MockTPPSPaidInvoiceReportProcessor struct { + mock.Mock +} + +func (m *MockTPPSPaidInvoiceReportProcessor) ProcessFile(appCtx appcontext.AppContext, syncadaPath string, text string) error { + args := m.Called(appCtx, syncadaPath, text) + return args.Error(0) +} + +type MockS3Client struct { + mock.Mock +} + +var globalFlagSet = func() *pflag.FlagSet { + fs := pflag.NewFlagSet("test", pflag.ContinueOnError) + cli.InitDatabaseFlags(fs) + return fs +}() + +func setupTestCommand() *cobra.Command { + mockCmd := &cobra.Command{} + mockCmd.Flags().AddFlagSet(globalFlagSet) + mockCmd.Flags().String(cli.ProcessTPPSCustomDateFile, "", "Custom TPPS file date") + mockCmd.Flags().String(cli.TPPSS3Bucket, "", "S3 bucket") + mockCmd.Flags().String(cli.TPPSS3Folder, "", "S3 folder") + return mockCmd +} + +func (m *MockS3Client) GetObjectTagging(ctx context.Context, input *s3.GetObjectTaggingInput, opts ...func(*s3.Options)) (*s3.GetObjectTaggingOutput, error) { + args := m.Called(ctx, input) + return args.Get(0).(*s3.GetObjectTaggingOutput), args.Error(1) +} + +func (m *MockS3Client) GetObject(ctx context.Context, input *s3.GetObjectInput, opts ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + args := m.Called(ctx, input) + return args.Get(0).(*s3.GetObjectOutput), args.Error(1) +} + +func runProcessTPPSWithMockS3(cmd *cobra.Command, args []string, mockS3 S3API) error { + originalS3Client := s3Client + defer func() { s3Client = originalS3Client }() + s3Client = mockS3 + return processTPPS(cmd, args) +} + +func TestMain(m *testing.M) { + // make sure global flag set is fresh before running tests + pflag.CommandLine = pflag.NewFlagSet(os.Args[0], pflag.ExitOnError) + os.Exit(m.Run()) +} + +func TestInitProcessTPPSFlags(t *testing.T) { + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + initProcessTPPSFlags(flagSet) + + dbFlag := flagSet.Lookup(cli.DbEnvFlag) + assert.NotNil(t, dbFlag, "Expected DbEnvFlag to be initialized") + + logFlag := flagSet.Lookup(cli.LoggingLevelFlag) + assert.NotNil(t, logFlag, "Expected LoggingLevelFlag to be initialized") + + assert.False(t, flagSet.SortFlags, "Expected flag sorting to be disabled") +} + +func TestProcessTPPSSuccess(t *testing.T) { + mockCmd := setupTestCommand() + + args := []string{ + "--process_tpps_custom_date_file=MILMOVE-en20250210.csv", + "--tpps_s3_bucket=test-bucket", + "--tpps_s3_folder=test-folder", + } + + err := mockCmd.ParseFlags(args) + assert.NoError(t, err) + + mockS3 := new(MockS3Client) + mockS3.On("GetObjectTagging", mock.Anything, mock.Anything). + Return(&s3.GetObjectTaggingOutput{ + TagSet: []types.Tag{ + {Key: aws.String("av-status"), Value: aws.String(AVStatusCLEAN)}, + }, + }, nil).Once() + + mockS3.On("GetObject", mock.Anything, mock.Anything). + Return(&s3.GetObjectOutput{Body: io.NopCloser(strings.NewReader("test-data"))}, nil).Once() + + err = runProcessTPPSWithMockS3(mockCmd, args, mockS3) + assert.NoError(t, err) + mockS3.AssertExpectations(t) +} + +func TestProcessTPPSS3Failure(t *testing.T) { + mockCmd := setupTestCommand() + + args := []string{ + "--tpps_s3_bucket=test-bucket", + "--tpps_s3_folder=test-folder", + "--process_tpps_custom_date_file=MILMOVE-en20250212.csv", + } + + err := mockCmd.ParseFlags(args) + assert.NoError(t, err) + + mockS3 := new(MockS3Client) + mockS3.On("GetObjectTagging", mock.Anything, mock.Anything). + Return(&s3.GetObjectTaggingOutput{}, fmt.Errorf("S3 error")).Once() + + err = runProcessTPPSWithMockS3(mockCmd, args, mockS3) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to get S3 object tags") + mockS3.AssertExpectations(t) +} + +func TestConvertToUTF8(t *testing.T) { + utf8Data := []byte("Invoice") + assert.Equal(t, "Invoice", convertToUTF8(utf8Data)) + + utf16LEData := []byte{0xFF, 0xFE, 'I', 0, 'n', 0, 'v', 0, 'o', 0, 'i', 0, 'c', 0, 'e', 0} + assert.Equal(t, "Invoice", convertToUTF8(utf16LEData)) + + utf16BEData := []byte{0xFE, 0xFF, 0, 'I', 0, 'n', 0, 'v', 0, 'o', 0, 'i', 0, 'c', 0, 'e'} + assert.Equal(t, "Invoice", convertToUTF8(utf16BEData)) + + emptyData := []byte{} + assert.Equal(t, "", convertToUTF8(emptyData)) +} + +func TestIsDirMutable(t *testing.T) { + // using the OS temp dir, should be mutable + assert.True(t, isDirMutable("/tmp")) + + // non-writable paths should not be mutable + assert.False(t, isDirMutable("/root")) +} + +func captureLogs(fn func(logger *zap.Logger)) string { + var logs strings.Builder + core := zapcore.NewCore( + zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig()), + zapcore.AddSync(&logs), + zapcore.DebugLevel, + ) + logger := zap.New(core) + + fn(logger) + return logs.String() +} + +func TestLogFileContentsFailedToOpenFile(t *testing.T) { + tempFile := filepath.Join(os.TempDir(), "write-only-file.txt") + // 0000 = no permissions + err := os.WriteFile(tempFile, []byte("test"), 0000) + assert.NoError(t, err) + defer os.Remove(tempFile) + + logOutput := captureLogs(func(logger *zap.Logger) { + logFileContents(logger, tempFile) + }) + + assert.Contains(t, logOutput, "Failed to open file for logging") +} + +func TestLogFileContentsFailedToReadFileContents(t *testing.T) { + tempDir := filepath.Join(os.TempDir(), "unopenable-dir") + err := os.Mkdir(tempDir, 0755) + assert.NoError(t, err) + defer os.Remove(tempDir) + + logOutput := captureLogs(func(logger *zap.Logger) { + logFileContents(logger, tempDir) + }) + + assert.Contains(t, logOutput, "Failed to read file contents") +} + +func TestLogFileContentsFileDoesNotExistOrCantBeAccessed(t *testing.T) { + logOutput := captureLogs(func(logger *zap.Logger) { + logFileContents(logger, "nonexistent-file.txt") + }) + + assert.Contains(t, logOutput, "File does not exist or cannot be accessed") +} + +func TestLogFileContentsEmptyFile(t *testing.T) { + tempFile := filepath.Join(os.TempDir(), "empty-file.txt") + err := os.WriteFile(tempFile, []byte(""), 0600) + assert.NoError(t, err) + defer os.Remove(tempFile) + + logOutput := captureLogs(func(logger *zap.Logger) { + logFileContents(logger, tempFile) + }) + + assert.Contains(t, logOutput, "File is empty") +} + +func TestLogFileContentsShortFilePreview(t *testing.T) { + tempFile := filepath.Join(os.TempDir(), "test-file.txt") + content := "Test test test short file" + err := os.WriteFile(tempFile, []byte(content), 0600) + assert.NoError(t, err) + defer os.Remove(tempFile) + + logOutput := captureLogs(func(logger *zap.Logger) { + logFileContents(logger, tempFile) + }) + + fmt.Println("Captured log output:", logOutput) + rawContent, _ := os.ReadFile(tempFile) + fmt.Println("Actual file content:", string(rawContent)) + + assert.Contains(t, logOutput, "File contents preview:") + assert.Contains(t, logOutput, content) +} + +func TestLogFileContentsLongFilePreview(t *testing.T) { + tempFile := filepath.Join(os.TempDir(), "large-file.txt") + // larger than maxPreviewSize of 5000 bytes + longContent := strings.Repeat("M", 6000) + err := os.WriteFile(tempFile, []byte(longContent), 0600) + assert.NoError(t, err) + defer os.Remove(tempFile) + + logOutput := captureLogs(func(logger *zap.Logger) { + logFileContents(logger, tempFile) + }) + + assert.Contains(t, logOutput, "File contents preview:") + assert.Contains(t, logOutput, "MMMMM") + assert.Contains(t, logOutput, "...") +} diff --git a/config/env/demo.process-tpps.env b/config/env/demo.process-tpps.env new file mode 100644 index 00000000000..ebff88ba9cd --- /dev/null +++ b/config/env/demo.process-tpps.env @@ -0,0 +1,8 @@ +DB_IAM=true +DB_NAME=app +DB_PORT=5432 +DB_RETRY_INTERVAL=5s +DB_SSL_MODE=verify-full +DB_SSL_ROOT_CERT=/bin/rds-ca-rsa4096-g1.pem +DB_USER=crud +DOD_CA_PACKAGE=/config/tls/api.exp.dp3.us.chain.der.p7b \ No newline at end of file diff --git a/config/env/exp.process-tpps.env b/config/env/exp.process-tpps.env new file mode 100644 index 00000000000..bfd80842ae9 --- /dev/null +++ b/config/env/exp.process-tpps.env @@ -0,0 +1,9 @@ +AWS_S3_KEY_NAMESPACE=app +DB_IAM=true +DB_NAME=app +DB_PORT=5432 +DB_RETRY_INTERVAL=5s +DB_SSL_MODE=verify-full +DB_SSL_ROOT_CERT=/bin/rds-ca-rsa4096-g1.pem +DB_USER=ecs_user +DOD_CA_PACKAGE=/config/tls/api.exp.dp3.us.chain.der.p7b diff --git a/config/env/loadtest.process-tpps.env b/config/env/loadtest.process-tpps.env new file mode 100644 index 00000000000..bfd80842ae9 --- /dev/null +++ b/config/env/loadtest.process-tpps.env @@ -0,0 +1,9 @@ +AWS_S3_KEY_NAMESPACE=app +DB_IAM=true +DB_NAME=app +DB_PORT=5432 +DB_RETRY_INTERVAL=5s +DB_SSL_MODE=verify-full +DB_SSL_ROOT_CERT=/bin/rds-ca-rsa4096-g1.pem +DB_USER=ecs_user +DOD_CA_PACKAGE=/config/tls/api.exp.dp3.us.chain.der.p7b diff --git a/config/env/prd.process-tpps.env b/config/env/prd.process-tpps.env new file mode 100644 index 00000000000..527bb690e04 --- /dev/null +++ b/config/env/prd.process-tpps.env @@ -0,0 +1,8 @@ +DB_IAM=true +DB_NAME=app +DB_PORT=5432 +DB_RETRY_INTERVAL=5s +DB_SSL_MODE=verify-full +DB_SSL_ROOT_CERT=/bin/rds-ca-rsa4096-g1.pem +DB_USER=crud +DOD_CA_PACKAGE=/config/tls/milmove-cert-bundle.p7b \ No newline at end of file diff --git a/config/env/stg.process-tpps.env b/config/env/stg.process-tpps.env new file mode 100644 index 00000000000..527bb690e04 --- /dev/null +++ b/config/env/stg.process-tpps.env @@ -0,0 +1,8 @@ +DB_IAM=true +DB_NAME=app +DB_PORT=5432 +DB_RETRY_INTERVAL=5s +DB_SSL_MODE=verify-full +DB_SSL_ROOT_CERT=/bin/rds-ca-rsa4096-g1.pem +DB_USER=crud +DOD_CA_PACKAGE=/config/tls/milmove-cert-bundle.p7b \ No newline at end of file diff --git a/migrations/app/migrations_manifest.txt b/migrations/app/migrations_manifest.txt index 1ab3f2c976e..f71dd548355 100644 --- a/migrations/app/migrations_manifest.txt +++ b/migrations/app/migrations_manifest.txt @@ -1097,5 +1097,6 @@ 20250206173204_add_hawaii_data.up.sql 20250207153450_add_fetch_documents_func.up.sql 20250210175754_B22451_update_dest_queue_to_consider_sit_extensions.up.sql +20250213214427_drop_received_by_gex_payment_request_status_type.up.sql 20250213151815_fix_spacing_fetch_documents.up.sql # nothing should be added below this line this file is archived and only needed for rebuilding db Locally to be run prior to new migrations process to keep the current state diff --git a/migrations/app/schema/20250213214427_drop_received_by_gex_payment_request_status_type.up.sql b/migrations/app/schema/20250213214427_drop_received_by_gex_payment_request_status_type.up.sql new file mode 100644 index 00000000000..e6fa11a91f3 --- /dev/null +++ b/migrations/app/schema/20250213214427_drop_received_by_gex_payment_request_status_type.up.sql @@ -0,0 +1,34 @@ +-- This migration removes unused payment request status type of RECEIVED_BY_GEX +-- all previous payment requests using type were updated to TPPS_RECEIVED in +-- migrations/app/schema/20240725190050_update_payment_request_status_tpps_received.up.sql + +-- update again in case new payment requests have used this status +UPDATE payment_requests SET status = 'TPPS_RECEIVED' where status = 'RECEIVED_BY_GEX'; + +--- rename existing enum +ALTER TYPE payment_request_status RENAME TO payment_request_status_temp; + +-- create a new enum with both old and new statuses - both old and new statuses must exist in the enum to do the update setting old to new +CREATE TYPE payment_request_status AS ENUM( + 'PENDING', + 'REVIEWED', + 'SENT_TO_GEX', + 'PAID', + 'REVIEWED_AND_ALL_SERVICE_ITEMS_REJECTED', + 'EDI_ERROR', + 'DEPRECATED', + 'TPPS_RECEIVED' + ); + +alter table payment_requests alter column status drop default; +alter table payment_requests alter column status drop not null; + +-- alter the payment_requests status column to use the new enum +ALTER TABLE payment_requests ALTER COLUMN status TYPE payment_request_status USING status::text::payment_request_status; + +-- get rid of the temp type +DROP TYPE payment_request_status_temp; + +ALTER TABLE payment_requests +ALTER COLUMN status SET DEFAULT 'PENDING', +ALTER COLUMN status SET NOT NULL; \ No newline at end of file diff --git a/pkg/cli/gex_sftp.go b/pkg/cli/gex_sftp.go index 00239275c52..576391250a0 100644 --- a/pkg/cli/gex_sftp.go +++ b/pkg/cli/gex_sftp.go @@ -41,15 +41,6 @@ const ( GEXSFTP824PickupDirectory string = "gex-sftp-824-pickup-directory" ) -// Pending completion of B-20560, uncomment the code below -/* -// Set of flags used for SFTPTPPSPaid -const ( - // SFTPTPPSPaidInvoiceReportPickupDirectory is the ENV var for the directory where TPPS delivers the TPPS paid invoice report - SFTPTPPSPaidInvoiceReportPickupDirectory string = "pending" // pending completion of B-20560 -) -*/ - // InitGEXSFTPFlags initializes GEX SFTP command line flags func InitGEXSFTPFlags(flag *pflag.FlagSet) { flag.Int(GEXSFTPPortFlag, 22, "GEX SFTP Port") @@ -60,7 +51,6 @@ func InitGEXSFTPFlags(flag *pflag.FlagSet) { flag.String(GEXSFTPHostKeyFlag, "", "GEX SFTP Host Key") flag.String(GEXSFTP997PickupDirectory, "", "GEX 997 SFTP Pickup Directory") flag.String(GEXSFTP824PickupDirectory, "", "GEX 834 SFTP Pickup Directory") - // flag.String(SFTPTPPSPaidInvoiceReportPickupDirectory, "", "TPPS Paid Invoice SFTP Pickup Directory") // pending completion of B-20560 } // CheckGEXSFTP validates GEX SFTP command line flags diff --git a/pkg/cli/tpps_processing.go b/pkg/cli/tpps_processing.go new file mode 100644 index 00000000000..3599d5f9952 --- /dev/null +++ b/pkg/cli/tpps_processing.go @@ -0,0 +1,44 @@ +package cli + +import ( + "fmt" + + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +const ( + // ProcessTPPSCustomDateFile is the env var for the date of a file that can be customized if we want to process a payment file other than the daily run of the task + ProcessTPPSCustomDateFile string = "process_tpps_custom_date_file" + // TPPSS3Bucket is the env var for the S3 bucket for TPPS payment files that we import from US bank + TPPSS3Bucket string = "tpps_s3_bucket" + // TPPSS3Folder is the env var for the S3 folder inside the tpps_s3_bucket for TPPS payment files that we import from US bank + TPPSS3Folder string = "tpps_s3_folder" +) + +// InitTPPSFlags initializes TPPS SFTP command line flags +func InitTPPSFlags(flag *pflag.FlagSet) { + flag.String(ProcessTPPSCustomDateFile, "", "Custom date for TPPS filename to process, format of MILMOVE-enYYYYMMDD.csv") + flag.String(TPPSS3Bucket, "", "S3 bucket for TPPS payment files that we import from US bank") + flag.String(TPPSS3Folder, "", "S3 folder inside the TPPSS3Bucket for TPPS payment files that we import from US bank") +} + +// CheckTPPSFlags validates the TPPS processing command line flags +func CheckTPPSFlags(v *viper.Viper) error { + ProcessTPPSCustomDateFile := v.GetString(ProcessTPPSCustomDateFile) + if ProcessTPPSCustomDateFile == "" { + return fmt.Errorf("invalid ProcessTPPSCustomDateFile %s, expecting the format of MILMOVE-enYYYYMMDD.csv", ProcessTPPSCustomDateFile) + } + + TPPSS3Bucket := v.GetString(TPPSS3Bucket) + if TPPSS3Bucket == "" { + return fmt.Errorf("no value for TPPSS3Bucket found") + } + + TPPSS3Folder := v.GetString(TPPSS3Folder) + if TPPSS3Folder == "" { + return fmt.Errorf("no value for TPPSS3Folder found") + } + + return nil +} diff --git a/pkg/cli/tpps_processing_test.go b/pkg/cli/tpps_processing_test.go new file mode 100644 index 00000000000..4baa042ebf4 --- /dev/null +++ b/pkg/cli/tpps_processing_test.go @@ -0,0 +1,63 @@ +package cli + +import ( + "testing" + + "github.com/spf13/pflag" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" +) + +func TestInitTPPSFlags(t *testing.T) { + flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) + InitTPPSFlags(flagSet) + + processTPPSCustomDateFile, _ := flagSet.GetString(ProcessTPPSCustomDateFile) + assert.Equal(t, "", processTPPSCustomDateFile, "Expected ProcessTPPSCustomDateFile to have an empty default value") + + tppsS3Bucket, _ := flagSet.GetString(TPPSS3Bucket) + assert.Equal(t, "", tppsS3Bucket, "Expected TPPSS3Bucket to have an empty default value") + + tppsS3Folder, _ := flagSet.GetString(TPPSS3Folder) + assert.Equal(t, "", tppsS3Folder, "Expected TPPSS3Folder to have an empty default value") +} + +func TestCheckTPPSFlagsValidInput(t *testing.T) { + v := viper.New() + v.Set(ProcessTPPSCustomDateFile, "MILMOVE-en20250210.csv") + v.Set(TPPSS3Bucket, "test-bucket") + v.Set(TPPSS3Folder, "test-folder") + + err := CheckTPPSFlags(v) + assert.NoError(t, err) +} + +func TestCheckTPPSFlagsMissingProcessTPPSCustomDateFile(t *testing.T) { + v := viper.New() + v.Set(TPPSS3Bucket, "test-bucket") + v.Set(TPPSS3Folder, "test-folder") + + err := CheckTPPSFlags(v) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid ProcessTPPSCustomDateFile") +} + +func TestCheckTPPSFlagsMissingTPPSS3Bucket(t *testing.T) { + v := viper.New() + v.Set(ProcessTPPSCustomDateFile, "MILMOVE-en20250210.csv") + v.Set(TPPSS3Folder, "test-folder") + + err := CheckTPPSFlags(v) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no value for TPPSS3Bucket found") +} + +func TestCheckTPPSFlagsMissingTPPSS3Folder(t *testing.T) { + v := viper.New() + v.Set(ProcessTPPSCustomDateFile, "MILMOVE-en20250210.csv") + v.Set(TPPSS3Bucket, "test-bucket") + + err := CheckTPPSFlags(v) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no value for TPPSS3Folder found") +} diff --git a/pkg/edi/tpps_paid_invoice_report/parser.go b/pkg/edi/tpps_paid_invoice_report/parser.go index c4cb9d6ef77..47d4b162a38 100644 --- a/pkg/edi/tpps_paid_invoice_report/parser.go +++ b/pkg/edi/tpps_paid_invoice_report/parser.go @@ -1,14 +1,20 @@ package tppspaidinvoicereport import ( - "bufio" + "bytes" + "encoding/csv" "fmt" "io" "os" - "path/filepath" + "regexp" "strings" + "unicode/utf8" "github.com/pkg/errors" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/transform" + + "github.com/transcom/mymove/pkg/appcontext" ) func VerifyHeadersParsedCorrectly(parsedHeadersFromFile TPPSData) bool { @@ -43,118 +49,135 @@ func VerifyHeadersParsedCorrectly(parsedHeadersFromFile TPPSData) bool { return allHeadersWereProcessedCorrectly } -// ProcessTPPSReportEntryForOneRow takes one tab-delimited data row, cleans it, and parses it into a string representation of the TPPSData struct -func ParseTPPSReportEntryForOneRow(row []string, columnIndexes map[string]int, headerIndicesNeedDefined bool) (TPPSData, map[string]int, bool) { - tppsReportEntryForOnePaymentRequest := strings.Split(row[0], "\t") - var tppsData TPPSData - var processedTPPSReportEntryForOnePaymentRequest []string - var columnHeaderIndices map[string]int - - if len(tppsReportEntryForOnePaymentRequest) > 0 { - - for indexOfOneEntry := range tppsReportEntryForOnePaymentRequest { - var processedEntry string - if tppsReportEntryForOnePaymentRequest[indexOfOneEntry] != "" { - // Remove any NULL characters - entryWithoutNulls := strings.Split(tppsReportEntryForOnePaymentRequest[indexOfOneEntry], "\x00") - for indexCleanedUp := range entryWithoutNulls { - // Clean up extra characters - cleanedUpEntryString := strings.Split(entryWithoutNulls[indexCleanedUp], ("\xff\xfe")) - for index := range cleanedUpEntryString { - if cleanedUpEntryString[index] != "" { - processedEntry += cleanedUpEntryString[index] - } - } - } - } - processedEntry = strings.TrimSpace(processedEntry) - processedEntry = strings.TrimLeft(processedEntry, "�") - // After we have fully processed an entry and have built a string, store it - processedTPPSReportEntryForOnePaymentRequest = append(processedTPPSReportEntryForOnePaymentRequest, processedEntry) - } - if headerIndicesNeedDefined { - columnHeaderIndices = make(map[string]int) - for i, columnHeader := range processedTPPSReportEntryForOnePaymentRequest { - columnHeaderIndices[columnHeader] = i - } - // only need to define the column header indices once per read of a file, so set to false after first pass through - headerIndicesNeedDefined = false - } else { - columnHeaderIndices = columnIndexes - } - tppsData.InvoiceNumber = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Invoice Number From Invoice"]] - tppsData.TPPSCreatedDocumentDate = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Document Create Date"]] - tppsData.SellerPaidDate = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Seller Paid Date"]] - tppsData.InvoiceTotalCharges = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Invoice Total Charges"]] - tppsData.LineDescription = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Line Description"]] - tppsData.ProductDescription = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Product Description"]] - tppsData.LineBillingUnits = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Line Billing Units"]] - tppsData.LineUnitPrice = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Line Unit Price"]] - tppsData.LineNetCharge = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Line Net Charge"]] - tppsData.POTCN = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["PO/TCN"]] - tppsData.LineNumber = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Line Number"]] - tppsData.FirstNoteCode = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["First Note Code"]] - tppsData.FirstNoteCodeDescription = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["First Note Code Description"]] - tppsData.FirstNoteTo = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["First Note To"]] - tppsData.FirstNoteMessage = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["First Note Message"]] - tppsData.SecondNoteCode = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Second Note Code"]] - tppsData.SecondNoteCodeDescription = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Second Note Code Description"]] - tppsData.SecondNoteTo = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Second Note To"]] - tppsData.SecondNoteMessage = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Second Note Message"]] - tppsData.ThirdNoteCode = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Third Note Code"]] - tppsData.ThirdNoteCodeDescription = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Third Note Code Description"]] - tppsData.ThirdNoteTo = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Third Note To"]] - tppsData.ThirdNoteMessage = processedTPPSReportEntryForOnePaymentRequest[columnHeaderIndices["Third Note Message"]] - } - return tppsData, columnHeaderIndices, headerIndicesNeedDefined -} - // Parse takes in a TPPS paid invoice report file and parses it into an array of TPPSData structs -func (t *TPPSData) Parse(stringTPPSPaidInvoiceReportFilePath string, testTPPSInvoiceString string) ([]TPPSData, error) { +func (t *TPPSData) Parse(appCtx appcontext.AppContext, stringTPPSPaidInvoiceReportFilePath string) ([]TPPSData, error) { var tppsDataFile []TPPSData - var dataToParse io.Reader - if stringTPPSPaidInvoiceReportFilePath != "" { - csvFile, err := os.Open(filepath.Clean(stringTPPSPaidInvoiceReportFilePath)) + appCtx.Logger().Info(fmt.Sprintf("Parsing TPPS data file: %s", stringTPPSPaidInvoiceReportFilePath)) + csvFile, err := os.Open(stringTPPSPaidInvoiceReportFilePath) if err != nil { return nil, errors.Wrap(err, (fmt.Sprintf("Unable to read TPPS paid invoice report from path %s", stringTPPSPaidInvoiceReportFilePath))) } - dataToParse = csvFile - } else { - dataToParse = strings.NewReader(testTPPSInvoiceString) - } - endOfFile := false - headersAreCorrect := false - needToDefineColumnIndices := true - var headerColumnIndices map[string]int - - scanner := bufio.NewScanner(dataToParse) - for scanner.Scan() { - rowIsHeader := false - row := strings.Split(scanner.Text(), "\n") - // If we have reached a NULL or empty row at the end of the file, do not continue parsing - if row[0] == "\x00" || row[0] == "" { - endOfFile = true + defer csvFile.Close() + + rawData, err := io.ReadAll(csvFile) + if err != nil { + return nil, fmt.Errorf("error reading file: %w", err) + } + + decoder := unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder() + utf8Data, _, err := transform.Bytes(decoder, rawData) + if err != nil { + return nil, fmt.Errorf("error converting file encoding to UTF-8: %w", err) } - if row != nil && !endOfFile { - tppsReportEntryForOnePaymentRequest, columnIndicesFound, keepFindingColumnIndices := ParseTPPSReportEntryForOneRow(row, headerColumnIndices, needToDefineColumnIndices) - // For first data row of file (headers), find indices of the columns - // For the rest of the file, use those same indices to parse in the data - if needToDefineColumnIndices { - // Only want to define header column indices once per file read - headerColumnIndices = columnIndicesFound + utf8Data = cleanHeaders(utf8Data) + + reader := csv.NewReader(bytes.NewReader(utf8Data)) + reader.Comma = '\t' + reader.LazyQuotes = true + reader.FieldsPerRecord = -1 + + headers, err := reader.Read() + if err != nil { + return nil, fmt.Errorf("error reading CSV headers: %w", err) + } + + columnHeaderIndices := make(map[string]int) + for i, col := range headers { + headers[i] = cleanText(col) + columnHeaderIndices[col] = i + } + + headersAreCorrect := false + headersTPPSData := convertToTPPSDataStruct(headers, columnHeaderIndices) + headersAreCorrect = VerifyHeadersParsedCorrectly(headersTPPSData) + + for rowIndex := 0; ; rowIndex++ { + rowIsHeader := false + row, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + fmt.Println("Error reading row:", err) + continue + } + + if len(row) < len(columnHeaderIndices) { + fmt.Println("Skipping row due to incorrect column count:", row) + continue + } + + for colIndex, value := range row { + row[colIndex] = cleanText(value) } - needToDefineColumnIndices = keepFindingColumnIndices - if tppsReportEntryForOnePaymentRequest.InvoiceNumber == "Invoice Number From Invoice" { + + tppsDataRow := convertToTPPSDataStruct(row, columnHeaderIndices) + + if tppsDataRow.InvoiceNumber == "Invoice Number From Invoice" { rowIsHeader = true - headersAreCorrect = VerifyHeadersParsedCorrectly(tppsReportEntryForOnePaymentRequest) } if !rowIsHeader && headersAreCorrect { // No need to append the header row to result set - tppsDataFile = append(tppsDataFile, tppsReportEntryForOnePaymentRequest) + tppsDataFile = append(tppsDataFile, tppsDataRow) } } + } else { + return nil, fmt.Errorf("TPPS data file path is empty") } - return tppsDataFile, nil } + +func convertToTPPSDataStruct(row []string, columnHeaderIndices map[string]int) TPPSData { + tppsReportEntryForOnePaymentRequest := TPPSData{ + InvoiceNumber: row[columnHeaderIndices["Invoice Number From Invoice"]], + TPPSCreatedDocumentDate: row[columnHeaderIndices["Document Create Date"]], + SellerPaidDate: row[columnHeaderIndices["Seller Paid Date"]], + InvoiceTotalCharges: row[columnHeaderIndices["Invoice Total Charges"]], + LineDescription: row[columnHeaderIndices["Line Description"]], + ProductDescription: row[columnHeaderIndices["Product Description"]], + LineBillingUnits: row[columnHeaderIndices["Line Billing Units"]], + LineUnitPrice: row[columnHeaderIndices["Line Unit Price"]], + LineNetCharge: row[columnHeaderIndices["Line Net Charge"]], + POTCN: row[columnHeaderIndices["PO/TCN"]], + LineNumber: row[columnHeaderIndices["Line Number"]], + FirstNoteCode: row[columnHeaderIndices["First Note Code"]], + FirstNoteCodeDescription: row[columnHeaderIndices["First Note Code Description"]], + FirstNoteTo: row[columnHeaderIndices["First Note To"]], + FirstNoteMessage: row[columnHeaderIndices["First Note Message"]], + SecondNoteCode: row[columnHeaderIndices["Second Note Code"]], + SecondNoteCodeDescription: row[columnHeaderIndices["Second Note Code Description"]], + SecondNoteTo: row[columnHeaderIndices["Second Note To"]], + SecondNoteMessage: row[columnHeaderIndices["Second Note Message"]], + ThirdNoteCode: row[columnHeaderIndices["Third Note Code"]], + ThirdNoteCodeDescription: row[columnHeaderIndices["Third Note Code Description"]], + ThirdNoteTo: row[columnHeaderIndices["Third Note To"]], + ThirdNoteMessage: row[columnHeaderIndices["Third Note Message"]], + } + return tppsReportEntryForOnePaymentRequest +} + +func cleanHeaders(rawTPPSData []byte) []byte { + // Remove first three UTF-8 bytes (0xEF 0xBB 0xBF) + if len(rawTPPSData) > 3 && rawTPPSData[0] == 0xEF && rawTPPSData[1] == 0xBB && rawTPPSData[2] == 0xBF { + rawTPPSData = rawTPPSData[3:] + } + + // Remove leading non-UTF8 bytes + for i := 0; i < len(rawTPPSData); i++ { + if utf8.Valid(rawTPPSData[i:]) { + return rawTPPSData[i:] + } + } + + return rawTPPSData +} + +func cleanText(text string) string { + // Remove non-ASCII characters like the �� on the header row of every TPPS file + re := regexp.MustCompile(`[^\x20-\x7E]`) + cleaned := re.ReplaceAllString(text, "") + + // Trim any unexpected spaces around the text + return strings.TrimSpace(cleaned) +} diff --git a/pkg/edi/tpps_paid_invoice_report/parser_test.go b/pkg/edi/tpps_paid_invoice_report/parser_test.go index a36e28394af..30fb20ff369 100644 --- a/pkg/edi/tpps_paid_invoice_report/parser_test.go +++ b/pkg/edi/tpps_paid_invoice_report/parser_test.go @@ -9,32 +9,27 @@ import ( ) type TPPSPaidInvoiceSuite struct { - testingsuite.BaseTestSuite + *testingsuite.PopTestSuite } func TestTPPSPaidInvoiceSuite(t *testing.T) { - hs := &TPPSPaidInvoiceSuite{} + ts := &TPPSPaidInvoiceSuite{ + PopTestSuite: testingsuite.NewPopTestSuite(testingsuite.CurrentPackage(), + testingsuite.WithPerTestTransaction()), + } - suite.Run(t, hs) + suite.Run(t, ts) + ts.PopTestSuite.TearDown() } func (suite *TPPSPaidInvoiceSuite) TestParse() { - suite.Run("successfully parse simple TPPS Paid Invoice string", func() { - // This is a string representation of a test .csv file. Rows are new-line delimited, columns in each row are tab delimited, file ends in a empty row. - sampleTPPSPaidInvoiceString := `Invoice Number From Invoice Document Create Date Seller Paid Date Invoice Total Charges Line Description Product Description Line Billing Units Line Unit Price Line Net Charge PO/TCN Line Number First Note Code First Note Code Description First Note To First Note Message Second Note Code Second Note Code Description Second Note To Second Note Message Third Note Code Third Note Code Description Third Note To Third Note Message -1841-7267-3 2024-07-29 2024-07-30 1151.55 DDP DDP 3760 0.0077 28.95 1841-7267-826285fc 1 INT Notes to My Company - INT CARR HQ50066 -1841-7267-3 2024-07-29 2024-07-30 1151.55 FSC FSC 3760 0.0014 5.39 1841-7267-aeb3cfea 4 INT Notes to My Company - INT CARR HQ50066 -1841-7267-3 2024-07-29 2024-07-30 1151.55 DLH DLH 3760 0.2656 998.77 1841-7267-c8ea170b 2 INT Notes to My Company - INT CARR HQ50066 -1841-7267-3 2024-07-29 2024-07-30 1151.55 DUPK DUPK 3760 0.0315 118.44 1841-7267-265c16d7 3 INT Notes to My Company - INT CARR HQ50066 -9436-4123-3 2024-07-29 2024-07-30 125.25 DDP DDP 7500 0.0167 125.25 9436-4123-93761f93 1 INT Notes to My Company - INT CARR HQ50057 - -` - + suite.Run("successfully parse simple TPPS Paid Invoice file", func() { + testTPPSPaidInvoiceReportFilePath := "../../services/invoice/fixtures/tpps_paid_invoice_report_testfile.csv" tppsPaidInvoice := TPPSData{} - tppsEntries, err := tppsPaidInvoice.Parse("", sampleTPPSPaidInvoiceString) + tppsEntries, err := tppsPaidInvoice.Parse(suite.AppContextForTest(), testTPPSPaidInvoiceReportFilePath) suite.NoError(err, "Successful parse of TPPS Paid Invoice string") - suite.Equal(len(tppsEntries), 5) + suite.Equal(5, len(tppsEntries)) for tppsEntryIndex := range tppsEntries { if tppsEntryIndex == 0 { @@ -137,4 +132,29 @@ func (suite *TPPSPaidInvoiceSuite) TestParse() { } }) + suite.Run("successfully parse large TPPS Paid Invoice .csv file", func() { + testTPPSPaidInvoiceReportFilePath := "../../services/invoice/fixtures/tpps_paid_invoice_report_testfile_large_encoded.csv" + tppsPaidInvoice := TPPSData{} + tppsEntries, err := tppsPaidInvoice.Parse(suite.AppContextForTest(), testTPPSPaidInvoiceReportFilePath) + suite.NoError(err, "Successful parse of TPPS Paid Invoice string") + suite.Equal(842, len(tppsEntries)) + }) + + suite.Run("fails when TPPS data file path is empty", func() { + tppsPaidInvoice := TPPSData{} + tppsEntries, err := tppsPaidInvoice.Parse(suite.AppContextForTest(), "") + + suite.Nil(tppsEntries) + suite.Error(err) + suite.Contains(err.Error(), "TPPS data file path is empty") + }) + + suite.Run("fails when file is not found", func() { + tppsPaidInvoice := TPPSData{} + tppsEntries, err := tppsPaidInvoice.Parse(suite.AppContextForTest(), "non_existent_file.csv") + + suite.Nil(tppsEntries) + suite.Error(err) + suite.Contains(err.Error(), "Unable to read TPPS paid invoice report from path non_existent_file.csv") + }) } diff --git a/pkg/services/invoice.go b/pkg/services/invoice.go index effc530de28..847132b3c14 100644 --- a/pkg/services/invoice.go +++ b/pkg/services/invoice.go @@ -6,8 +6,11 @@ import ( "os" "time" + "github.com/gobuffalo/validate/v3" + "github.com/transcom/mymove/pkg/appcontext" ediinvoice "github.com/transcom/mymove/pkg/edi/invoice" + tppsResponse "github.com/transcom/mymove/pkg/edi/tpps_paid_invoice_report" "github.com/transcom/mymove/pkg/models" ) @@ -73,3 +76,11 @@ type SyncadaFileProcessor interface { ProcessFile(appCtx appcontext.AppContext, syncadaPath string, text string) error EDIType() models.EDIType } + +// TPPSPaidInvoiceReportProcessor defines an interface for storing TPPS payment files in the database +// +//go:generate mockery --name TPPSPaidInvoiceReportProcessor +type TPPSPaidInvoiceReportProcessor interface { + ProcessFile(appCtx appcontext.AppContext, syncadaPath string, text string) error + StoreTPPSPaidInvoiceReportInDatabase(appCtx appcontext.AppContext, tppsData []tppsResponse.TPPSData) (*validate.Errors, int, int, error) +} diff --git a/pkg/services/invoice/fixtures/tpps_paid_invoice_report_testfile_large_encoded.csv b/pkg/services/invoice/fixtures/tpps_paid_invoice_report_testfile_large_encoded.csv new file mode 100644 index 00000000000..6c1c72a0993 Binary files /dev/null and b/pkg/services/invoice/fixtures/tpps_paid_invoice_report_testfile_large_encoded.csv differ diff --git a/pkg/services/invoice/process_tpps_paid_invoice_report.go b/pkg/services/invoice/process_tpps_paid_invoice_report.go index ee192fd3e1b..9f8881a7866 100644 --- a/pkg/services/invoice/process_tpps_paid_invoice_report.go +++ b/pkg/services/invoice/process_tpps_paid_invoice_report.go @@ -1,12 +1,16 @@ package invoice import ( + "database/sql" + "errors" "fmt" "strconv" "strings" "time" "github.com/gobuffalo/validate/v3" + "github.com/gofrs/uuid" + "github.com/lib/pq" "go.uber.org/zap" "github.com/transcom/mymove/pkg/appcontext" @@ -47,77 +51,88 @@ type TPPSData struct { } // NewTPPSPaidInvoiceReportProcessor returns a new TPPS paid invoice report processor -func NewTPPSPaidInvoiceReportProcessor() services.SyncadaFileProcessor { - +func NewTPPSPaidInvoiceReportProcessor() services.TPPSPaidInvoiceReportProcessor { return &tppsPaidInvoiceReportProcessor{} } // ProcessFile parses a TPPS paid invoice report response and updates the payment request status func (t *tppsPaidInvoiceReportProcessor) ProcessFile(appCtx appcontext.AppContext, TPPSPaidInvoiceReportFilePath string, stringTPPSPaidInvoiceReport string) error { + + if TPPSPaidInvoiceReportFilePath == "" { + appCtx.Logger().Info("No valid filepath found to process TPPS Paid Invoice Report", zap.String("TPPSPaidInvoiceReportFilePath", TPPSPaidInvoiceReportFilePath)) + return nil + } tppsPaidInvoiceReport := tppsReponse.TPPSData{} - tppsData, err := tppsPaidInvoiceReport.Parse(TPPSPaidInvoiceReportFilePath, "") + appCtx.Logger().Info(fmt.Sprintf("Processing filepath: %s\n", TPPSPaidInvoiceReportFilePath)) + + tppsData, err := tppsPaidInvoiceReport.Parse(appCtx, TPPSPaidInvoiceReportFilePath) if err != nil { appCtx.Logger().Error("unable to parse TPPS paid invoice report", zap.Error(err)) return fmt.Errorf("unable to parse TPPS paid invoice report") - } else { - appCtx.Logger().Info("Successfully parsed TPPS Paid Invoice Report") } - appCtx.Logger().Info("RECEIVED: TPPS Paid Invoice Report Processor received a TPPS Paid Invoice Report") - if tppsData != nil { - verrs, errs := t.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) + appCtx.Logger().Info(fmt.Sprintf("Successfully parsed data from the TPPS paid invoice report: %s", TPPSPaidInvoiceReportFilePath)) + verrs, processedRowCount, errorProcessingRowCount, err := t.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) if err != nil { - return errs + return err } else if verrs.HasAny() { return verrs } else { - appCtx.Logger().Info("Successfully stored TPPS Paid Invoice Report information in the database") + appCtx.Logger().Info("Stored TPPS Paid Invoice Report information in the database") + appCtx.Logger().Info(fmt.Sprintf("Rows successfully stored in DB: %d", processedRowCount)) + appCtx.Logger().Info(fmt.Sprintf("Rows not stored in DB due to foreign key constraint or other error: %d", errorProcessingRowCount)) } - transactionError := appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { - var paymentRequestWithStatusUpdatedToPaid = map[string]string{} + var paymentRequestWithStatusUpdatedToPaid = map[string]string{} - // For the data in the TPPS Paid Invoice Report, find the payment requests that match the - // invoice numbers of the rows in the report and update the payment request status to PAID - for _, tppsDataForOnePaymentRequest := range tppsData { - var paymentRequest models.PaymentRequest + // For the data in the TPPS Paid Invoice Report, find the payment requests that match the + // invoice numbers of the rows in the report and update the payment request status to PAID + updatedPaymentRequestStatusCount := 0 + for _, tppsDataForOnePaymentRequest := range tppsData { + var paymentRequest models.PaymentRequest - err = txnAppCtx.DB().Q(). - Where("payment_requests.payment_request_number = ?", tppsDataForOnePaymentRequest.InvoiceNumber). - First(&paymentRequest) + err = appCtx.DB().Q(). + Where("payment_requests.payment_request_number = ?", tppsDataForOnePaymentRequest.InvoiceNumber). + First(&paymentRequest) - if err != nil { - return err + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + appCtx.Logger().Warn(fmt.Sprintf("No matching existing payment request found for invoice number %s, can't update status to PAID", tppsDataForOnePaymentRequest.InvoiceNumber)) + continue + } else { + appCtx.Logger().Error(fmt.Sprintf("Database error while looking up payment request for invoice number %s", tppsDataForOnePaymentRequest.InvoiceNumber), zap.Error(err)) + continue } + } - // Since there can be many rows in a TPPS report that reference the same payment request, we want - // to keep track of which payment requests we've already updated the status to PAID for and - // only update it's status once, using a map to keep track of already updated payment requests - _, paymentRequestExistsInUpdatedStatusMap := paymentRequestWithStatusUpdatedToPaid[paymentRequest.ID.String()] - if !paymentRequestExistsInUpdatedStatusMap { - paymentRequest.Status = models.PaymentRequestStatusPaid - err = txnAppCtx.DB().Update(&paymentRequest) - if err != nil { - txnAppCtx.Logger().Error("failure updating payment request to PAID", zap.Error(err)) - return fmt.Errorf("failure updating payment request status to PAID: %w", err) - } - - txnAppCtx.Logger().Info("SUCCESS: TPPS Paid Invoice Report Processor updated Payment Request to PAID status") - t.logTPPSInvoiceReportWithPaymentRequest(txnAppCtx, tppsDataForOnePaymentRequest, paymentRequest) + if paymentRequest.ID == uuid.Nil { + appCtx.Logger().Error(fmt.Sprintf("Invalid payment request ID for invoice number %s", tppsDataForOnePaymentRequest.InvoiceNumber)) + continue + } + _, paymentRequestExistsInUpdatedStatusMap := paymentRequestWithStatusUpdatedToPaid[paymentRequest.ID.String()] + if !paymentRequestExistsInUpdatedStatusMap { + paymentRequest.Status = models.PaymentRequestStatusPaid + err = appCtx.DB().Update(&paymentRequest) + if err != nil { + appCtx.Logger().Info(fmt.Sprintf("Failure updating payment request %s to PAID status", paymentRequest.PaymentRequestNumber)) + continue + } else { + if tppsDataForOnePaymentRequest.InvoiceNumber != uuid.Nil.String() && paymentRequest.ID != uuid.Nil { + t.logTPPSInvoiceReportWithPaymentRequest(appCtx, tppsDataForOnePaymentRequest, paymentRequest) + } + updatedPaymentRequestStatusCount += 1 paymentRequestWithStatusUpdatedToPaid[paymentRequest.ID.String()] = paymentRequest.PaymentRequestNumber } } - return nil - }) - - if transactionError != nil { - appCtx.Logger().Error(transactionError.Error()) - return transactionError } + appCtx.Logger().Info(fmt.Sprintf("Payment requests that had status updated to PAID in DB: %d", updatedPaymentRequestStatusCount)) + return nil + } else { + appCtx.Logger().Info("No TPPS Paid Invoice Report data was parsed, so no data was stored in the database") } return nil @@ -128,7 +143,7 @@ func (t *tppsPaidInvoiceReportProcessor) EDIType() models.EDIType { } func (t *tppsPaidInvoiceReportProcessor) logTPPSInvoiceReportWithPaymentRequest(appCtx appcontext.AppContext, tppsResponse tppsReponse.TPPSData, paymentRequest models.PaymentRequest) { - appCtx.Logger().Info("TPPS Paid Invoice Report log", + appCtx.Logger().Info("Updated payment request status to PAID", zap.String("TPPSPaidInvoiceReportEntry.InvoiceNumber", tppsResponse.InvoiceNumber), zap.String("PaymentRequestNumber", paymentRequest.PaymentRequestNumber), zap.String("PaymentRequest.Status", string(paymentRequest.Status)), @@ -184,43 +199,57 @@ func priceToMillicents(rawPrice string) (int, error) { return millicents, nil } -func (t *tppsPaidInvoiceReportProcessor) StoreTPPSPaidInvoiceReportInDatabase(appCtx appcontext.AppContext, tppsData []tppsReponse.TPPSData) (*validate.Errors, error) { +func (t *tppsPaidInvoiceReportProcessor) StoreTPPSPaidInvoiceReportInDatabase(appCtx appcontext.AppContext, tppsData []tppsReponse.TPPSData) (*validate.Errors, int, int, error) { var verrs *validate.Errors - transactionError := appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { + var failedEntries []error + DateParamFormat := "2006-01-02" + processedRowCount := 0 + errorProcessingRowCount := 0 - DateParamFormat := "2006-01-02" + for _, tppsEntry := range tppsData { + timeOfTPPSCreatedDocumentDate, err := time.Parse(DateParamFormat, tppsEntry.TPPSCreatedDocumentDate) + if err != nil { + appCtx.Logger().Warn("unable to parse TPPSCreatedDocumentDate", zap.String("invoiceNumber", tppsEntry.InvoiceNumber), zap.Error(err)) + failedEntries = append(failedEntries, fmt.Errorf("invoiceNumber %s: %v", tppsEntry.InvoiceNumber, err)) + continue + } - for _, tppsEntry := range tppsData { - timeOfTPPSCreatedDocumentDate, err := time.Parse(DateParamFormat, tppsEntry.TPPSCreatedDocumentDate) - if err != nil { - appCtx.Logger().Info("unable to parse TPPSCreatedDocumentDate from TPPS paid invoice report", zap.Error(err)) - } - timeOfSellerPaidDate, err := time.Parse(DateParamFormat, tppsEntry.SellerPaidDate) - if err != nil { - appCtx.Logger().Info("unable to parse SellerPaidDate from TPPS paid invoice report", zap.Error(err)) - return verrs - } - invoiceTotalChargesInMillicents, err := priceToMillicents(tppsEntry.InvoiceTotalCharges) - if err != nil { - appCtx.Logger().Info("unable to parse InvoiceTotalCharges from TPPS paid invoice report", zap.Error(err)) - return verrs - } - intLineBillingUnits, err := strconv.Atoi(tppsEntry.LineBillingUnits) - if err != nil { - appCtx.Logger().Info("unable to parse LineBillingUnits from TPPS paid invoice report", zap.Error(err)) - return verrs - } - lineUnitPriceInMillicents, err := priceToMillicents(tppsEntry.LineUnitPrice) - if err != nil { - appCtx.Logger().Info("unable to parse LineUnitPrice from TPPS paid invoice report", zap.Error(err)) - return verrs - } - lineNetChargeInMillicents, err := priceToMillicents(tppsEntry.LineNetCharge) - if err != nil { - appCtx.Logger().Info("unable to parse LineNetCharge from TPPS paid invoice report", zap.Error(err)) - return verrs - } + timeOfSellerPaidDate, err := time.Parse(DateParamFormat, tppsEntry.SellerPaidDate) + if err != nil { + appCtx.Logger().Warn("unable to parse SellerPaidDate", zap.String("invoiceNumber", tppsEntry.InvoiceNumber), zap.Error(err)) + failedEntries = append(failedEntries, fmt.Errorf("invoiceNumber %s: %v", tppsEntry.InvoiceNumber, err)) + continue + } + + invoiceTotalChargesInMillicents, err := priceToMillicents(tppsEntry.InvoiceTotalCharges) + if err != nil { + appCtx.Logger().Warn("unable to parse InvoiceTotalCharges", zap.String("invoiceNumber", tppsEntry.InvoiceNumber), zap.Error(err)) + failedEntries = append(failedEntries, fmt.Errorf("invoiceNumber %s: %v", tppsEntry.InvoiceNumber, err)) + continue + } + + intLineBillingUnits, err := strconv.Atoi(tppsEntry.LineBillingUnits) + if err != nil { + appCtx.Logger().Warn("unable to parse LineBillingUnits", zap.String("invoiceNumber", tppsEntry.InvoiceNumber), zap.Error(err)) + failedEntries = append(failedEntries, fmt.Errorf("invoiceNumber %s: %v", tppsEntry.InvoiceNumber, err)) + continue + } + + lineUnitPriceInMillicents, err := priceToMillicents(tppsEntry.LineUnitPrice) + if err != nil { + appCtx.Logger().Warn("unable to parse LineUnitPrice", zap.String("invoiceNumber", tppsEntry.InvoiceNumber), zap.Error(err)) + failedEntries = append(failedEntries, fmt.Errorf("invoiceNumber %s: %v", tppsEntry.InvoiceNumber, err)) + continue + } + lineNetChargeInMillicents, err := priceToMillicents(tppsEntry.LineNetCharge) + if err != nil { + appCtx.Logger().Warn("unable to parse LineNetCharge", zap.String("invoiceNumber", tppsEntry.InvoiceNumber), zap.Error(err)) + failedEntries = append(failedEntries, fmt.Errorf("invoiceNumber %s: %v", tppsEntry.InvoiceNumber, err)) + continue + } + + txnErr := appCtx.NewTransaction(func(txnAppCtx appcontext.AppContext) error { tppsEntryModel := models.TPPSPaidInvoiceReportEntry{ InvoiceNumber: tppsEntry.InvoiceNumber, TPPSCreatedDocumentDate: &timeOfTPPSCreatedDocumentDate, @@ -249,22 +278,41 @@ func (t *tppsPaidInvoiceReportProcessor) StoreTPPSPaidInvoiceReportInDatabase(ap verrs, err = txnAppCtx.DB().ValidateAndSave(&tppsEntryModel) if err != nil { - appCtx.Logger().Error("failure saving entry from TPPS paid invoice report", zap.Error(err)) - return err + if isForeignKeyConstraintViolation(err) { + appCtx.Logger().Warn(fmt.Sprintf("skipping entry due to missing foreign key reference for invoice number %s", tppsEntry.InvoiceNumber)) + failedEntries = append(failedEntries, fmt.Errorf("invoice number %s: foreign key constraint violation", tppsEntry.InvoiceNumber)) + return fmt.Errorf("rolling back transaction to prevent blocking") + } + + appCtx.Logger().Error(fmt.Sprintf("failed to save entry for invoice number %s", tppsEntry.InvoiceNumber), zap.Error(err)) + failedEntries = append(failedEntries, fmt.Errorf("invoice number %s: %v", tppsEntry.InvoiceNumber, err)) + return fmt.Errorf("rolling back transaction to prevent blocking") } - } - return nil - }) + appCtx.Logger().Info(fmt.Sprintf("successfully saved entry in DB for invoice number: %s", tppsEntry.InvoiceNumber)) + processedRowCount += 1 + return nil + }) - if transactionError != nil { - appCtx.Logger().Error(transactionError.Error()) - return verrs, transactionError + if txnErr != nil { + appCtx.Logger().Error(fmt.Sprintf("transaction error for invoice number %s", tppsEntry.InvoiceNumber), zap.Error(txnErr)) + errorProcessingRowCount += 1 + } } - if verrs.HasAny() { - appCtx.Logger().Error("unable to process TPPS paid invoice report", zap.Error(verrs)) - return verrs, nil + + if len(failedEntries) > 0 { + for _, err := range failedEntries { + appCtx.Logger().Error("failed entry", zap.Error(err)) + } } - return nil, nil + // Return verrs but not a hard failure so we can process the rest of the entries + return verrs, processedRowCount, errorProcessingRowCount, nil +} + +func isForeignKeyConstraintViolation(err error) bool { + if pqErr, ok := err.(*pq.Error); ok { + return pqErr.Code == "23503" + } + return false } diff --git a/pkg/services/invoice/process_tpps_paid_invoice_report_test.go b/pkg/services/invoice/process_tpps_paid_invoice_report_test.go index eb074b672a9..4dec4a50a96 100644 --- a/pkg/services/invoice/process_tpps_paid_invoice_report_test.go +++ b/pkg/services/invoice/process_tpps_paid_invoice_report_test.go @@ -1,11 +1,16 @@ package invoice import ( + "bytes" "testing" "time" "github.com/stretchr/testify/suite" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "github.com/transcom/mymove/pkg/appcontext" + tppsResponse "github.com/transcom/mymove/pkg/edi/tpps_paid_invoice_report" "github.com/transcom/mymove/pkg/factory" "github.com/transcom/mymove/pkg/models" "github.com/transcom/mymove/pkg/testingsuite" @@ -177,6 +182,128 @@ func (suite *ProcessTPPSPaidInvoiceReportSuite) TestParsingTPPSPaidInvoiceReport } }) + suite.Run("successfully stores valid entries to database even if invalid liens (no matching payment request number) found in file", func() { + // 1841-7267-3 is a payment request that the test TPPS file references + // 9436-4123-3 is a payment request that the test TPPS file references, but we WON'T create it + paymentRequestOne := factory.BuildPaymentRequest(suite.DB(), []factory.Customization{ + { + Model: models.PaymentRequest{ + Status: models.PaymentRequestStatusPaid, + PaymentRequestNumber: "1841-7267-3", + }, + }, + }, nil) + suite.NotNil(paymentRequestOne) + + testTPPSPaidInvoiceReportFilePath := "../../../pkg/services/invoice/fixtures/tpps_paid_invoice_report_testfile.csv" + + err := tppsPaidInvoiceReportProcessor.ProcessFile(suite.AppContextForTest(), testTPPSPaidInvoiceReportFilePath, "") + suite.NoError(err) + + tppsEntries := []models.TPPSPaidInvoiceReportEntry{} + err = suite.DB().All(&tppsEntries) + suite.NoError(err) + // instead of 5 entries, we only have 4 since line 6 in the test file references a payment request number that doesn't exist: 9436-4123-3 + suite.Equal(4, len(tppsEntries)) + + // find the paymentRequests and verify that they have all been updated to have a status of PAID after processing the report + paymentRequests := []models.PaymentRequest{} + err = suite.DB().All(&paymentRequests) + suite.NoError(err) + // only 1 payment request should have its status updated to PAID + suite.Equal(len(paymentRequests), 1) + + for _, paymentRequest := range paymentRequests { + suite.Equal(models.PaymentRequestStatusPaid, paymentRequest.Status) + } + + for tppsEntryIndex := range tppsEntries { + + if tppsEntryIndex == 0 { + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceNumber, "1841-7267-3") + suite.Equal(*tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate, time.Date(2024, time.July, 29, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].SellerPaidDate, time.Date(2024, time.July, 30, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceTotalChargesInMillicents, unit.Millicents(115155000)) // 1151.55 + suite.Equal(tppsEntries[tppsEntryIndex].LineDescription, "DDP") + suite.Equal(tppsEntries[tppsEntryIndex].ProductDescription, "DDP") + suite.Equal(tppsEntries[tppsEntryIndex].LineBillingUnits, 3760) + suite.Equal(tppsEntries[tppsEntryIndex].LineUnitPrice, unit.Millicents(770)) // 0.0077 + suite.Equal(tppsEntries[tppsEntryIndex].LineNetCharge, unit.Millicents(2895000)) // 28.95 + suite.Equal(tppsEntries[tppsEntryIndex].POTCN, "1841-7267-826285fc") + suite.Equal(tppsEntries[tppsEntryIndex].LineNumber, "1") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCode, "INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteDescription, "Notes to My Company - INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeTo, "CARR") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeMessage, "HQ50066") + } + if tppsEntryIndex == 1 { + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceNumber, "1841-7267-3") + suite.Equal(*tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate, time.Date(2024, time.July, 29, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].SellerPaidDate, time.Date(2024, time.July, 30, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceTotalChargesInMillicents, unit.Millicents(115155000)) // 1151.55 + suite.Equal(tppsEntries[tppsEntryIndex].LineDescription, "FSC") + suite.Equal(tppsEntries[tppsEntryIndex].ProductDescription, "FSC") + suite.Equal(tppsEntries[tppsEntryIndex].LineBillingUnits, 3760) + suite.Equal(tppsEntries[tppsEntryIndex].LineUnitPrice, unit.Millicents(140)) // 0.0014 + suite.Equal(tppsEntries[tppsEntryIndex].LineNetCharge, unit.Millicents(539000)) // 5.39 + suite.Equal(tppsEntries[tppsEntryIndex].POTCN, "1841-7267-aeb3cfea") + suite.Equal(tppsEntries[tppsEntryIndex].LineNumber, "4") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCode, "INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteDescription, "Notes to My Company - INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeTo, "CARR") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeMessage, "HQ50066") + + } + if tppsEntryIndex == 2 { + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceNumber, "1841-7267-3") + suite.Equal(*tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate, time.Date(2024, time.July, 29, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].SellerPaidDate, time.Date(2024, time.July, 30, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceTotalChargesInMillicents, unit.Millicents(115155000)) // 1151.55 + suite.Equal(tppsEntries[tppsEntryIndex].LineDescription, "DLH") + suite.Equal(tppsEntries[tppsEntryIndex].ProductDescription, "DLH") + suite.Equal(tppsEntries[tppsEntryIndex].LineBillingUnits, 3760) + suite.Equal(tppsEntries[tppsEntryIndex].LineUnitPrice, unit.Millicents(26560)) // 0.2656 + suite.Equal(tppsEntries[tppsEntryIndex].LineNetCharge, unit.Millicents(99877000)) // 998.77 + suite.Equal(tppsEntries[tppsEntryIndex].POTCN, "1841-7267-c8ea170b") + suite.Equal(tppsEntries[tppsEntryIndex].LineNumber, "2") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCode, "INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteDescription, "Notes to My Company - INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeTo, "CARR") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeMessage, "HQ50066") + + } + if tppsEntryIndex == 3 { + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceNumber, "1841-7267-3") + suite.Equal(*tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate, time.Date(2024, time.July, 29, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].SellerPaidDate, time.Date(2024, time.July, 30, 0, 0, 0, 0, tppsEntries[tppsEntryIndex].TPPSCreatedDocumentDate.Location())) + suite.Equal(tppsEntries[tppsEntryIndex].InvoiceTotalChargesInMillicents, unit.Millicents(115155000)) // 1151.55 + suite.Equal(tppsEntries[tppsEntryIndex].LineDescription, "DUPK") + suite.Equal(tppsEntries[tppsEntryIndex].ProductDescription, "DUPK") + suite.Equal(tppsEntries[tppsEntryIndex].LineBillingUnits, 3760) + suite.Equal(tppsEntries[tppsEntryIndex].LineUnitPrice, unit.Millicents(3150)) // 0.0315 + suite.Equal(tppsEntries[tppsEntryIndex].LineNetCharge, unit.Millicents(11844000)) // 118.44 + suite.Equal(tppsEntries[tppsEntryIndex].POTCN, "1841-7267-265c16d7") + suite.Equal(tppsEntries[tppsEntryIndex].LineNumber, "3") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCode, "INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteDescription, "Notes to My Company - INT") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeTo, "CARR") + suite.Equal(*tppsEntries[tppsEntryIndex].FirstNoteCodeMessage, "HQ50066") + } + + suite.NotNil(tppsEntries[tppsEntryIndex].ID) + suite.NotNil(tppsEntries[tppsEntryIndex].CreatedAt) + suite.NotNil(tppsEntries[tppsEntryIndex].UpdatedAt) + suite.Equal(*tppsEntries[tppsEntryIndex].SecondNoteCode, "") + suite.Equal(*tppsEntries[tppsEntryIndex].SecondNoteDescription, "") + suite.Equal(*tppsEntries[tppsEntryIndex].SecondNoteCodeTo, "") + suite.Equal(*tppsEntries[tppsEntryIndex].SecondNoteCodeMessage, "") + suite.Equal(*tppsEntries[tppsEntryIndex].ThirdNoteCode, "") + suite.Equal(*tppsEntries[tppsEntryIndex].ThirdNoteDescription, "") + suite.Equal(*tppsEntries[tppsEntryIndex].ThirdNoteCodeTo, "") + suite.Equal(*tppsEntries[tppsEntryIndex].ThirdNoteCodeMessage, "") + } + }) + suite.Run("successfully processes a TPPSPaidInvoiceReport from a file directly from the TPPS pickup directory and stores it in the database", func() { // payment requests 1-4 with a payment request numbers of 1841-7267-3, 1208-5962-1, // 8801-2773-2, and 8801-2773-3 must exist because the TPPS invoice report's invoice @@ -493,7 +620,13 @@ func (suite *ProcessTPPSPaidInvoiceReportSuite) TestParsingTPPSPaidInvoiceReport } }) - suite.Run("error opening filepath returns descriptive error for failing to parse TPPS paid invoice report", func() { + suite.Run("returns nil when file path is empty", func() { + tppsPaidInvoiceReportProcessor := NewTPPSPaidInvoiceReportProcessor() + err := tppsPaidInvoiceReportProcessor.ProcessFile(suite.AppContextForTest(), "", "") + suite.NoError(err) + }) + + suite.Run("returns error for failing to parse TPPS paid invoice report", func() { // given a path to a nonexistent file testTPPSPaidInvoiceReportFilePath := "../../../pkg/services/invoice/AFileThatDoesNotExist.csv" @@ -507,4 +640,187 @@ func (suite *ProcessTPPSPaidInvoiceReportSuite) TestParsingTPPSPaidInvoiceReport suite.NoError(err) suite.Equal(len(tppsEntries), 0) }) + + suite.Run("Logs message if invalid TPPSCreatedDocumentDate found", func() { + var logBuffer bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuffer), + zap.DebugLevel, + ) + logger := zap.New(core) + appCtx := appcontext.NewAppContext(nil, logger, nil) + + tppsData := []tppsResponse.TPPSData{ + { + TPPSCreatedDocumentDate: "INVALID_DATE-01-14", + }, + } + + verrs, processedCount, errorCount, err := tppsPaidInvoiceReportProcessor.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) + + suite.NoError(err) + suite.False(verrs.HasAny()) + suite.Equal(0, processedCount) + suite.Equal(0, errorCount) + + logOutput := logBuffer.String() + suite.Contains(logOutput, "unable to parse TPPSCreatedDocumentDate") + + }) + + suite.Run("Logs message if invalid SellerPaidDate found", func() { + var logBuffer bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuffer), + zap.DebugLevel, + ) + logger := zap.New(core) + appCtx := appcontext.NewAppContext(nil, logger, nil) + + tppsData := []tppsResponse.TPPSData{ + { + TPPSCreatedDocumentDate: "2025-01-14", + SellerPaidDate: "INVALID_DATE", + }, + } + + verrs, processedCount, errorCount, err := tppsPaidInvoiceReportProcessor.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) + + suite.NoError(err) + suite.False(verrs.HasAny()) + suite.Equal(0, processedCount) + suite.Equal(0, errorCount) + + logOutput := logBuffer.String() + suite.Contains(logOutput, "unable to parse SellerPaidDate") + + }) + + suite.Run("Logs message if invalid InvoiceTotalCharges found", func() { + var logBuffer bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuffer), + zap.DebugLevel, + ) + logger := zap.New(core) + appCtx := appcontext.NewAppContext(nil, logger, nil) + + tppsData := []tppsResponse.TPPSData{ + { + TPPSCreatedDocumentDate: "2025-01-14", + SellerPaidDate: "2025-01-14", + InvoiceTotalCharges: "abc", + }, + } + + verrs, processedCount, errorCount, err := tppsPaidInvoiceReportProcessor.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) + + suite.NoError(err) + suite.False(verrs.HasAny()) + suite.Equal(0, processedCount) + suite.Equal(0, errorCount) + + logOutput := logBuffer.String() + suite.Contains(logOutput, "unable to parse InvoiceTotalCharges") + + }) + + suite.Run("Logs message if invalid LineBillingUnits found", func() { + var logBuffer bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuffer), + zap.DebugLevel, + ) + logger := zap.New(core) + appCtx := appcontext.NewAppContext(nil, logger, nil) + + tppsData := []tppsResponse.TPPSData{ + { + TPPSCreatedDocumentDate: "2025-01-14", + SellerPaidDate: "2025-01-14", + InvoiceTotalCharges: "009823", + LineBillingUnits: "abc", + }, + } + + verrs, processedCount, errorCount, err := tppsPaidInvoiceReportProcessor.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) + + suite.NoError(err) + suite.False(verrs.HasAny()) + suite.Equal(0, processedCount) + suite.Equal(0, errorCount) + + logOutput := logBuffer.String() + suite.Contains(logOutput, "unable to parse LineBillingUnits") + + }) + + suite.Run("Logs message if invalid LineUnitPrice found", func() { + var logBuffer bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuffer), + zap.DebugLevel, + ) + logger := zap.New(core) + appCtx := appcontext.NewAppContext(nil, logger, nil) + + tppsData := []tppsResponse.TPPSData{ + { + TPPSCreatedDocumentDate: "2025-01-14", + SellerPaidDate: "2025-01-14", + InvoiceTotalCharges: "009823", + LineBillingUnits: "1234", + LineUnitPrice: "abc", + }, + } + + verrs, processedCount, errorCount, err := tppsPaidInvoiceReportProcessor.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) + + suite.NoError(err) + suite.False(verrs.HasAny()) + suite.Equal(0, processedCount) + suite.Equal(0, errorCount) + + logOutput := logBuffer.String() + suite.Contains(logOutput, "unable to parse LineUnitPrice") + + }) + + suite.Run("Logs message if invalid LineNetCharge found", func() { + var logBuffer bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuffer), + zap.DebugLevel, + ) + logger := zap.New(core) + appCtx := appcontext.NewAppContext(nil, logger, nil) + + tppsData := []tppsResponse.TPPSData{ + { + TPPSCreatedDocumentDate: "2025-01-14", + SellerPaidDate: "2025-01-14", + InvoiceTotalCharges: "009823", + LineBillingUnits: "1234", + LineUnitPrice: "1234", + LineNetCharge: "abc", + }, + } + + verrs, processedCount, errorCount, err := tppsPaidInvoiceReportProcessor.StoreTPPSPaidInvoiceReportInDatabase(appCtx, tppsData) + + suite.NoError(err) + suite.False(verrs.HasAny()) + suite.Equal(0, processedCount) + suite.Equal(0, errorCount) + + logOutput := logBuffer.String() + suite.Contains(logOutput, "unable to parse LineNetCharge") + + }) } diff --git a/pkg/services/mocks/TPPSPaidInvoiceReportProcessor.go b/pkg/services/mocks/TPPSPaidInvoiceReportProcessor.go new file mode 100644 index 00000000000..b0b66d005bf --- /dev/null +++ b/pkg/services/mocks/TPPSPaidInvoiceReportProcessor.go @@ -0,0 +1,93 @@ +// Code generated by mockery. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + appcontext "github.com/transcom/mymove/pkg/appcontext" + + tppspaidinvoicereport "github.com/transcom/mymove/pkg/edi/tpps_paid_invoice_report" + + validate "github.com/gobuffalo/validate/v3" +) + +// TPPSPaidInvoiceReportProcessor is an autogenerated mock type for the TPPSPaidInvoiceReportProcessor type +type TPPSPaidInvoiceReportProcessor struct { + mock.Mock +} + +// ProcessFile provides a mock function with given fields: appCtx, syncadaPath, text +func (_m *TPPSPaidInvoiceReportProcessor) ProcessFile(appCtx appcontext.AppContext, syncadaPath string, text string) error { + ret := _m.Called(appCtx, syncadaPath, text) + + if len(ret) == 0 { + panic("no return value specified for ProcessFile") + } + + var r0 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, string, string) error); ok { + r0 = rf(appCtx, syncadaPath, text) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// StoreTPPSPaidInvoiceReportInDatabase provides a mock function with given fields: appCtx, tppsData +func (_m *TPPSPaidInvoiceReportProcessor) StoreTPPSPaidInvoiceReportInDatabase(appCtx appcontext.AppContext, tppsData []tppspaidinvoicereport.TPPSData) (*validate.Errors, int, int, error) { + ret := _m.Called(appCtx, tppsData) + + if len(ret) == 0 { + panic("no return value specified for StoreTPPSPaidInvoiceReportInDatabase") + } + + var r0 *validate.Errors + var r1 int + var r2 int + var r3 error + if rf, ok := ret.Get(0).(func(appcontext.AppContext, []tppspaidinvoicereport.TPPSData) (*validate.Errors, int, int, error)); ok { + return rf(appCtx, tppsData) + } + if rf, ok := ret.Get(0).(func(appcontext.AppContext, []tppspaidinvoicereport.TPPSData) *validate.Errors); ok { + r0 = rf(appCtx, tppsData) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*validate.Errors) + } + } + + if rf, ok := ret.Get(1).(func(appcontext.AppContext, []tppspaidinvoicereport.TPPSData) int); ok { + r1 = rf(appCtx, tppsData) + } else { + r1 = ret.Get(1).(int) + } + + if rf, ok := ret.Get(2).(func(appcontext.AppContext, []tppspaidinvoicereport.TPPSData) int); ok { + r2 = rf(appCtx, tppsData) + } else { + r2 = ret.Get(2).(int) + } + + if rf, ok := ret.Get(3).(func(appcontext.AppContext, []tppspaidinvoicereport.TPPSData) error); ok { + r3 = rf(appCtx, tppsData) + } else { + r3 = ret.Error(3) + } + + return r0, r1, r2, r3 +} + +// NewTPPSPaidInvoiceReportProcessor creates a new instance of TPPSPaidInvoiceReportProcessor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewTPPSPaidInvoiceReportProcessor(t interface { + mock.TestingT + Cleanup(func()) +}) *TPPSPaidInvoiceReportProcessor { + mock := &TPPSPaidInvoiceReportProcessor{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/scripts/deploy-app-tasks b/scripts/deploy-app-tasks index fac6d101650..bdc20acde20 100755 --- a/scripts/deploy-app-tasks +++ b/scripts/deploy-app-tasks @@ -52,5 +52,6 @@ readonly image="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ap scripts/ecs-deploy-task-container connect-to-gex-via-sftp "${image}" "${APP_ENVIRONMENT}" scripts/ecs-deploy-task-container post-file-to-gex "${image}" "${APP_ENVIRONMENT}" scripts/ecs-deploy-task-container process-edis "${image}" "${APP_ENVIRONMENT}" +scripts/ecs-deploy-task-container process-tpps "${image}" "${APP_ENVIRONMENT}" scripts/ecs-deploy-task-container save-ghc-fuel-price-data "${image}" "${APP_ENVIRONMENT}" scripts/ecs-deploy-task-container send-payment-reminder "${image}" "${APP_ENVIRONMENT}" diff --git a/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequest.test.jsx b/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequest.test.jsx index 6b6a8a30caf..fc8f35f1cd3 100644 --- a/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequest.test.jsx +++ b/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequest.test.jsx @@ -28,6 +28,18 @@ describe('when a payment request has an update', () => { }, }; + const historyRecord3 = { + action: 'UPDATE', + tableName: 'payment_requests', + eventName: '', + changedValues: { + status: 'PAID', + }, + oldValues: { + payment_request_number: '4462-6355-3', + }, + }; + const historyRecordWithError = { action: 'UPDATE', tableName: 'payment_requests', @@ -56,8 +68,9 @@ describe('when a payment request has an update', () => { describe('should display the proper labeled details when payment status is changed', () => { it.each([ ['Status', ': Sent to GEX', historyRecord], - ['Status', ': Received', historyRecord2], - ['Status', ': EDI error', historyRecordWithError], + ['Status', ': TPPS Received', historyRecord2], + ['Status', ': TPPS Paid', historyRecord3], + ['Status', ': EDI Error', historyRecordWithError], ])('label `%s` should have value `%s`', (label, value, record) => { const template = getTemplate(record); render(template.getDetails(record)); diff --git a/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequestJobRunner.test.jsx b/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequestJobRunner.test.jsx index 6cab43c2f53..869150630a4 100644 --- a/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequestJobRunner.test.jsx +++ b/src/constants/MoveHistory/EventTemplates/UpdatePaymentRequest/updatePaymentRequestJobRunner.test.jsx @@ -26,6 +26,17 @@ describe('when a payment request has an update', () => { }, }; + const historyRecord3 = { + action: 'UPDATE', + tableName: 'payment_requests', + changedValues: { + status: 'PAID', + }, + oldValues: { + payment_request_number: '4462-6355-3', + }, + }; + const historyRecordWithError = { action: 'UPDATE', tableName: 'payment_requests', @@ -54,8 +65,9 @@ describe('when a payment request has an update', () => { describe('should display the proper labeled details when payment status is changed', () => { it.each([ ['Status', ': Sent to GEX', historyRecord], - ['Status', ': Received', historyRecord2], - ['Status', ': EDI error', historyRecordWithError], + ['Status', ': TPPS Received', historyRecord2], + ['Status', ': TPPS Paid', historyRecord3], + ['Status', ': EDI Error', historyRecordWithError], ])('label `%s` should have value `%s`', (label, value, record) => { const template = getTemplate(record); render(template.getDetails(record)); diff --git a/src/constants/paymentRequestStatus.js b/src/constants/paymentRequestStatus.js index 7d4a7873049..276247eae9f 100644 --- a/src/constants/paymentRequestStatus.js +++ b/src/constants/paymentRequestStatus.js @@ -10,12 +10,12 @@ export default { }; export const PAYMENT_REQUEST_STATUS_LABELS = { - PENDING: 'Payment requested', + PENDING: 'Payment Requested', REVIEWED: 'Reviewed', SENT_TO_GEX: 'Sent to GEX', - TPPS_RECEIVED: 'Received', + TPPS_RECEIVED: 'TPPS Received', REVIEWED_AND_ALL_SERVICE_ITEMS_REJECTED: 'Rejected', - PAID: 'Paid', - EDI_ERROR: 'EDI error', + PAID: 'TPPS Paid', + EDI_ERROR: 'EDI Error', DEPRECATED: 'Deprecated', }; diff --git a/src/pages/Office/PaymentRequestQueue/PaymentRequestQueue.test.jsx b/src/pages/Office/PaymentRequestQueue/PaymentRequestQueue.test.jsx index 5d1f3363409..a72aecad41d 100644 --- a/src/pages/Office/PaymentRequestQueue/PaymentRequestQueue.test.jsx +++ b/src/pages/Office/PaymentRequestQueue/PaymentRequestQueue.test.jsx @@ -214,7 +214,7 @@ describe('PaymentRequestQueue', () => { expect(firstPaymentRequest.find('td.customerName').text()).toBe('Spacemen, Leo'); expect(firstPaymentRequest.find('td.edipi').text()).toBe('3305957632'); expect(firstPaymentRequest.find('td.emplid').text()).toBe('1253694'); - expect(firstPaymentRequest.find('td.status').text()).toBe('Payment requested'); + expect(firstPaymentRequest.find('td.status').text()).toBe('Payment Requested'); expect(firstPaymentRequest.find('td.age').text()).toBe('Less than 1 day'); expect(firstPaymentRequest.find('td.submittedAt').text()).toBe('15 Oct 2020'); expect(firstPaymentRequest.find('td.locator').text()).toBe('R993T7'); @@ -227,7 +227,7 @@ describe('PaymentRequestQueue', () => { expect(secondPaymentRequest.find('td.customerName').text()).toBe('Booga, Ooga'); expect(secondPaymentRequest.find('td.edipi').text()).toBe('1234567'); expect(secondPaymentRequest.find('td.emplid').text()).toBe(''); - expect(secondPaymentRequest.find('td.status').text()).toBe('Payment requested'); + expect(secondPaymentRequest.find('td.status').text()).toBe('Payment Requested'); expect(secondPaymentRequest.find('td.age').text()).toBe('Less than 1 day'); expect(secondPaymentRequest.find('td.submittedAt').text()).toBe('17 Oct 2020'); expect(secondPaymentRequest.find('td.locator').text()).toBe('0OOGAB'); @@ -444,7 +444,7 @@ describe('PaymentRequestQueue', () => { , ); // expect Payment requested status to appear in the TIO queue - expect(screen.getAllByText('Payment requested')).toHaveLength(2); + expect(screen.getAllByText('Payment Requested')).toHaveLength(2); // expect other statuses NOT to appear in the TIO queue expect(screen.queryByText('Deprecated')).not.toBeInTheDocument(); expect(screen.queryByText('Error')).not.toBeInTheDocument(); diff --git a/src/utils/formatters.test.js b/src/utils/formatters.test.js index b09ac4b0937..07bbe66e07c 100644 --- a/src/utils/formatters.test.js +++ b/src/utils/formatters.test.js @@ -237,7 +237,7 @@ describe('formatters', () => { describe('paymentRequestStatusReadable', () => { it('returns expected string for PENDING', () => { - expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.PENDING)).toEqual('Payment requested'); + expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.PENDING)).toEqual('Payment Requested'); }); it('returns expected string for REVIEWED', () => { @@ -249,15 +249,15 @@ describe('formatters', () => { }); it('returns expected string for TPPS_RECEIVED', () => { - expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.TPPS_RECEIVED)).toEqual('Received'); + expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.TPPS_RECEIVED)).toEqual('TPPS Received'); }); it('returns expected string for PAID', () => { - expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.PAID)).toEqual('Paid'); + expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.PAID)).toEqual('TPPS Paid'); }); it('returns expected string for EDI_ERROR', () => { - expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.EDI_ERROR)).toEqual('EDI error'); + expect(formatters.paymentRequestStatusReadable(PAYMENT_REQUEST_STATUS.EDI_ERROR)).toEqual('EDI Error'); }); it('returns expected string for DEPRECATED', () => {