From 1b522c5715bf3b0899540a5fb21025b79ed89ec6 Mon Sep 17 00:00:00 2001 From: klouddb-dev Date: Thu, 19 Dec 2024 21:36:10 +0530 Subject: [PATCH] Enhance backup audit tool management and PostgreSQL configuration - Added a new backup audit tool feature to track and report missing backup dates. - Introduced a PostgreSQL configuration generator with improved settings management. - Updated Go module dependencies for better compatibility and performance. - Created a GitHub Actions workflow for PostgreSQL configuration testing. - Added HTML report templates for backup audit results. - Implemented error handling improvements in various components. These changes aim to enhance the functionality and usability of the backup and configuration management tools. --- .github/workflows/postgresconf.yml | 114 ++++ cmd/ciscollector/backuphistory.go | 136 +++++ cmd/ciscollector/main.go | 11 +- docker_testing/postgresconfig/input-13-1.txt | 20 + docker_testing/postgresconfig/input-13-10.txt | 13 + docker_testing/postgresconfig/input-13-2.txt | 51 ++ docker_testing/postgresconfig/input-13-3.txt | 50 ++ docker_testing/postgresconfig/input-13-4.txt | 33 ++ docker_testing/postgresconfig/input-13-5.txt | 31 ++ docker_testing/postgresconfig/input-13-6.txt | 31 ++ docker_testing/postgresconfig/input-13-7.txt | 15 + docker_testing/postgresconfig/input-13-8.txt | 15 + docker_testing/postgresconfig/input-13-9.txt | 25 + docker_testing/postgresconfig/input-14-1.txt | 20 + docker_testing/postgresconfig/input-14-2.txt | 51 ++ docker_testing/postgresconfig/input-14-3.txt | 50 ++ docker_testing/postgresconfig/input-14-4.txt | 33 ++ docker_testing/postgresconfig/input-14-5.txt | 31 ++ docker_testing/postgresconfig/input-14-6.txt | 31 ++ docker_testing/postgresconfig/input-14-7.txt | 15 + docker_testing/postgresconfig/input-14-8.txt | 15 + docker_testing/postgresconfig/input-14-9.txt | 25 + docker_testing/postgresconfig/input-15-1.txt | 20 + docker_testing/postgresconfig/input-15-2.txt | 51 ++ docker_testing/postgresconfig/input-15-3.txt | 50 ++ docker_testing/postgresconfig/input-15-4.txt | 33 ++ docker_testing/postgresconfig/input-15-5.txt | 31 ++ docker_testing/postgresconfig/input-15-6.txt | 31 ++ docker_testing/postgresconfig/input-15-7.txt | 15 + docker_testing/postgresconfig/input-15-8.txt | 15 + docker_testing/postgresconfig/input-15-9.txt | 25 + docker_testing/postgresconfig/input-16-1.txt | 27 + docker_testing/postgresconfig/input-16-2.txt | 51 ++ docker_testing/postgresconfig/input-16-3.txt | 50 ++ docker_testing/postgresconfig/input-16-4.txt | 33 ++ docker_testing/postgresconfig/input-16-5.txt | 31 ++ docker_testing/postgresconfig/input-16-6.txt | 31 ++ docker_testing/postgresconfig/input-16-7.txt | 15 + docker_testing/postgresconfig/input-16-8.txt | 15 + docker_testing/postgresconfig/input-16-9.txt | 25 + .../postgresconfig/kshieldconfig.toml | 8 + go.mod | 13 +- go.sum | 26 +- htmlreport/backup_history_helper.go | 7 + htmlreport/piireport.go | 8 + htmlreport/template/backup_history.tmpl | 58 ++ htmlreport/template/tab_mapping.tmpl | 2 + pkg/backuphistory/backuphistory.go | 393 +++++++++++++ pkg/backuphistory/terminal.go | 22 + pkg/config/config.go | 148 +++-- pkg/config/help.go | 5 + pkg/const/const.go | 46 +- postgres/configaudit/audit.go | 235 ++++++++ postgresconfig/examplepostgres.conf | 48 +- postgresconfig/generator.go | 79 +-- postgresconfig/helper_functions.go | 179 +++++- postgresconfig/summary.txt | 15 +- postgresconfig/user_input.go | 516 ++++++++++++++---- 58 files changed, 2929 insertions(+), 244 deletions(-) create mode 100644 .github/workflows/postgresconf.yml create mode 100644 cmd/ciscollector/backuphistory.go create mode 100644 docker_testing/postgresconfig/input-13-1.txt create mode 100644 docker_testing/postgresconfig/input-13-10.txt create mode 100644 docker_testing/postgresconfig/input-13-2.txt create mode 100644 docker_testing/postgresconfig/input-13-3.txt create mode 100644 docker_testing/postgresconfig/input-13-4.txt create mode 100644 docker_testing/postgresconfig/input-13-5.txt create mode 100644 docker_testing/postgresconfig/input-13-6.txt create mode 100644 docker_testing/postgresconfig/input-13-7.txt create mode 100644 docker_testing/postgresconfig/input-13-8.txt create mode 100644 docker_testing/postgresconfig/input-13-9.txt create mode 100644 docker_testing/postgresconfig/input-14-1.txt create mode 100644 docker_testing/postgresconfig/input-14-2.txt create mode 100644 docker_testing/postgresconfig/input-14-3.txt create mode 100644 docker_testing/postgresconfig/input-14-4.txt create mode 100644 docker_testing/postgresconfig/input-14-5.txt create mode 100644 docker_testing/postgresconfig/input-14-6.txt create mode 100644 docker_testing/postgresconfig/input-14-7.txt create mode 100644 docker_testing/postgresconfig/input-14-8.txt create mode 100644 docker_testing/postgresconfig/input-14-9.txt create mode 100644 docker_testing/postgresconfig/input-15-1.txt create mode 100644 docker_testing/postgresconfig/input-15-2.txt create mode 100644 docker_testing/postgresconfig/input-15-3.txt create mode 100644 docker_testing/postgresconfig/input-15-4.txt create mode 100644 docker_testing/postgresconfig/input-15-5.txt create mode 100644 docker_testing/postgresconfig/input-15-6.txt create mode 100644 docker_testing/postgresconfig/input-15-7.txt create mode 100644 docker_testing/postgresconfig/input-15-8.txt create mode 100644 docker_testing/postgresconfig/input-15-9.txt create mode 100644 docker_testing/postgresconfig/input-16-1.txt create mode 100644 docker_testing/postgresconfig/input-16-2.txt create mode 100644 docker_testing/postgresconfig/input-16-3.txt create mode 100644 docker_testing/postgresconfig/input-16-4.txt create mode 100644 docker_testing/postgresconfig/input-16-5.txt create mode 100644 docker_testing/postgresconfig/input-16-6.txt create mode 100644 docker_testing/postgresconfig/input-16-7.txt create mode 100644 docker_testing/postgresconfig/input-16-8.txt create mode 100644 docker_testing/postgresconfig/input-16-9.txt create mode 100644 docker_testing/postgresconfig/kshieldconfig.toml create mode 100644 htmlreport/backup_history_helper.go create mode 100644 htmlreport/template/backup_history.tmpl create mode 100644 pkg/backuphistory/backuphistory.go create mode 100644 pkg/backuphistory/terminal.go diff --git a/.github/workflows/postgresconf.yml b/.github/workflows/postgresconf.yml new file mode 100644 index 0000000..14d69c7 --- /dev/null +++ b/.github/workflows/postgresconf.yml @@ -0,0 +1,114 @@ +name: postgresconf + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: write + +jobs: + postgresconf: + name: "Postgres version : ${{ matrix.postgres_version }}, Testcase : ${{ matrix.testcases }}" + runs-on: ubuntu-latest + strategy: + matrix: + postgres_version: [13,14,15,16] + testcases: [9] + max-parallel: 4 + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version: 1.21.1 + cache: true + - name: make install + run: make install + - name: Create temp directory with random name + run: | + cd docker_testing/postgresconfig + RANDOM_NUM=$((RANDOM % 10000)) + TIMESTAMP=$(date +%Y%m%d_%H%M%S) + TEMP_DIR="$PWD/temp_${TIMESTAMP}_${RANDOM_NUM}" + mkdir -p "$TEMP_DIR" + echo "Temp directory created: $TEMP_DIR" + echo "TEMP_DIR=$TEMP_DIR" >> $GITHUB_ENV + - name: generate postgres config + run: | + cd $TEMP_DIR + ciscollector -r -config=.. < ../input-${{ matrix.postgres_version }}-${{ matrix.testcases }}.txt + - name: print postgres config + run: | + cat $TEMP_DIR/postgresql.conf + - name: Set container name + run: | + echo "CONTAINER_NAME=postgres-test-${{ matrix.postgres_version }}-${{ matrix.testcases }}" >> $GITHUB_ENV + - name: Run PostgreSQL Container + run: | + echo "Container name : ${CONTAINER_NAME}" + docker run --name $CONTAINER_NAME \ + -e POSTGRES_PASSWORD=mysecretpassword \ + -d postgres:${{ matrix.postgres_version }} + - name: Wait for PostgreSQL to be ready + run: sleep 2 + - name: Copy Custom Config + run: | + docker cp $TEMP_DIR/postgresql.conf $CONTAINER_NAME:/var/lib/postgresql/data/postgresql.conf + - name: Wait for PostgreSQL to be ready after config + run: sleep 2 + - name: Reload PostgreSQL with New Config without restart + run: | + docker exec $CONTAINER_NAME \ + psql -U postgres -c "SELECT pg_reload_conf();" + - name: Run Validation Script after reload + run: | + errors=$(docker exec $CONTAINER_NAME \ + psql -U postgres -t -c "SELECT COUNT(*) FROM pg_file_settings WHERE error IS NOT NULL;") + + if [ "$errors" -ne 0 ]; then + echo -e "\033[0;31mSome configuration settings require restart.\033[0m" + + docker exec $CONTAINER_NAME \ + psql -U postgres -c "SELECT * FROM pg_file_settings WHERE error IS NOT NULL;" + else + echo -e "\033[0;32mNo configuration errors found.\033[0m" + echo -e "\033[0;32mApplied configuration is.\033[0m" + docker exec $CONTAINER_NAME \ + psql -U postgres -c "SELECT * FROM pg_file_settings;" + fi + - name: Restart PostgreSQL Container + run: | + docker restart $CONTAINER_NAME + - name: Wait for PostgreSQL to be ready after restart + run: sleep 2 + - name: Run Validation Script after restart + run: | + container_status=$(docker inspect -f '{{.State.Running}}' $CONTAINER_NAME 2>/dev/null || echo "not_found") + + if [ "$container_status" != "true" ]; then + echo "Container is not running or not found. Attempting to print logs." + docker logs $CONTAINER_NAME || echo "No logs found or unable to retrieve logs." + exit 1 + fi + + errors=$(docker exec $CONTAINER_NAME \ + psql -U postgres -t -c "SELECT COUNT(*) FROM pg_file_settings WHERE error IS NOT NULL;") + + if [ "$errors" -ne 0 ]; then + echo "Container logs" + docker logs $CONTAINER_NAME + + echo -e "\033[0;31mConfiguration errors found in pg_file_settings.\033[0m" + + docker exec $CONTAINER_NAME \ + psql -U postgres -c "SELECT * FROM pg_file_settings WHERE error IS NOT NULL;" + exit 1 + else + echo -e "\033[0;32mNo configuration errors found.\033[0m" + echo -e "\033[0;32mApplied configuration is.\033[0m" + docker exec $CONTAINER_NAME \ + psql -U postgres -c "SELECT * FROM pg_file_settings;" + fi diff --git a/cmd/ciscollector/backuphistory.go b/cmd/ciscollector/backuphistory.go new file mode 100644 index 0000000..e93fdbd --- /dev/null +++ b/cmd/ciscollector/backuphistory.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "fmt" + "slices" + "sort" + "time" + + "github.com/klouddb/klouddbshield/htmlreport" + "github.com/klouddb/klouddbshield/pkg/backuphistory" +) + +type backupHistory struct { + backupHistoryInput backuphistory.BackupHistoryInput + htmlReportHelper *htmlreport.HtmlReportHelper +} + +func newBackupHistory(backupHistoryInput backuphistory.BackupHistoryInput, htmlReportHelper *htmlreport.HtmlReportHelper) *backupHistory { + return &backupHistory{ + backupHistoryInput: backupHistoryInput, + htmlReportHelper: htmlReportHelper, + } +} + +func (h *backupHistory) cronProcess(ctx context.Context) error { + return h.run(ctx) +} + +func (h *backupHistory) run(_ context.Context) error { + if h.backupHistoryInput.BackupFrequency == "" { + return fmt.Errorf("backup frequency is required") + } + + var backupHistory []time.Time + var err error + + switch h.backupHistoryInput.BackupTool { + case "pgbackrest": + backupHistory, err = backuphistory.GetBackupHistoryForPgBackrest() + if err != nil { + return err + } + case "pg_dump", "pg_dumpall", "pg_basebackup": + backupHistory, err = backuphistory.GetBackupHistory(h.backupHistoryInput.BackupPath) + if err != nil { + return err + } + default: + return fmt.Errorf("unsupported backup tool: %s", h.backupHistoryInput.BackupTool) + } + + if len(backupHistory) == 0 { + return fmt.Errorf("no backup history found") + } else if len(backupHistory) == 1 { + return fmt.Errorf("only one backup history found") + } + + sort.Slice(backupHistory, func(i, j int) bool { + return backupHistory[i].After(backupHistory[j]) + }) + + missingDates := []string{} + previousDate := backupHistory[len(backupHistory)-1] + + if h.backupHistoryInput.BackupFrequency == "daily" { + for i := len(backupHistory) - 2; i >= 0; i-- { + if previousDate.Format(time.DateOnly) == backupHistory[i].Format(time.DateOnly) { + continue + } + + nextDate := previousDate.AddDate(0, 0, 1) + for nextDate.Format(time.DateOnly) != backupHistory[i].Format(time.DateOnly) { + missingDates = append(missingDates, nextDate.Format(time.DateOnly)) + nextDate = nextDate.AddDate(0, 0, 1) + } + + previousDate = backupHistory[i] + } + } else if h.backupHistoryInput.BackupFrequency == "weekly" { + for i := len(backupHistory) - 2; i >= 0; i-- { + previousYear, previousWeek := previousDate.ISOWeek() + currentYear, currentWeek := backupHistory[i].ISOWeek() + + if previousYear == currentYear && previousWeek == currentWeek { + continue + } + + nextDate := previousDate.AddDate(0, 0, 7) + nextDateYear, nextDateWeek := nextDate.ISOWeek() + for nextDateYear != currentYear || nextDateWeek != currentWeek { + missingDates = append(missingDates, fmt.Sprintf("year %d - week %d", nextDateYear, nextDateWeek)) + nextDate = nextDate.AddDate(0, 0, 7) + nextDateYear, nextDateWeek = nextDate.ISOWeek() + } + + previousDate = backupHistory[i] + } + } else if h.backupHistoryInput.BackupFrequency == "monthly" { + for i := len(backupHistory) - 2; i >= 0; i-- { + if previousDate.Format("2006-01") == backupHistory[i].Format("2006-01") { + continue + } + + nextDate := previousDate.AddDate(0, 1, 0) + for nextDate.Format("2006-01") != backupHistory[i].Format("2006-01") { + missingDates = append(missingDates, nextDate.Format("2006-01")) + nextDate = nextDate.AddDate(0, 1, 0) + } + + previousDate = backupHistory[i] + } + } else { + return fmt.Errorf("unsupported backup frequency: %s", h.backupHistoryInput.BackupFrequency) + } + + uniqueMissingDates := []string{} + for _, v := range missingDates { + if !slices.Contains(uniqueMissingDates, v) { + uniqueMissingDates = append(uniqueMissingDates, v) + } + } + + output := backuphistory.BackupHistoryOutput{ + MissingDates: uniqueMissingDates, + StartDate: backupHistory[len(backupHistory)-1].Format("2006-01-02"), + EndDate: backupHistory[0].Format("2006-01-02"), + BackupFrequency: h.backupHistoryInput.BackupFrequency, + } + + backuphistory.PrintBackupHistory(output) + + h.htmlReportHelper.RegisterBackupHistory(output) + + return nil +} diff --git a/cmd/ciscollector/main.go b/cmd/ciscollector/main.go index ffecbe3..779247b 100644 --- a/cmd/ciscollector/main.go +++ b/cmd/ciscollector/main.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + _ "net/http/pprof" "os" "strings" "time" @@ -183,10 +184,14 @@ func main() { } } + if cnf.BackupHistoryInput.BackupTool != "" { + overviewErrorMap[cons.RootCMD_BackupAuditTool] = newBackupHistory(cnf.BackupHistoryInput, htmlReportHelper).run(ctx) + } + if cnf.CreatePostgresConfig { - overviewErrorMap[cons.RootCMD_CreatePostgresconfig] = postgresconfig.NewProcessor(".").Run(context.TODO()) - if overviewErrorMap[cons.RootCMD_CreatePostgresconfig] != nil { - fmt.Println("> Error while generating postgresql.conf file: ", text.FgHiRed.Sprint(overviewErrorMap[cons.RootCMD_CreatePostgresconfig])) + overviewErrorMap[cons.RootCMD_CreatePostgresConfig] = postgresconfig.NewProcessor(".").Run(context.TODO()) + if overviewErrorMap[cons.RootCMD_CreatePostgresConfig] != nil { + fmt.Println("> Error while generating postgresql.conf file: ", text.FgHiRed.Sprint(overviewErrorMap[cons.RootCMD_CreatePostgresConfig])) } } diff --git a/docker_testing/postgresconfig/input-13-1.txt b/docker_testing/postgresconfig/input-13-1.txt new file mode 100644 index 0000000..8257f69 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-1.txt @@ -0,0 +1,20 @@ +15 +Y +13 +4GB +32 +1 +100GB +1 +3 +100 +y +* +1GB +60 +-1 +2 +3 +off +60000 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-10.txt b/docker_testing/postgresconfig/input-13-10.txt new file mode 100644 index 0000000..e83cc72 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-10.txt @@ -0,0 +1,13 @@ +5 +13 +8GB +8 +1 +1 +3 +* +5432 +3 +1GB +y +y \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-2.txt b/docker_testing/postgresconfig/input-13-2.txt new file mode 100644 index 0000000..9d5daa4 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-2.txt @@ -0,0 +1,51 @@ +15 +ye +Ys +y +1.4 +13.0 +13 +0GB +1GB +4.0 +0 +1 +-1 +4 +1 +0GB +100GB +10 +-5 +1 +-100 +-10 +0 +-10 +0 +2 +yess +randomstring +y +first(abc) +0GB +1GB +1ms +1 +-2 +-10 +-1 +-2 +-10 +-1 +-2 +-10 +1 +onn +off +-2 +-10 +0 +-2 +-10 +0 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-3.txt b/docker_testing/postgresconfig/input-13-3.txt new file mode 100644 index 0000000..d97aad4 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-3.txt @@ -0,0 +1,50 @@ +15 +Ys +Ye +y +randomstring +13 +10 GB +4GB +1.5 +-1 +4 +2.5 +1Tb +1TB +0 +6 +2 +-1 +-1.5 +1 +1 +2.0 +3 +1 +0 +YES +ANY(s1,s2,s3) +2.0GB +5.8GB +1000000GB +2147484 +10 +11000 +-1ms +0 +110000000000000000 +1100 +0 +0 +262144 +2 +1 +-1 +on +-11000 +-1ms +1 +1Ms +0ms +10 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-4.txt b/docker_testing/postgresconfig/input-13-4.txt new file mode 100644 index 0000000..4283439 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-4.txt @@ -0,0 +1,33 @@ +15 +yES +13.0 +13 +4gB +0TB +40GB +8 +3 +0TB +4tb +100000000TB +3 +2.9 +9223372036854775808 +3 +3000000000 +randomstring +5 +YeS +FIRST(s1,s2) +100GB +randomstring +60 +0ms +1Ms +1 +1 +10 +262143 +oFF +10 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-5.txt b/docker_testing/postgresconfig/input-13-5.txt new file mode 100644 index 0000000..117a579 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-5.txt @@ -0,0 +1,31 @@ +15 +Noo +nNo +no +13 +8GB +10 +1 +5000000GB +4 +4 +300.0.0.1 +-10 +0 +1234 +-10 +0 +3 +2147483647 +YES +yEs +y +* +1TB +2147483 +10000 +100 +3 +on +60000 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-6.txt b/docker_testing/postgresconfig/input-13-6.txt new file mode 100644 index 0000000..0ff611b --- /dev/null +++ b/docker_testing/postgresconfig/input-13-6.txt @@ -0,0 +1,31 @@ +15 +no +13 +100GB +40 +2 +5000000TB +5 +20 +-10.-10.500.1 +999.999.999.999 +65536 +70000 +65535 +1 +2 +5 +3000000000 +10000 +No +yEs +y +first(abc) +10000000000TB +10000 +1000 +15 +10 +off +120000 +3600000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-7.txt b/docker_testing/postgresconfig/input-13-7.txt new file mode 100644 index 0000000..2e8af38 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-7.txt @@ -0,0 +1,15 @@ +15 +no +13 +16GB +4 +1 +100GB +1 +0 +999.999.999.999 +65535 +4 +1024 +y +n \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-8.txt b/docker_testing/postgresconfig/input-13-8.txt new file mode 100644 index 0000000..1d1bec5 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-8.txt @@ -0,0 +1,15 @@ +15 +no +13 +1GB +1 +1 +100GB +4 +0 +0.0.0.0 +5432 +4 +512 +y +y \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-13-9.txt b/docker_testing/postgresconfig/input-13-9.txt new file mode 100644 index 0000000..004b499 --- /dev/null +++ b/docker_testing/postgresconfig/input-13-9.txt @@ -0,0 +1,25 @@ +5 +13 +8GB +8 +1 +1 +3 +* +5432 +3 +1GB +y +y +y +* +1GB +N +off +60000 +1800000 +n +on +on +all +off \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-1.txt b/docker_testing/postgresconfig/input-14-1.txt new file mode 100644 index 0000000..0041c62 --- /dev/null +++ b/docker_testing/postgresconfig/input-14-1.txt @@ -0,0 +1,20 @@ +15 +Y +14 +4GB +32 +1 +100GB +1 +3 +100 +y +* +1GB +60 +-1 +2 +3 +off +60000 +1800000 diff --git a/docker_testing/postgresconfig/input-14-2.txt b/docker_testing/postgresconfig/input-14-2.txt new file mode 100644 index 0000000..9914082 --- /dev/null +++ b/docker_testing/postgresconfig/input-14-2.txt @@ -0,0 +1,51 @@ +15 +ye +Ys +y +1.4 +14.0 +14 +0GB +1GB +4.0 +0 +1 +-1 +4 +1 +0GB +100GB +10 +-5 +1 +-100 +-10 +0 +-10 +0 +2 +yess +randomstring +y +first(abc) +0GB +1GB +1ms +1 +-2 +-10 +-1 +-2 +-10 +-1 +-2 +-10 +1 +onn +off +-2 +-10 +0 +-2 +-10 +0 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-3.txt b/docker_testing/postgresconfig/input-14-3.txt new file mode 100644 index 0000000..d97aad4 --- /dev/null +++ b/docker_testing/postgresconfig/input-14-3.txt @@ -0,0 +1,50 @@ +15 +Ys +Ye +y +randomstring +13 +10 GB +4GB +1.5 +-1 +4 +2.5 +1Tb +1TB +0 +6 +2 +-1 +-1.5 +1 +1 +2.0 +3 +1 +0 +YES +ANY(s1,s2,s3) +2.0GB +5.8GB +1000000GB +2147484 +10 +11000 +-1ms +0 +110000000000000000 +1100 +0 +0 +262144 +2 +1 +-1 +on +-11000 +-1ms +1 +1Ms +0ms +10 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-4.txt b/docker_testing/postgresconfig/input-14-4.txt new file mode 100644 index 0000000..fb33b66 --- /dev/null +++ b/docker_testing/postgresconfig/input-14-4.txt @@ -0,0 +1,33 @@ +15 +yES +14.0 +14 +4gB +0TB +40GB +8 +3 +0TB +4tb +100000000TB +3 +2.9 +9223372036854775808 +3 +3000000000 +randomstring +5 +YeS +FIRST(s1,s2) +100GB +randomstring +60 +0ms +1Ms +1 +1 +10 +262143 +oFF +10 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-5.txt b/docker_testing/postgresconfig/input-14-5.txt new file mode 100644 index 0000000..b7426dd --- /dev/null +++ b/docker_testing/postgresconfig/input-14-5.txt @@ -0,0 +1,31 @@ +15 +Noo +nNo +no +14 +8GB +10 +1 +5000000GB +4 +4 +300.0.0.1 +-10 +0 +1234 +-10 +0 +3 +2147483647 +YES +yEs +y +* +1TB +2147483 +10000 +100 +3 +on +60000 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-6.txt b/docker_testing/postgresconfig/input-14-6.txt new file mode 100644 index 0000000..6a353bd --- /dev/null +++ b/docker_testing/postgresconfig/input-14-6.txt @@ -0,0 +1,31 @@ +15 +no +14 +100GB +40 +2 +5000000TB +5 +20 +-10.-10.500.1 +999.999.999.999 +65536 +70000 +65535 +1 +2 +5 +3000000000 +10000 +No +yEs +y +first(abc) +10000000000TB +10000 +1000 +15 +10 +off +120000 +3600000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-7.txt b/docker_testing/postgresconfig/input-14-7.txt new file mode 100644 index 0000000..1beb2dc --- /dev/null +++ b/docker_testing/postgresconfig/input-14-7.txt @@ -0,0 +1,15 @@ +15 +no +14 +32GB +32 +1 +100GB +1 +0 +0.0.0.0 +5432 +4 +2048 +y +y \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-8.txt b/docker_testing/postgresconfig/input-14-8.txt new file mode 100644 index 0000000..628e818 --- /dev/null +++ b/docker_testing/postgresconfig/input-14-8.txt @@ -0,0 +1,15 @@ +15 +no +14 +64GB +64 +1 +100GB +4 +0 +0.0.0.0 +5432 +4 +512 +y +n \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-14-9.txt b/docker_testing/postgresconfig/input-14-9.txt new file mode 100644 index 0000000..a4f79bb --- /dev/null +++ b/docker_testing/postgresconfig/input-14-9.txt @@ -0,0 +1,25 @@ +5 +14 +8GB +8 +1 +1 +3 +* +5432 +3 +1GB +y +y +y +* +1GB +N +off +60000 +1800000 +n +on +on +all +off \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-1.txt b/docker_testing/postgresconfig/input-15-1.txt new file mode 100644 index 0000000..966eaef --- /dev/null +++ b/docker_testing/postgresconfig/input-15-1.txt @@ -0,0 +1,20 @@ +15 +Y +15 +4GB +32 +1 +100GB +1 +3 +100 +y +* +1GB +60 +-1 +2 +3 +off +60000 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-2.txt b/docker_testing/postgresconfig/input-15-2.txt new file mode 100644 index 0000000..7b798b1 --- /dev/null +++ b/docker_testing/postgresconfig/input-15-2.txt @@ -0,0 +1,51 @@ +15 +ye +Ys +y +1.4 +13.0 +15 +0GB +1GB +4.0 +0 +1 +-1 +4 +1 +0GB +100GB +10 +-5 +1 +-100 +-10 +0 +-10 +0 +2 +yess +randomstring +y +first(abc) +0GB +1GB +1ms +1 +-2 +-10 +-1 +-2 +-10 +-1 +-2 +-10 +1 +onn +off +-2 +-10 +0 +-2 +-10 +0 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-3.txt b/docker_testing/postgresconfig/input-15-3.txt new file mode 100644 index 0000000..d97aad4 --- /dev/null +++ b/docker_testing/postgresconfig/input-15-3.txt @@ -0,0 +1,50 @@ +15 +Ys +Ye +y +randomstring +13 +10 GB +4GB +1.5 +-1 +4 +2.5 +1Tb +1TB +0 +6 +2 +-1 +-1.5 +1 +1 +2.0 +3 +1 +0 +YES +ANY(s1,s2,s3) +2.0GB +5.8GB +1000000GB +2147484 +10 +11000 +-1ms +0 +110000000000000000 +1100 +0 +0 +262144 +2 +1 +-1 +on +-11000 +-1ms +1 +1Ms +0ms +10 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-4.txt b/docker_testing/postgresconfig/input-15-4.txt new file mode 100644 index 0000000..827eaec --- /dev/null +++ b/docker_testing/postgresconfig/input-15-4.txt @@ -0,0 +1,33 @@ +15 +yES +15.0 +15 +4gB +0TB +40GB +8 +3 +0TB +4tb +100000000TB +3 +2.9 +9223372036854775808 +3 +3000000000 +randomstring +5 +YeS +FIRST(s1,s2) +100GB +randomstring +60 +0ms +1Ms +1 +1 +10 +262143 +oFF +10 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-5.txt b/docker_testing/postgresconfig/input-15-5.txt new file mode 100644 index 0000000..b7426dd --- /dev/null +++ b/docker_testing/postgresconfig/input-15-5.txt @@ -0,0 +1,31 @@ +15 +Noo +nNo +no +14 +8GB +10 +1 +5000000GB +4 +4 +300.0.0.1 +-10 +0 +1234 +-10 +0 +3 +2147483647 +YES +yEs +y +* +1TB +2147483 +10000 +100 +3 +on +60000 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-6.txt b/docker_testing/postgresconfig/input-15-6.txt new file mode 100644 index 0000000..d107b7f --- /dev/null +++ b/docker_testing/postgresconfig/input-15-6.txt @@ -0,0 +1,31 @@ +15 +no +15 +100GB +40 +2 +5000000TB +5 +20 +-10.-10.500.1 +999.999.999.999 +65536 +70000 +65535 +1 +2 +5 +3000000000 +10000 +No +yEs +y +first(abc) +10000000000TB +10000 +1000 +15 +10 +off +120000 +3600000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-7.txt b/docker_testing/postgresconfig/input-15-7.txt new file mode 100644 index 0000000..c30e014 --- /dev/null +++ b/docker_testing/postgresconfig/input-15-7.txt @@ -0,0 +1,15 @@ +15 +no +15 +1GB +1 +2 +100GB +3 +0 +0.0.0.0 +5432 +4 +1024 +y +y \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-8.txt b/docker_testing/postgresconfig/input-15-8.txt new file mode 100644 index 0000000..301fbd2 --- /dev/null +++ b/docker_testing/postgresconfig/input-15-8.txt @@ -0,0 +1,15 @@ +15 +no +15 +16GB +4 +2 +100GB +5 +0 +0.0.0.0 +5432 +4 +10 +y +n \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-15-9.txt b/docker_testing/postgresconfig/input-15-9.txt new file mode 100644 index 0000000..59995f4 --- /dev/null +++ b/docker_testing/postgresconfig/input-15-9.txt @@ -0,0 +1,25 @@ +5 +15 +8GB +8 +1 +1 +3 +* +5432 +3 +1GB +y +y +y +* +1GB +N +off +60000 +1800000 +n +on +on +all +off \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-1.txt b/docker_testing/postgresconfig/input-16-1.txt new file mode 100644 index 0000000..1c2e443 --- /dev/null +++ b/docker_testing/postgresconfig/input-16-1.txt @@ -0,0 +1,27 @@ +15 +n +16 +2GB +32 +1 +100GB +1 +3 +* +5432 +3 +100 +N +N +y +y +y +* +1GB +60 +-1 +2 +3 +off +100 +100 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-2.txt b/docker_testing/postgresconfig/input-16-2.txt new file mode 100644 index 0000000..2852789 --- /dev/null +++ b/docker_testing/postgresconfig/input-16-2.txt @@ -0,0 +1,51 @@ +15 +ye +Ys +y +1.4 +13.0 +16 +0GB +1GB +4.0 +0 +1 +-1 +4 +1 +0GB +100GB +10 +-5 +1 +-100 +-10 +0 +-10 +0 +2 +yess +randomstring +y +first(abc) +0GB +1GB +1ms +1 +-2 +-10 +-1 +-2 +-10 +-1 +-2 +-10 +1 +onn +off +-2 +-10 +0 +-2 +-10 +0 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-3.txt b/docker_testing/postgresconfig/input-16-3.txt new file mode 100644 index 0000000..d97aad4 --- /dev/null +++ b/docker_testing/postgresconfig/input-16-3.txt @@ -0,0 +1,50 @@ +15 +Ys +Ye +y +randomstring +13 +10 GB +4GB +1.5 +-1 +4 +2.5 +1Tb +1TB +0 +6 +2 +-1 +-1.5 +1 +1 +2.0 +3 +1 +0 +YES +ANY(s1,s2,s3) +2.0GB +5.8GB +1000000GB +2147484 +10 +11000 +-1ms +0 +110000000000000000 +1100 +0 +0 +262144 +2 +1 +-1 +on +-11000 +-1ms +1 +1Ms +0ms +10 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-4.txt b/docker_testing/postgresconfig/input-16-4.txt new file mode 100644 index 0000000..173864f --- /dev/null +++ b/docker_testing/postgresconfig/input-16-4.txt @@ -0,0 +1,33 @@ +15 +yES +16.0 +16 +4gB +0TB +40GB +8 +3 +0TB +4tb +100000000TB +3 +2.9 +9223372036854775808 +3 +3000000000 +randomstring +5 +YeS +FIRST(s1,s2) +100GB +randomstring +60 +0ms +1Ms +1 +1 +10 +262143 +oFF +10 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-5.txt b/docker_testing/postgresconfig/input-16-5.txt new file mode 100644 index 0000000..cbe552a --- /dev/null +++ b/docker_testing/postgresconfig/input-16-5.txt @@ -0,0 +1,31 @@ +15 +Noo +nNo +no +16 +8GB +10 +1 +5000000GB +4 +4 +300.0.0.1 +-10 +0 +1234 +-10 +0 +3 +2147483647 +YES +yEs +y +* +1TB +2147483 +10000 +100 +3 +on +60000 +1800000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-6.txt b/docker_testing/postgresconfig/input-16-6.txt new file mode 100644 index 0000000..bdf683a --- /dev/null +++ b/docker_testing/postgresconfig/input-16-6.txt @@ -0,0 +1,31 @@ +15 +no +16 +100GB +40 +2 +5000000TB +5 +20 +-10.-10.500.1 +999.999.999.999 +65536 +70000 +65535 +1 +2 +5 +3000000000 +10000 +No +yEs +y +first(abc) +10000000000TB +10000 +1000 +15 +10 +off +120000 +3600000 \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-7.txt b/docker_testing/postgresconfig/input-16-7.txt new file mode 100644 index 0000000..d342ab0 --- /dev/null +++ b/docker_testing/postgresconfig/input-16-7.txt @@ -0,0 +1,15 @@ +15 +no +16 +64GB +64 +2 +100GB +3 +0 +0.0.0.0 +5432 +4 +1024 +y +n \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-8.txt b/docker_testing/postgresconfig/input-16-8.txt new file mode 100644 index 0000000..31ea3c9 --- /dev/null +++ b/docker_testing/postgresconfig/input-16-8.txt @@ -0,0 +1,15 @@ +15 +no +16 +16GB +4 +3 +100GB +1 +0 +0.0.0.0 +5432 +4 +512 +y +n \ No newline at end of file diff --git a/docker_testing/postgresconfig/input-16-9.txt b/docker_testing/postgresconfig/input-16-9.txt new file mode 100644 index 0000000..646b291 --- /dev/null +++ b/docker_testing/postgresconfig/input-16-9.txt @@ -0,0 +1,25 @@ +5 +16 +8GB +8 +1 +1 +3 +* +5432 +3 +1GB +y +y +y +* +1GB +N +off +60000 +1800000 +n +on +on +all +off \ No newline at end of file diff --git a/docker_testing/postgresconfig/kshieldconfig.toml b/docker_testing/postgresconfig/kshieldconfig.toml new file mode 100644 index 0000000..e813dd7 --- /dev/null +++ b/docker_testing/postgresconfig/kshieldconfig.toml @@ -0,0 +1,8 @@ +[postgres] +host="localhost" +port="5432" +user="postgres" +dbname="postgres" +password="xxxxx" +maxIdleConn = 2 +maxOpenConn = 2 \ No newline at end of file diff --git a/go.mod b/go.mod index 656e61f..ef35ddf 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/golang-migrate/migrate/v4 v4.17.1 github.com/google/uuid v1.4.0 github.com/hashicorp/go-version v1.6.0 + github.com/muesli/termenv v0.15.2 github.com/olekukonko/tablewriter v0.0.5 github.com/robfig/cron v1.2.0 github.com/robfig/cron/v3 v3.0.1 @@ -21,19 +22,21 @@ require ( github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 go.uber.org/mock v0.2.0 golang.org/x/net v0.26.0 - golang.org/x/sync v0.8.0 - golang.org/x/term v0.21.0 + golang.org/x/sync v0.10.0 + golang.org/x/term v0.27.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df ) require ( github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-openapi/errors v0.20.2 // indirect github.com/go-openapi/strfmt v0.21.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/oklog/ulid v1.3.1 // indirect @@ -41,7 +44,7 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect go.mongodb.org/mongo-driver v1.12.1 // indirect go.uber.org/atomic v1.9.0 // indirect - golang.org/x/crypto v0.24.0 // indirect + golang.org/x/crypto v0.31.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect ) @@ -61,8 +64,8 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 01892b2..a0569e5 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -180,6 +182,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= @@ -199,6 +203,8 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= @@ -298,8 +304,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -391,8 +397,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -436,13 +442,13 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -453,8 +459,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/htmlreport/backup_history_helper.go b/htmlreport/backup_history_helper.go new file mode 100644 index 0000000..4c3fd89 --- /dev/null +++ b/htmlreport/backup_history_helper.go @@ -0,0 +1,7 @@ +package htmlreport + +import "github.com/klouddb/klouddbshield/pkg/backuphistory" + +func (h *HtmlReportHelper) RegisterBackupHistory(output backuphistory.BackupHistoryOutput) { + h.AddTab("Backup Audit Tool", output) +} diff --git a/htmlreport/piireport.go b/htmlreport/piireport.go index 50349f0..8df9bc1 100644 --- a/htmlreport/piireport.go +++ b/htmlreport/piireport.go @@ -3,5 +3,13 @@ package htmlreport import "github.com/klouddb/klouddbshield/pkg/piiscanner" func (h *HtmlReportHelper) RegisterPIIReport(result *piiscanner.DatabasePIIScanOutput) { + if result == nil { + return + } + + if len(result.Data) == 0 { + return + } + h.AddTab("PII Scanner Report", result) } diff --git a/htmlreport/template/backup_history.tmpl b/htmlreport/template/backup_history.tmpl new file mode 100644 index 0000000..b086357 --- /dev/null +++ b/htmlreport/template/backup_history.tmpl @@ -0,0 +1,58 @@ +{{ define "backupAuditToolTab" }} +
+
+

PostgreSQL Backup Audit Tool

+ +
+
+

Database {{ .BackupFrequency }} backup scan results for period: {{ .StartDate }} to {{ .EndDate }}

+
+ +
Backup Timeline
+ {{ if eq (len .MissingDates) 0 }} +
+ {{ if eq .BackupFrequency "daily" }} +

No missing daily backups found in the specified period. Your backup schedule is being followed correctly.

+ {{ else if eq .BackupFrequency "weekly" }} +

No missing weekly backups found in the specified period. Your backup schedule is being followed correctly.

+ {{ else if eq .BackupFrequency "monthly" }} +

No missing monthly backups found in the specified period. Your backup schedule is being followed correctly.

+ {{ end }} +
+ {{ else }} +
+ {{ if eq .BackupFrequency "daily" }} + Warning: Found {{ len .MissingDates }} days with missing backups + {{ else if eq .BackupFrequency "weekly" }} + Warning: Found {{ len .MissingDates }} weeks with missing backups + {{ else if eq .BackupFrequency "monthly" }} + Warning: Found {{ len .MissingDates }} months with missing backups + {{ end }} +
+ + + + + {{ if eq .BackupFrequency "daily" }} + + {{ else if eq .BackupFrequency "weekly" }} + + {{ else if eq .BackupFrequency "monthly" }} + + {{ end }} + + + + {{ range $index, $date := .MissingDates }} + + + + + {{ end }} + +
No.Missing DaysMissing WeeksMissing Months
{{ add $index 1 }}{{ $date }}
+ {{ end }} +
+
+
+{{ end }} \ No newline at end of file diff --git a/htmlreport/template/tab_mapping.tmpl b/htmlreport/template/tab_mapping.tmpl index 27d0f10..0eca665 100644 --- a/htmlreport/template/tab_mapping.tmpl +++ b/htmlreport/template/tab_mapping.tmpl @@ -23,6 +23,8 @@ {{ template "transactionWraparound" .Body }} {{ else if eq .Title "SSL Report" }} {{ template "sslAuditTab" .Body }} + {{ else if eq .Title "Backup Audit Tool" }} + {{ template "backupAuditToolTab" .Body }} {{ end }} {{ end }} diff --git a/pkg/backuphistory/backuphistory.go b/pkg/backuphistory/backuphistory.go new file mode 100644 index 0000000..8690c71 --- /dev/null +++ b/pkg/backuphistory/backuphistory.go @@ -0,0 +1,393 @@ +package backuphistory + +import ( + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +var weekDayMap = map[string]int{ + "Monday": 1, + "Tuesday": 2, + "Wednesday": 3, + "Thursday": 4, + "Friday": 5, + "Saturday": 6, + "Sunday": 7, + "Mon": 1, + "Tue": 2, + "Wed": 3, + "Thu": 4, + "Fri": 5, + "Sat": 6, + "Sun": 7, +} + +var monthMap = map[string]int{ + "January": 1, + "February": 2, + "March": 3, + "April": 4, + "May": 5, + "June": 6, + "July": 7, + "August": 8, + "September": 9, + "October": 10, + "November": 11, + "December": 12, + "Jan": 1, + "Feb": 2, + "Mar": 3, + "Apr": 4, + "Jun": 6, + "Jul": 7, + "Aug": 8, + "Sep": 9, + "Oct": 10, + "Nov": 11, + "Dec": 12, +} + +var mapForPrefix = map[string]placeholderUpdate{ + "%Y": { + Regex: `\d{4}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Year, err = strconv.Atoi(s) + return err + }, + }, + "%y": { + Regex: `\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Year, err = strconv.Atoi(s) + return err + }, + }, + "%m": { + Regex: `\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Month, err = strconv.Atoi(s) + return err + }, + }, + "%d": { + Regex: `\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Day, err = strconv.Atoi(s) + return err + }, + }, + "%H": { + Regex: `\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Hour, err = strconv.Atoi(s) + if err != nil { + return err + } + if t.Hour >= 12 { + t.Add12InHour = true + t.Hour -= 12 + } + return nil + }, + }, + "%M": { + Regex: `\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Minute, err = strconv.Atoi(s) + return err + }, + }, + "%S": { + Regex: `\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Second, err = strconv.Atoi(s) + return err + }, + }, + "%A": { + Regex: `\w+`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + t.Weekday = weekDayMap[s] + return nil + }, + }, + "%a": { + Regex: `\w+`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + t.Weekday = weekDayMap[s] + return nil + }, + }, + "%w": { + Regex: `\d{1,2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + t.Weekday = weekDayMap[s] + return nil + }, + }, + "%W": { + Regex: `\d{1,2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + t.WeekNum, _ = strconv.Atoi(s) + return nil + }, + }, + "%B": { + Regex: `\w+`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + t.Month = monthMap[s] + return nil + }, + }, + "%b": { + Regex: `\w+`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + t.Month = monthMap[s] + return nil + }, + }, + "%p": { + Regex: `\w+`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + t.Add12InHour = s == "P" + return nil + }, + }, + "%I": { + Regex: `\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + var err error + t.Hour, err = strconv.Atoi(s) + return err + }, + }, + "%T": { + Regex: `\d{2}:\d{2}:\d{2}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + v := strings.Split(s, ":") + var err error + t.Hour, err = strconv.Atoi(v[0]) + if err != nil { + return err + } + + if t.Hour >= 12 { + t.Add12InHour = true + t.Hour -= 12 + } + + t.Minute, err = strconv.Atoi(v[1]) + if err != nil { + return err + } + + t.Second, err = strconv.Atoi(v[2]) + if err != nil { + return err + } + return nil + }, + }, + "%D": { + Regex: `\d{2}/\d{2}/\d{4}`, + TimeParserFunction: func(s string, t *backupParsedTime) error { + v := strings.Split(s, "/") + var err error + t.Day, err = strconv.Atoi(v[0]) + if err != nil { + return err + } + t.Month, err = strconv.Atoi(v[1]) + if err != nil { + return err + } + t.Year, err = strconv.Atoi(v[2]) + if err != nil { + return err + } + return nil + }, + }, +} + +type BackupHistoryInput struct { + BackupTool string + BackupPath string + BackupFrequency string +} + +type BackupHistoryOutput struct { + MissingDates []string + StartDate string + EndDate string + BackupFrequency string +} + +type placeholderUpdate struct { + Regex string + TimeParserFunction func(string, *backupParsedTime) error +} + +type backupParsedTime struct { + Year int + Month int + Day int + Hour int + Minute int + Second int + Add12InHour bool + + Weekday int + WeekNum int +} + +func (t *backupParsedTime) GetTime() (time.Time, error) { + if t.Year != 0 && t.Month != 0 && t.Day != 0 { + hour := t.Hour + if t.Add12InHour { + hour += 12 + } + return time.Date(t.Year, time.Month(t.Month), t.Day, hour, t.Minute, t.Second, 0, time.Local), nil + } + + if t.Weekday != 0 && t.WeekNum != 0 && t.Year != 0 { + days := (t.Weekday - 1) * 7 + days += t.WeekNum + + t := time.Date(t.Year, time.January, 1, 0, 0, 0, 0, time.Local) + t = t.AddDate(0, 0, days) + return t, nil + } + + return time.Time{}, fmt.Errorf("invalid time") +} + +func GetBackupHistory(backupPath string) ([]time.Time, error) { + + if backupPath == "" { + return nil, fmt.Errorf("backup path is required") + } + + patternBuilder := strings.Builder{} + regBuilder := strings.Builder{} + + ind := []string{} + + for i := 0; i < len(backupPath); i++ { + if backupPath[i] == '%' { + patternBuilder.WriteString("*") + regBuilder.WriteString(fmt.Sprintf("(%s)", mapForPrefix[string(backupPath[i:i+2])].Regex)) + ind = append(ind, string(backupPath[i:i+2])) + i += 1 + } else { + patternBuilder.WriteString(string(backupPath[i])) + regBuilder.WriteString(string(backupPath[i])) + } + } + + if len(ind) == 0 { + return nil, fmt.Errorf("no placeholders found in backup path") + } + + pattern := patternBuilder.String() + reg := regBuilder.String() + + files, err := filepath.Glob(pattern) + if err != nil { + log.Fatal(err) + } + + re := regexp.MustCompile(reg) + backupHistory := []time.Time{} + + for _, file := range files { + if re.MatchString(file) { + v := re.FindAllStringSubmatch(file, -1) + + backupParsedTime := backupParsedTime{} + for i := 0; i < len(ind); i++ { + if err := mapForPrefix[ind[i]].TimeParserFunction(v[0][i+1], &backupParsedTime); err != nil { + return nil, fmt.Errorf("failed to parse time: %v", err) + } + } + + totalSize := 0 + // get size of all files in this directory + files, err := os.ReadDir(file) + if err != nil { + return nil, err + } + + for _, file := range files { + info, err := file.Info() + if err != nil { + return nil, err + } + totalSize += int(info.Size()) + } + + if totalSize == 0 { + continue + } + + t, err := backupParsedTime.GetTime() + if err != nil { + return nil, err + } + + backupHistory = append(backupHistory, t) + } + } + + return backupHistory, nil +} + +// GetBackupHistoryForPgBackrest returns the history of backups using +// pgbackrest. +// +// e.g pgbackrest info +func GetBackupHistoryForPgBackrest() ([]time.Time, error) { + cmd := exec.Command("pgbackrest", "info") + output, err := cmd.Output() + if err != nil { + if len(output) == 0 { + return nil, fmt.Errorf("failed to run pgbackrest info: %v", err) + } + return nil, fmt.Errorf("failed to run pgbackrest info: %v", string(output)) + } + + dateRegex := `(\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2})(?:\+\d{2}:\d{2})?` + re := regexp.MustCompile(`timestamp\s+start/stop:\s+` + dateRegex + `\s+/\s+` + dateRegex) + + matches := re.FindAllStringSubmatch(string(output), -1) + + backupHistory := []time.Time{} + for _, match := range matches { + if len(match) != 3 { + continue + } + + start, err := time.Parse(time.DateTime, match[1]) + if err != nil { + return nil, err + } + backupHistory = append(backupHistory, start) + } + + return backupHistory, nil +} diff --git a/pkg/backuphistory/terminal.go b/pkg/backuphistory/terminal.go new file mode 100644 index 0000000..7d51150 --- /dev/null +++ b/pkg/backuphistory/terminal.go @@ -0,0 +1,22 @@ +package backuphistory + +import ( + "fmt" +) + +func PrintBackupHistory(output BackupHistoryOutput) { + fmt.Println("Backup Scanning Period: " + output.StartDate + " - " + output.EndDate) + + if len(output.MissingDates) == 0 { + fmt.Println("No missing dates found") + return + } + + fmt.Println("Backup History:") + + for _, v := range output.MissingDates { + fmt.Println("> " + v) + } + + fmt.Println("") +} diff --git a/pkg/config/config.go b/pkg/config/config.go index df115a1..9cf3182 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/viper" "github.com/klouddb/klouddbshield/model" + "github.com/klouddb/klouddbshield/pkg/backuphistory" cons "github.com/klouddb/klouddbshield/pkg/const" "github.com/klouddb/klouddbshield/pkg/piiscanner" "github.com/klouddb/klouddbshield/pkg/postgresdb" @@ -52,6 +53,10 @@ type Config struct { CompareConfig []string `toml:"compare-config"` CompareConfigBaseServer string `toml:"compare-config-base-server"` + + // BackupPath is the path to the backup directory which we will use to + // understand if we are taking backup on daily basis or not + BackupHistoryInput backuphistory.BackupHistoryInput `toml:"-"` } func NewPiiInteractiveMode(pgConfig *postgresdb.Postgres, printAll, spacyOnly, summary bool) (*piiscanner.Config, error) { @@ -59,21 +64,19 @@ func NewPiiInteractiveMode(pgConfig *postgresdb.Postgres, printAll, spacyOnly, s return nil, fmt.Errorf(cons.Err_PostgresConfig_Missing) } - reader := newInputReader() - var readOption string if !spacyOnly { - readOption = strings.TrimSpace(reader.Read("Please enter run option", piiscanner.RunOption_DataScan_String)) + readOption = strings.TrimSpace(ReadInput("Please enter run option", piiscanner.RunOption_DataScan_String)) _, ok := piiscanner.RunOptionMap[readOption] if !ok { return nil, fmt.Errorf("invalid run option %s, valid options are %s", readOption, strings.Join(piiscanner.RunOptionSlice(), ", ")) } } - readExcludeTable := strings.TrimSpace(reader.Read("Please enter exclude tables ( e.g table1,table2,table3 )", "")) - readIncludeTable := strings.TrimSpace(reader.Read("Please enter include tables ( e.g table1,table2,table3 )", "")) + readExcludeTable := strings.TrimSpace(ReadInput("Please enter exclude tables ( e.g table1,table2,table3 )", "")) + readIncludeTable := strings.TrimSpace(ReadInput("Please enter include tables ( e.g table1,table2,table3 )", "")) - readDatabase := strings.TrimSpace(reader.Read("Please enter database name", pgConfig.DBName)) - readSchema := strings.TrimSpace(reader.Read("Please enter schema name", "public")) + readDatabase := strings.TrimSpace(ReadInput("Please enter database name", pgConfig.DBName)) + readSchema := strings.TrimSpace(ReadInput("Please enter schema name", "public")) fmt.Println() @@ -357,19 +360,24 @@ func NewConfig() (*Config, error) { flag.BoolVar(&transactionWraparound, "transaction-wraparound", transactionWraparound, "Generate transaction wraparound report") var createPostgresConfig bool - // flag.BoolVar(&createPostgresConfig, "create-postgres-config", false, "Create postgres config") + flag.BoolVar(&createPostgresConfig, "create-postgres-config", false, "Create postgres config") var configAudit bool - // flag.BoolVar(&configAudit, "config-audit", configAudit, "Config audit") + flag.BoolVar(&configAudit, "config-audit", configAudit, "Config audit") var sslCheck bool flag.BoolVar(&sslCheck, "ssl-check", sslCheck, "SSL check") + var backupHistoryInput backuphistory.BackupHistoryInput + flag.StringVar(&backupHistoryInput.BackupPath, "backup-path", "", "Backup path") + flag.StringVar(&backupHistoryInput.BackupTool, "backup-tool", "", "Backup tool") + flag.StringVar(&backupHistoryInput.BackupFrequency, "backup-frequency", "", "Backup frequency") + var compareConfig compareConfigFlag - // flag.Var(&compareConfig, "compare-config", "Connection strings for multiple PostgreSQL servers to compare (can be specified multiple times)") + flag.Var(&compareConfig, "compare-config", "Connection strings for multiple PostgreSQL servers to compare (can be specified multiple times)") - // var compareConfigBaseServer string - // flag.StringVar(&compareConfigBaseServer, "compare-config-base-server", "", "Base server for comparison") + var compareConfigBaseServer string + flag.StringVar(&compareConfigBaseServer, "compare-config-base-server", "", "Base server for comparison") flag.Parse() @@ -400,7 +408,7 @@ func NewConfig() (*Config, error) { !spacyOnly && !configAudit && !sslCheck && !transactionWraparound && !runPostgres && !runMySql && !runRds && !hbaScanner && !runPostgresConnTest && !runGeneratePassword && !runGenerateEncryptedPassword && - !runPwnedUsers && !runPwnedPassword && + !runPwnedUsers && !runPwnedPassword && backupHistoryInput.BackupTool == "" && !createPostgresConfig && len(compareConfig) == 0 { fmt.Println("> For Help: " + text.FgGreen.Sprint("ciscollector --help")) os.Exit(0) @@ -529,15 +537,64 @@ func NewConfig() (*Config, error) { case cons.SelectionIndex_Exit: // Exit os.Exit(0) - // case cons.SelectionIndex_CreatePostgresConfig: - // createPostgresConfig = true + case cons.SelectionIndex_CreatePostgresConfig: + createPostgresConfig = true + + case cons.SelectionIndex_ConfigAuditing: + configAudit = true + + case cons.SelectionIndex_CompareConfig: + compareConfigBaseServer = ReadInput("Enter the base server for comparison", "") + if compareConfigBaseServer == "" { + fmt.Println("Base server is required") + os.Exit(1) + } + + configs := ReadInput("Enter the connection strings for the servers to compare (can be specified comma separated)", "") + if configs == "" { + fmt.Println("No connection strings provided") + os.Exit(1) + } + + for _, config := range strings.Split(configs, ",") { + config = strings.TrimSpace(config) + if config == "" { + continue + } + compareConfig = append(compareConfig, config) + } - // case cons.SelectionIndex_ConfigAudit: - // configAudit = true + if len(compareConfig) == 0 { + fmt.Println("No connection strings provided") + os.Exit(1) + } + + fmt.Println(compareConfig) case cons.SelectionIndex_SSLCheck: sslCheck = true + case cons.SelectionIndex_BackupAuditTool: + backupHistoryInput.BackupTool = ReadInput("Enter the backup tool (e.g pg_dump, pg_basebackup, pgbackrest)", "pg_dump") + if backupHistoryInput.BackupTool != "pgbackrest" { + backupHistoryInput.BackupPath = ReadInput("Enter the backup path (e.g /path/to/backup)", "") + } + backupHistoryInput.BackupFrequency = ReadInput("Enter the backup frequency (e.g daily, weekly, monthly)", "") + + if backupHistoryInput.BackupTool == "" || backupHistoryInput.BackupFrequency == "" { + fmt.Println("Backup tool and frequency are required") + os.Exit(1) + } + + if backupHistoryInput.BackupTool != "pgbackrest" && backupHistoryInput.BackupTool != "pg_dump" && backupHistoryInput.BackupTool != "pg_dumpall" && backupHistoryInput.BackupTool != "pg_basebackup" { + fmt.Println("Invalid backup tool. Supported tools are pg_dump, pg_dumpall, pg_basebackup") + os.Exit(1) + } + if backupHistoryInput.BackupTool != "pgbackrest" && backupHistoryInput.BackupPath == "" { + fmt.Println("Backup path is required for " + backupHistoryInput.BackupTool) + os.Exit(1) + } + default: fmt.Println("Invalid Choice, Please Try Again.") os.Exit(1) @@ -546,8 +603,9 @@ func NewConfig() (*Config, error) { c.PiiScannerConfig = piiConfig c.PostgresCheckSet = utils.NewDummyContainsAllSet[string]() - // c.CreatePostgresConfig = createPostgresConfig - // c.ConfigAudit = configAudit + c.BackupHistoryInput = backupHistoryInput + c.CreatePostgresConfig = createPostgresConfig + c.ConfigAudit = configAudit c.SSLCheck = sslCheck if c.CustomTemplate != "" { var checkNumbers []string @@ -690,11 +748,15 @@ func NewConfig() (*Config, error) { case cons.SelectionIndex_Exit: os.Exit(0) - // case cons.SelectionIndex_CreatePostgresConfig: - // createPostgresConfig = true + case cons.SelectionIndex_CreatePostgresConfig: + createPostgresConfig = true - // case cons.SelectionIndex_ConfigAudit: - // configAudit = true + case cons.SelectionIndex_ConfigAuditing: + c.ConfigAudit = true + + case cons.SelectionIndex_CompareConfig: + fmt.Println("Verbose feature is not available for Compare Config yet .. Will be added in future releases") + os.Exit(1) case cons.SelectionIndex_SSLCheck: fmt.Println("Verbose feature is not available for SSL Check yet .. Will be added in future releases") @@ -713,7 +775,7 @@ func NewConfig() (*Config, error) { return nil, fmt.Errorf("getting hostname: %v", err) } } - if c.MySQL == nil && c.Postgres == nil && !runRds && c.LogParser == nil { + if c.MySQL == nil && c.Postgres == nil && !runRds && c.LogParser == nil && c.BackupHistoryInput.BackupTool == "" { return nil, fmt.Errorf(cons.Err_PostgresConfig_Missing) } if c.MySQL != nil && c.Postgres != nil && !runRds { @@ -766,13 +828,11 @@ func NewConfig() (*Config, error) { } } - // c.CompareConfig = compareConfig - // c.CompareConfigBaseServer = compareConfigBaseServer - // if compareConfigBaseServer != "" && len(compareConfig) == 0 { - // return nil, fmt.Errorf("compare-config flag requires at least one connection string") - // } else if compareConfigBaseServer == "" && len(compareConfig) > 1 { - // return nil, fmt.Errorf("compare-config flag requires only one connection string when compare-config-base-server is not provided") - // } + c.CompareConfig = compareConfig + c.CompareConfigBaseServer = compareConfigBaseServer + if c.CompareConfigBaseServer != "" && len(c.CompareConfig) == 0 { + return nil, fmt.Errorf("with base server, at least one connection string is required") + } return c, nil } @@ -801,23 +861,15 @@ func LoadConfig(configPath string) (*Config, error) { return c, nil } -type inputReader struct { - reader *bufio.Reader -} - -func newInputReader() *inputReader { - return &inputReader{ - reader: bufio.NewReader(os.Stdin), - } -} +func ReadInput(msg, detault string) string { + reader := bufio.NewReader(os.Stdin) -func (i *inputReader) Read(msg, detault string) string { fmt.Print("> " + msg) if detault != "" { fmt.Print(" [" + detault + "]") } fmt.Print(": ") - input, err := i.reader.ReadString('\n') + input, err := reader.ReadString('\n') if err != nil { fmt.Println("Invalid input for logparser:", err) os.Exit(1) @@ -862,12 +914,10 @@ func getLogParserInputs(postgresConf *postgresdb.Postgres, command string) (*Log } } - reader := newInputReader() - - prefix := reader.Read("Enter Log Line Prefix", prefixSuggestion) - logfile := reader.Read("Enter Log File Path", logfileSuggestion) - beginTime := reader.Read("Enter Begin Time (format: 2006-01-02 15:04:05) [optional]", "") - endTime := reader.Read("Enter End Time (format: 2006-01-02 15:04:05) [optional]", "") + prefix := ReadInput("Enter Log Line Prefix", prefixSuggestion) + logfile := ReadInput("Enter Log File Path", logfileSuggestion) + beginTime := ReadInput("Enter Begin Time (format: 2006-01-02 15:04:05) [optional]", "") + endTime := ReadInput("Enter End Time (format: 2006-01-02 15:04:05) [optional]", "") // var ipfile string // if command == cons.LogParserCMD_MismatchIPs { @@ -876,7 +926,7 @@ func getLogParserInputs(postgresConf *postgresdb.Postgres, command string) (*Log var hbaConfigFile string if command == cons.LogParserCMD_HBAUnusedLines || command == cons.LogParserCMD_All { - hbaConfigFile = reader.Read("Enter pg_hba.conf File Path", hbaConfigSuggestion) + hbaConfigFile = ReadInput("Enter pg_hba.conf File Path", hbaConfigSuggestion) } l, err := NewLogParser(command, beginTime, endTime, prefix, logfile, hbaConfigFile) diff --git a/pkg/config/help.go b/pkg/config/help.go index 7fcdead..45f7e1c 100644 --- a/pkg/config/help.go +++ b/pkg/config/help.go @@ -248,6 +248,11 @@ func PrintHelp() { fmt.Println(text.FgCyan.Sprint(`maxIdleConn = 2`)) fmt.Println(text.FgCyan.Sprint(`maxOpenConn = 2`)) + case cons.SelectionIndex_BackupAuditTool, + cons.SelectionIndex_CreatePostgresConfig, + cons.SelectionIndex_ConfigAuditing, + cons.SelectionIndex_CompareConfig: + default: fmt.Println("Invalid Choice, Please Try Again.") os.Exit(1) diff --git a/pkg/const/const.go b/pkg/const/const.go index 0f48846..6f611a8 100644 --- a/pkg/const/const.go +++ b/pkg/const/const.go @@ -37,6 +37,7 @@ const ( SelectionIndex_PostgresChecks SelectionIndex_HBAScanner SelectionIndex_PIIScanner + SelectionIndex_CreatePostgresConfig SelectionIndex_InactiveUsers SelectionIndex_UniqueIPs SelectionIndex_HBAUnusedLines @@ -46,9 +47,10 @@ const ( SelectionIndex_AWSAurora SelectionIndex_MySQL SelectionIndex_TransactionWraparound - // SelectionIndex_CreatePostgresconfig - // SelectionIndex_ConfigAuditing SelectionIndex_SSLCheck + SelectionIndex_ConfigAuditing + SelectionIndex_CompareConfig + SelectionIndex_BackupAuditTool SelectionIndex_Exit ) @@ -74,46 +76,62 @@ var CommandList = []CommandTitle{ Title: "Postgres PII report", }, { // 5 + CMD: RootCMD_CreatePostgresConfig, + Title: "Postgres Config Generator", + }, + { // 6 CMD: LogParserCMD_InactiveUser, Title: "Inactive user report", }, - { // 6 + { // 7 CMD: LogParserCMD_UniqueIPs, Title: "Client ip report", }, - { // 7 + { // 8 CMD: LogParserCMD_HBAUnusedLines, Title: "HBA unused lines report", }, - { // 8 + { // 9 CMD: RootCMD_PasswordManager, Title: "Password Manager", }, - { // 9 + { // 10 CMD: LogParserCMD_PasswordLeakScanner, Title: "Password leak scanner", }, - { // 10 + { // 11 CMD: RootCMD_AWSRDS, Title: "AWS RDS Sec Report", }, - { // 11 + { // 12 CMD: RootCMD_AWSAurora, Title: "AWS Aurora Sec Report", }, - { // 12 + { // 13 CMD: RootCMD_MySQL, Title: "MySQL Report", }, - { // 13 + { // 14 CMD: RootCMD_TransactionWraparound, Title: "Transaction Wraparound Report", }, - { // 14 + { // 15 CMD: RootCMD_SSLCheck, Title: "SSL Check", }, - { // 15 + { // 16 + CMD: RootCMD_ConfigAuditing, + Title: "Config Audit", + }, + { // 17 + CMD: RootCMD_CompareConfig, + Title: "Compare Config", + }, + { // 18 + CMD: RootCMD_BackupAuditTool, + Title: "Backup Audit Tool", + }, + { // 19 CMD: RootCMD_Exit, Title: "Exit", }, @@ -151,11 +169,11 @@ const ( RootCMD_MySQL = "mysql" RootCMD_PiiScanner = "pii_scanner" RootCMD_TransactionWraparound = "transaction_wraparound" - RootCMD_CreatePostgresconfig = "create_postgresconfig" + RootCMD_CreatePostgresConfig = "create_postgresconfig" RootCMD_ConfigAuditing = "config_auditing" RootCMD_SSLCheck = "ssl_check" RootCMD_Exit = "exit" - + RootCMD_BackupAuditTool = "backup_audit_tool" // LogParserCMD_MismatchIPs = "mismatch_ips" LogParserCMD_UniqueIPs = "unique_ip" LogParserCMD_InactiveUser = "inactive_users" diff --git a/postgres/configaudit/audit.go b/postgres/configaudit/audit.go index 5dc2fc3..85fb5f7 100644 --- a/postgres/configaudit/audit.go +++ b/postgres/configaudit/audit.go @@ -10,6 +10,8 @@ import ( "github.com/klouddb/klouddbshield/pkg/utils" ) +var LogLinePrefixSubstrings = []string{"%m", "%p", "%q", "%u", "%d", "%a"} + func AuditConfig(ctx context.Context, store *sql.DB) ([]*model.ConfigAuditResult, error) { out := make([]*model.ConfigAuditResult, 0, 5) @@ -45,6 +47,42 @@ func AuditConfig(ctx context.Context, store *sql.DB) ([]*model.ConfigAuditResult } out = append(out, result) + result, err = FullPageWrites(ctx, store) + if err != nil { + return nil, err + } + out = append(out, result) + + result, err = MaxWalSize(ctx, store) + if err != nil { + return nil, err + } + out = append(out, result) + + result, err = LogLinePrefix(ctx, store) + if err != nil { + return nil, err + } + out = append(out, result) + + result, err = LogConnections(ctx, store) + if err != nil { + return nil, err + } + out = append(out, result) + + result, err = StatementTimeout(ctx, store) + if err != nil { + return nil, err + } + out = append(out, result) + + result, err = IdleInTransationSessionTimeout(ctx, store) + if err != nil { + return nil, err + } + out = append(out, result) + return out, nil } @@ -202,3 +240,200 @@ func TempFileLimit(ctx context.Context, store *sql.DB) (*model.ConfigAuditResult } return result, nil // Placeholder, replace with actual implementation } + +func FullPageWrites(ctx context.Context, store *sql.DB) (*model.ConfigAuditResult, error) { + + result := &model.ConfigAuditResult{ + Name: "FullPageWrites", + Status: "Pass", + FailReason: "", // Placeholder, replace with actual implementation when implemented in CheckFsyncFlag function. + } + + // If full_page_writes is set to off - CRITICAL + query := `SELECT name, setting + FROM pg_settings + WHERE name = 'full_page_writes';` + + data, err := utils.GetJSON(store, query) + if err != nil { + result.Status = "Fail" + result.FailReason = err.Error() + return result, nil + } + + for _, obj := range data { + if obj["setting"] != nil && fmt.Sprint(obj["setting"]) == "off" { + result.Status = "Critical" + result.FailReason = "full_page_writes is set to off for this server" + return result, nil + } + } + + return result, nil // Placeholder, replace with actual implementation +} + +func MaxWalSize(ctx context.Context, store *sql.DB) (*model.ConfigAuditResult, error) { + + result := &model.ConfigAuditResult{ + Name: "MaxWalSize", + Status: "Pass", + FailReason: "", // Placeholder, replace with actual implementation when implemented in CheckFsyncFlag function. + } + + // If max_wal_size is set to default - WARNING + query := `SELECT name, setting + FROM pg_settings + WHERE name = 'max_wal_size';` + + data, err := utils.GetJSON(store, query) + if err != nil { + result.Status = "Fail" + result.FailReason = err.Error() + return result, nil + } + + for _, obj := range data { + if obj["setting"] != nil && fmt.Sprint(obj["setting"]) == "1024" { + result.Status = "WARNING" + result.FailReason = "max_wal_size is set to default value i.e 1GB" + return result, nil + } + } + + return result, nil // Placeholder, replace with actual implementation +} + +func LogLinePrefix(ctx context.Context, store *sql.DB) (*model.ConfigAuditResult, error) { + + result := &model.ConfigAuditResult{ + Name: "LogLinePrefix", + Status: "Pass", + FailReason: "", // Placeholder, replace with actual implementation when implemented in CheckFsyncFlag function. + } + + // If log_line_prefix doesn't contains expected letters - CRITICAL + query := `SELECT name, setting + FROM pg_settings + WHERE name = 'log_line_prefix';` + + data, err := utils.GetJSON(store, query) + if err != nil { + result.Status = "Fail" + result.FailReason = err.Error() + return result, nil + } + + for _, obj := range data { + if obj["setting"] != nil { + logLinePrefix := fmt.Sprint(obj["setting"]) + missing := []string{} + + for _, sub := range LogLinePrefixSubstrings { + if !strings.Contains(logLinePrefix, sub) { + missing = append(missing, sub) + } + } + + if len(missing) > 0 { + result.Status = "Critical" + result.FailReason = fmt.Sprintf("log_line_prefix must contain %v, Missing values : %v", LogLinePrefixSubstrings, missing) + return result, nil + } + } + } + + return result, nil // Placeholder, replace with actual implementation +} + +func LogConnections(ctx context.Context, store *sql.DB) (*model.ConfigAuditResult, error) { + + result := &model.ConfigAuditResult{ + Name: "LogConnections", + Status: "Pass", + FailReason: "", // Placeholder, replace with actual implementation when implemented in CheckFsyncFlag function. + } + + // If log_connections or log_disconnections is set to off - CRITICAL + query := `SELECT name, setting + FROM pg_settings + WHERE name IN ('log_connections', 'log_disconnections');` + + data, err := utils.GetJSON(store, query) + if err != nil { + result.Status = "Fail" + result.FailReason = err.Error() + return result, nil + } + + for _, obj := range data { + if obj["setting"] != nil && fmt.Sprint(obj["setting"]) == "off" { + result.Status = "Critical" + result.FailReason = fmt.Sprintf("%s is set to off for this server", obj["name"]) + return result, nil + } + } + + return result, nil // Placeholder, replace with actual implementation +} + +func StatementTimeout(ctx context.Context, store *sql.DB) (*model.ConfigAuditResult, error) { + + result := &model.ConfigAuditResult{ + Name: "StatementTimeout", + Status: "Pass", + FailReason: "", // Placeholder, replace with actual implementation when implemented in CheckFsyncFlag function. + } + + // If statement_timeout is set to 0 (default) - CRITICAL + query := `SELECT name, setting + FROM pg_settings + WHERE name = 'statement_timeout';` + + data, err := utils.GetJSON(store, query) + if err != nil { + result.Status = "Fail" + result.FailReason = err.Error() + return result, nil + } + + for _, obj := range data { + if obj["setting"] != nil && fmt.Sprint(obj["setting"]) == "0" { + result.Status = "Critical" + result.FailReason = "statement_timeout is set to 0 (default) for this server" + return result, nil + } + } + + return result, nil // Placeholder, replace with actual implementation +} + +func IdleInTransationSessionTimeout(ctx context.Context, store *sql.DB) (*model.ConfigAuditResult, error) { + + result := &model.ConfigAuditResult{ + Name: "IdleInTransationSessionTimeout", + Status: "Pass", + FailReason: "", // Placeholder, replace with actual implementation when implemented in CheckFsyncFlag function. + } + + // If idle_in_transaction_session_timeout is set to 0 (default) - CRITICAL + query := `SELECT name, setting + FROM pg_settings + WHERE name = 'idle_in_transaction_session_timeout';` + + data, err := utils.GetJSON(store, query) + if err != nil { + result.Status = "Fail" + result.FailReason = err.Error() + return result, nil + } + + for _, obj := range data { + if obj["setting"] != nil && fmt.Sprint(obj["setting"]) == "0" { + result.Status = "Critical" + result.FailReason = "idle_in_transaction_session_timeout is set to 0 (default) for this server" + return result, nil + } + } + + return result, nil // Placeholder, replace with actual implementation +} diff --git a/postgresconfig/examplepostgres.conf b/postgresconfig/examplepostgres.conf index 1ec4ace..8d64852 100644 --- a/postgresconfig/examplepostgres.conf +++ b/postgresconfig/examplepostgres.conf @@ -1,10 +1,10 @@ -# Connectivity +# Connection Settings max_connections=%d superuser_reserved_connections=%s listen_addresses='%s' port=%s -# Memory Settings +# Memory Related Settings shared_buffers='%d MB' work_mem='%d kB' maintenance_work_mem='%d MB' @@ -13,36 +13,39 @@ effective_cache_size='%d GB' effective_io_concurrency=%d random_page_cost=%.2f -# Monitoring -shared_preload_libraries='pg_stat_statements' -track_io_timing=on -track_functions=pl +# Security Settings +log_connections=%s +log_disconnections=%s +log_statement=%s +ssl=%s +log_line_prefix='%s' +logging_collector=%s +log_destination=%s + +# Logging Settings +log_checkpoints=%s +log_lock_waits=%s +log_temp_files=%s +log_autovacuum_min_duration=%s +log_min_duration_statement=%s -# Replication +# Replication Settings wal_level='%s' max_wal_senders=%d synchronous_commit='%s' -# Checkpointing +# Checkpointing and WAL settings checkpoint_timeout='15 min' checkpoint_completion_target=0.9 max_wal_size='%.0f MB' min_wal_size='%.0f MB' -%s -# WAL writing wal_compression=%v wal_buffers=-1 %s -# Background writer -bgwriter_delay=200ms -bgwriter_lru_maxpages=100 -bgwriter_lru_multiplier=2.0 -bgwriter_flush_after=0 -# Parallel queries %s -# Advanced features +# Parallelism Settings %s # synchronous_standby_names @@ -67,14 +70,9 @@ autovacuum_max_workers=%s # autovacuum_vacuum_scale_factor # autovacuum_analyze_scale_factor -default_statistics_target=%d - # Special parameters +default_statistics_target=%d +shared_preload_libraries='pg_stat_statements' +track_io_timing=on statement_timeout=%s idle_in_transaction_session_timeout=%s - -# General notes -# Note that not all settings are automatically tuned. -# Consider contacting experts at -# KloudDB -# for more professional expertise. diff --git a/postgresconfig/generator.go b/postgresconfig/generator.go index cebc173..ed1ba0e 100644 --- a/postgresconfig/generator.go +++ b/postgresconfig/generator.go @@ -94,9 +94,9 @@ func calculateWorkerMem(totalRam, sharedBuffersValue, maxConnectionsValue int, d printTestLog("\tfor unknown setting workMemResult to %d", workMemResult) } - if workMemResult < 64 { - workMemResult = 64 - printTestLog("\tworkMemResult < 64 so setting to 64") + if workMemResult < 4096 { + workMemResult = 4096 + printTestLog("\tworkMemResult < 4096 so setting to 4096") } return workMemResult @@ -143,7 +143,7 @@ func ConfigGenerator(inputMap map[string]string) string { return "" } - databaseSize := parseSize(inputMap["databaseSize"]) + // databaseSize := parseSize(inputMap["databaseSize"]) dbType, err := getInt(inputMap["dbType"]) if err != nil { @@ -162,6 +162,10 @@ func ConfigGenerator(inputMap map[string]string) string { fmt.Println("Error parsing 'max_wal_size':", err) return "" } + if maxWalSize < 512 { + maxWalSize = 512 + printTestLog("Max Wal Size is less than 512MB so setting to 512MB") + } // Determine the number of connections based on the database type var connections int @@ -272,12 +276,6 @@ func ConfigGenerator(inputMap map[string]string) string { maxWalSenders := math.Max(10, float64(replicas+4)) printTestLog("Max Wal Senders: math.Max(10, replicas+4) = math.Max(10, %d+4) = %f", replicas, maxWalSenders) printTestLog("\n") - // if backup == 3 { - // walLevel = "logical" - // } else if backup == 1 && replicas == 0 { - // walLevel = "minimal" - // maxWalSenders = 0 - // } // SKIPPED // TODO @@ -299,7 +297,8 @@ func ConfigGenerator(inputMap map[string]string) string { walArchiving = "\n# WAL archiving\narchive_mode=on\narchive_command='/bin/true'\n" printTestLog("WAL Archiving: walLevel != 'minimal' so setting to on") } else { - printTestLog("WAL Level is minimal so not setting WAL Archiving") + printTestLog("WAL Level is minimal so not setting WAL Archiving and setting Max WAL Senders as 0") + maxWalSenders = 0 } printTestLog("\n") @@ -316,9 +315,9 @@ func ConfigGenerator(inputMap map[string]string) string { printTestLog("\n") parallel := "" - advanced := "" + // advanced := "" if version > 10 { - printTestLog("Version is greater than 10 so setting advanced and parallel settings") + printTestLog("Version is greater than 10 so setting parallel settings") maxParallelWorkersPerGather := cpu / 2 parallel = fmt.Sprintf( @@ -335,27 +334,27 @@ func ConfigGenerator(inputMap map[string]string) string { cpu, ) - advanced = "enable_partitionwise_join=on\nenable_partitionwise_aggregate=on" - printTestLog("\tAdvanced Settings: enable_partitionwise_join=on\n\t\tenable_partitionwise_aggregate=on") + // advanced = "enable_partitionwise_join=on\nenable_partitionwise_aggregate=on" + // printTestLog("\tAdvanced Settings: enable_partitionwise_join=on\n\t\tenable_partitionwise_aggregate=on") - if inputMap["jit"] == "on" { - advanced += "\njit=on" - printTestLog("\tAdvanced Settings: jit=on") - } + // if inputMap["jit"] == "on" { + // advanced += "\njit=on" + // printTestLog("\tAdvanced Settings: jit=on") + // } - if version >= 14 { - printTestLog("Version is greater than or equal to 14 so setting max slot wal keep size and track wal io timing") + // if version >= 14 { + // printTestLog("Version is greater than or equal to 14 so setting max slot wal keep size and track wal io timing") - maxSlotWalKeepSize := math.Max(float64(databaseSize)*0.1, 1000) - printTestLog("\tMax Slot Wal Keep Size: math.Max(databaseSize*0.1, 1000) = math.Max(%d*0.1, 1000) = %f", databaseSize, maxSlotWalKeepSize) + // maxSlotWalKeepSize := math.Max(float64(databaseSize)*0.1, 1000) + // printTestLog("\tMax Slot Wal Keep Size: math.Max(databaseSize*0.1, 1000) = math.Max(%d*0.1, 1000) = %f", databaseSize, maxSlotWalKeepSize) - advanced += fmt.Sprintf("\nmax_slot_wal_keep_size='%.0f MB'\ntrack_wal_io_timing=on", - maxSlotWalKeepSize, - ) - printTestLog("\tAdvanced Settings: max_slot_wal_keep_size='%.0f MB'\ntrack_wal_io_timing=on", - maxSlotWalKeepSize, - ) - } + // advanced += fmt.Sprintf("\nmax_slot_wal_keep_size='%.0f MB'\ntrack_wal_io_timing=on", + // maxSlotWalKeepSize, + // ) + // printTestLog("\tAdvanced Settings: max_slot_wal_keep_size='%.0f MB'\ntrack_wal_io_timing=on", + // maxSlotWalKeepSize, + // ) + // } } else { parallel = fmt.Sprintf("max_worker_processes=%d\nmax_parallel_workers_per_gather=%d\nmax_parallel_workers=%d", @@ -379,16 +378,27 @@ func ConfigGenerator(inputMap map[string]string) string { int(effectiveCacheSize), int(effectiveIOConcurrency), randomPageCost, + inputMap["log_connections"], + inputMap["log_disconnections"], + inputMap["log_statement"], + inputMap["ssl"], + inputMap["log_line_prefix"], + inputMap["logging_collector"], + inputMap["log_destination"], + inputMap["log_checkpoints"], + inputMap["log_lock_waits"], + inputMap["log_temp_files"], + inputMap["log_autovacuum_min_duration"], + inputMap["log_min_duration_statement"], walLevel, int(maxWalSenders), inputMap["synchronous_commit"], maxWalSize, minWalSize, - walArchiving, inputMap["wal_compression"], replication, + walArchiving, parallel, - advanced, inputMap["synchronous_standby_names"], inputMap["temp_file_limit"], inputMap["autovacuum_naptime"], @@ -414,7 +424,7 @@ func WriteToFile(configString, filepath string) error { return err } -var logOutput *os.File +// var logOutput *os.File func init() { // var err error @@ -425,5 +435,6 @@ func init() { } func printTestLog(str string, args ...interface{}) { - fmt.Fprintf(logOutput, " > "+str+"\n", args...) + + // fmt.Fprintf(logOutput, " > "+str+"\n", args...) } diff --git a/postgresconfig/helper_functions.go b/postgresconfig/helper_functions.go index fb0e28f..00cac98 100644 --- a/postgresconfig/helper_functions.go +++ b/postgresconfig/helper_functions.go @@ -2,7 +2,9 @@ package postgresconfig import ( _ "embed" + "errors" "fmt" + "net" "regexp" "strconv" "strings" @@ -29,9 +31,14 @@ var validateBool = func(_ map[string]string, val string) error { var validateIntRange = func(i, j int) func(m map[string]string, val string) error { return func(m map[string]string, val string) error { + val = strings.TrimSpace(val) + if val == "" { + return fmt.Errorf("The input value is empty. Please provide a valid integer") //lint:ignore ST1005 Error message formatting + } + numb, err := strconv.Atoi(val) if err != nil { - return err + return fmt.Errorf("'%v' is not a valid integer", val) } if i <= numb && numb <= j { @@ -54,21 +61,49 @@ var validateIntRangeAllowEmpty = func(i, j int) func(m map[string]string, val st } var validateInt = func(m map[string]string, val string) error { + val = strings.TrimSpace(val) + if val == "" { + return fmt.Errorf("input value is empty. please provide a valid integer") + } + _, err := strconv.Atoi(val) + if err != nil { + return fmt.Errorf("'%v' is not a valid integer", val) + } - return err + return nil } -var validateRegExp = func(r *regexp.Regexp) func(m map[string]string, val string) error { +var validateRegExp = func(r *regexp.Regexp, errMsg string) func(m map[string]string, val string) error { return func(m map[string]string, val string) error { if !r.MatchString(val) { - return fmt.Errorf("input data is not matching with pattern") + return fmt.Errorf("%s", errMsg) } return nil } } +var validateSize = func(m map[string]string, val string) error { + if !regexp.MustCompile(`^\s*[1-9]\d*\s*(?:MB|GB|TB|M|G|T|Mb|Gb|Tb|mB|gB|tB|mb|gb|tb)$`).MatchString(val) { + return fmt.Errorf("input data is not a valid size") + } + + return nil +} + +var validIP = func(m map[string]string, ip string) error { + if ip = strings.TrimSpace(ip); ip == "*" || ip == "" { + return nil + } + + if net.ParseIP(ip) == nil { + return fmt.Errorf("input value '%s' is not a valid IP", ip) + } + + return nil +} + func simplifyBoolVal(val string) string { val = strings.ToLower(val) for _, v := range []string{"y", "yes"} { @@ -92,6 +127,7 @@ var fieldSetFunction = func(fieldName string) func(m map[string]string, val stri return nil } } + var fieldSetFunctionDefault = func(fieldName, defaultVal string) func(m map[string]string, val string) error { return func(m map[string]string, val string) error { m[fieldName] = val @@ -102,6 +138,141 @@ var fieldSetFunctionDefault = func(fieldName, defaultVal string) func(m map[stri } } +var fieldSetSize = func(fieldName string) func(m map[string]string, val string) error { + return func(m map[string]string, val string) error { + normalized := regexp.MustCompile(`\s+`).ReplaceAllString(val, "") // Remove whitespaces + normalized = strings.ToUpper(normalized) + m[fieldName] = normalized + return nil + } +} + +var fieldSetSizeDefault = func(fieldName, defaultVal string) func(m map[string]string, val string) error { + return func(m map[string]string, val string) error { + if val == "" { + m[fieldName] = defaultVal + return nil + } + return fieldSetSize(fieldName)(m, val) + } +} + +var fieldSetSizeInMB = func(fieldName string, args ...string) func(m map[string]string, val string) error { + return func(m map[string]string, val string) error { + + // Check if a default value is provided + defaultVal := "" + if len(args) > 0 { + defaultVal = args[0] + } + + // Use default value if val is empty + if val == "" { + if defaultVal != "" { + m[fieldName] = defaultVal + return nil + } + return fmt.Errorf("value is empty and no default provided") + } + + normalized := regexp.MustCompile(`\s+`).ReplaceAllString(val, "") // Remove whitespaces + normalized = strings.ToUpper(normalized) + + // Regular expression to match value followed by units (MB, GB, TB, etc.) + re := regexp.MustCompile(`^(\d+(\.\d+)?)\s*(MB|GB|TB|M|G|T|Mb|Gb|Tb|mB|gB|tB|mb|gb|tb)?$`) + matches := re.FindStringSubmatch(normalized) + if len(matches) == 0 { + return errors.New("invalid size format") + } + + // Extract the numeric value and unit + valueStr := matches[1] + unit := matches[3] + + // Parse the numeric value + value, err := strconv.ParseFloat(valueStr, 64) + if err != nil { + return err + } + + // Convert based on the unit + switch unit { + case "T", "TB", "Tb", "tB", "tb": + value = value * 1024 * 1024 // TB to MB + case "G", "GB", "Gb", "gB", "gb": + value = value * 1024 // GB to MB + case "M", "MB", "Mb", "mB", "mb": + // No conversion needed if it's already in MB + default: + // Return error if unrecognized unit + return errors.New("unrecognized unit") + } + + // Store the converted value + m[fieldName] = strconv.Itoa(int(value)) + return nil + } +} + +var fieldSetSizeInKB = func(fieldName string, args ...string) func(m map[string]string, val string) error { + return func(m map[string]string, val string) error { + + // Check if a default value is provided + defaultVal := "" + if len(args) > 0 { + defaultVal = args[0] + } + + // Use default value if val is empty + if val == "" { + if defaultVal != "" { + m[fieldName] = defaultVal + return nil + } + return fmt.Errorf("value is empty and no default provided") + } + + normalized := regexp.MustCompile(`\s+`).ReplaceAllString(val, "") // Remove whitespaces + normalized = strings.ToUpper(normalized) + + // Regular expression to match value followed by units (MB, GB, TB, etc.) + re := regexp.MustCompile(`^(\d+(\.\d+)?)\s*(KB|MB|GB|TB|K|M|G|T|Kb|Mb|Gb|Tb|kB|mB|gB|tB|kb|mb|gb|tb)?$`) + matches := re.FindStringSubmatch(normalized) + if len(matches) == 0 { + return errors.New("invalid size format") + } + + // Extract the numeric value and unit + valueStr := matches[1] + unit := matches[3] + + // Parse the numeric value + value, err := strconv.ParseFloat(valueStr, 64) + if err != nil { + return err + } + + // Convert based on the unit + switch unit { + case "T", "TB", "Tb", "tB", "tb": + value = value * 1024 * 1024 * 1024 // TB to MB + case "G", "GB", "Gb", "gB", "gb": + value = value * 1024 * 1024 // GB to MB + case "M", "MB", "Mb", "mB", "mb": + value = value * 1024 + case "K", "KB", "Kb", "kB", "kb": + // No conversion needed if it's already in KB + default: + // Return error if unrecognized unit + return errors.New("unrecognized unit") + } + + // Store the converted value + m[fieldName] = strconv.Itoa(int(value)) + return nil + } +} + var skipIfSetFieldFunction = func(fieldName string) func(m map[string]string) (bool, error) { return func(m map[string]string) (bool, error) { _, ok := m[fieldName] diff --git a/postgresconfig/summary.txt b/postgresconfig/summary.txt index 02b23da..5277df0 100644 --- a/postgresconfig/summary.txt +++ b/postgresconfig/summary.txt @@ -7,7 +7,6 @@ CPU Cores: {{.cpu}} Disk Type: {{.diskType}} Host IP Address: {{.listen_addr}} Port: {{.port}} -Database Size: {{.databaseSize}} Database Type: {{.dbType}} Database Replicas: {{.replica}} @@ -17,6 +16,20 @@ Max WAL Size: {{.max_wal_size}} WAL Compression: {{.wal_compression}} WAL Level: {{.wal_level}} +Log Connections: {{.log_connections}} +Log Disconnections: {{.log_disconnections}} +Log Statement: {{.log_statement}} +SSL: {{.ssl}} +Log Line Prefix: {{.log_line_prefix}} +Log Collector: {{.logging_collector}} +Log Destination: {{.log_destination}} + +Log Checkpoints: {{.log_checkpoints}} +Log Lock Waits: {{.log_lock_waits}} +Log Temp Files: {{.log_temp_files}} +Log Autovaccum Min Duration: {{.log_autovacuum_min_duration}} +Log Min Duration Statement: {{.log_min_duration_statement}} + Synchronous Commit: {{.synchronous_commit}} Include Replicas: {{.synchronous_standby_names}} Temp File Limit: {{.temp_file_limit}} diff --git a/postgresconfig/user_input.go b/postgresconfig/user_input.go index 24deb8e..6eb61f8 100644 --- a/postgresconfig/user_input.go +++ b/postgresconfig/user_input.go @@ -1,22 +1,30 @@ package postgresconfig import ( + "bufio" "context" "fmt" "math" + "os" "path/filepath" "regexp" + "strconv" + "strings" "github.com/jedib0t/go-pretty/v6/text" + "github.com/muesli/termenv" + "golang.org/x/term" ) type node struct { - label string - helpMessage string - options []string - validation func(m map[string]string, val string) error - setFunc func(m map[string]string, val string) error - skipFunc func(m map[string]string) (bool, error) + heading string + label string + helpMessage string + extraMessage string + options []string + validation func(m map[string]string, val string) error + setFunc func(m map[string]string, val string) error + skipFunc func(m map[string]string) (bool, error) } type processor struct { @@ -30,24 +38,24 @@ func NewProcessor(configDir string) *processor { return &processor{ data: map[string]string{}, nodes: []*node{ - { - label: "Would you like to use Advanced mode? " + text.FgGreen.Sprint("recommended for production systems") + " may take 5-10 minutes", - helpMessage: "Y/n", - validation: validateBool, - setFunc: func(m map[string]string, val string) error { - if simplifyBoolVal(val) == "no" { - return nil - } + // { + // label: "Would you like to use Advanced mode? " + text.FgGreen.Sprint("recommended for production systems") + " may take 5-10 minutes", + // helpMessage: "Y/n", + // validation: validateBool, + // setFunc: func(m map[string]string, val string) error { + // if simplifyBoolVal(val) == "no" { + // return nil + // } - m["listen_addr"] = "*" // Set defaults for quick mode - m["port"] = "5432" - m["superuser_reserved_connections"] = "3" - m["wal_level"] = "replica" - m["synchronous_commit"] = "ON" - return nil - }, - skipFunc: EmptySkipFunc, - }, + // m["listen_addr"] = "*" // Set defaults for quick mode + // m["port"] = "5432" + // m["superuser_reserved_connections"] = "3" + // m["wal_level"] = "replica" + // m["synchronous_commit"] = "ON" + // return nil + // }, + // skipFunc: EmptySkipFunc, + // }, { label: "Which PostgreSQL version are you using?", helpMessage: "Enter a number (e.g., 13, 14, 15, or 16)", @@ -57,9 +65,9 @@ func NewProcessor(configDir string) *processor { }, { label: "How much RAM does your system have?", - helpMessage: "Enter a value followed by GB or TB (e.g., 4GB, 8GB, 1TB)", - validation: validateRegExp(regexp.MustCompile(`^[1-9]\d*(?:GB|TB)$`)), - setFunc: fieldSetFunction("ram"), + helpMessage: "Enter a value followed by MB, GB or TB (e.g., 256MB, 8GB, 1TB)", + validation: validateSize, + setFunc: fieldSetSize("ram"), skipFunc: EmptySkipFunc, }, { @@ -70,9 +78,10 @@ func NewProcessor(configDir string) *processor { skipFunc: EmptySkipFunc, }, { - label: "What type of storage are you using?", - options: []string{"SSD", "HDD", "Network (SAN)"}, - validation: EmptyValidationfunc, + label: "What type of storage are you using?", + options: []string{"SSD", "HDD", "Network (SAN)"}, + helpMessage: "Enter a number between 1 and 3", + validation: EmptyValidationfunc, setFunc: func(m map[string]string, val string) error { switch val { case "SSD": @@ -89,17 +98,18 @@ func NewProcessor(configDir string) *processor { }, skipFunc: EmptySkipFunc, }, + // { + // label: "What is the total size of your database?", + // helpMessage: "Enter a value followed by GB or TB (e.g., 100GB, 1TB, 10TB)", + // validation: validateSize, + // setFunc: fieldSetSize("databaseSize"), + // skipFunc: EmptySkipFunc, + // }, { - label: "What is the total size of your database?", - helpMessage: "Enter a value followed by GB or TB (e.g., 100GB, 1TB, 10TB)", - validation: validateRegExp(regexp.MustCompile(`^[1-9]\d*(?:GB|TB)$`)), - setFunc: fieldSetFunction("databaseSize"), - skipFunc: EmptySkipFunc, - }, - { - label: "What is the primary use case for your database?", - options: []string{"Web application", "OLTP", "Data warehouse", "Desktop application", "Mixed workload"}, - validation: EmptyValidationfunc, + label: "What is the primary use case for your database?", + options: []string{"Web application", "OLTP", "Data warehouse", "Desktop application", "Mixed workload"}, + helpMessage: "Enter a number between 1 and 5", + validation: EmptyValidationfunc, setFunc: func(m map[string]string, val string) error { switch val { case "Web application": @@ -125,32 +135,45 @@ func NewProcessor(configDir string) *processor { skipFunc: EmptySkipFunc, }, { - label: "Enter a valid IP address or * for all", - helpMessage: "e.g 43.23.54.10 or *", - validation: validateRegExp(regexp.MustCompile(`^(?:\*|(?:[0-9]{1,3}\.){3}[0-9]{1,3})?$`)), - setFunc: fieldSetFunctionDefault("listen_addr", "*"), - skipFunc: skipIfSetFieldFunction("listen_addr"), + label: "Listen_addresses parameter : Enter valid IP address or * for all", + helpMessage: "e.g 43.23.54.10 , Default: *", + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - By default, we use * for the listen_addr parameter, but we recommend specifying an IP address range or particular IP addresses for better security. Allowing * opens access to all IPs, which is not a secure practice"), + ), + validation: validIP, + setFunc: fieldSetFunctionDefault("listen_addr", "*"), + skipFunc: skipIfSetFieldFunction("listen_addr"), }, { - label: "Enter a valid port. Avoid using 5432 for better security.", - helpMessage: "e.g 1234 (1-65535)", - validation: validateIntRange(1, 65535), - setFunc: fieldSetFunction("port"), - skipFunc: skipIfSetFieldFunction("port"), + label: "Enter a valid port. Avoid using 5432 for better security.", + helpMessage: "e.g 1234 (1-65535). Default: 5432", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateIntRangeAllowEmpty(1, 65535), + setFunc: fieldSetFunctionDefault("port", "5432"), + skipFunc: skipIfSetFieldFunction("port"), }, { - label: "superuser reserved connections", - helpMessage: "greater than 3.", - validation: validateIntRange(3, math.MaxInt), - setFunc: fieldSetFunctionDefault("superuser_reserved_connections", "3"), - skipFunc: skipIfSetFieldFunction("superuser_reserved_connections"), + label: "superuser reserved connections-Sets the number of connection slots reserved for superusers", + helpMessage: "greater than 3. Default: 3", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateIntRangeAllowEmpty(3, math.MaxInt), + setFunc: fieldSetFunctionDefault("superuser_reserved_connections", "3"), + skipFunc: skipIfSetFieldFunction("superuser_reserved_connections"), }, { label: "Set the maximum WAL (Write-Ahead Log) size", - helpMessage: "Enter a value in MB. Recommended: at least 3x the WAL generated during a 15-minute peak period", - validation: validateIntRange(2, 2147483647), - setFunc: fieldSetFunction("max_wal_size"), - skipFunc: EmptySkipFunc, + helpMessage: "Enter a value in MB or GB, Default: 1GB", + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - The max_wal_size parameter is critical for performance. It should be set to at least three times the amount of WALs generated during a 15-minute peak traffic period. Note: This assumes the checkpoint_timeout is set to 15 minutes"), + ), + validation: validateRegExp( + regexp.MustCompile(`^(?:\s*[1-9]\d*\s*(?:MB|GB|TB|M|G|T|Mb|Gb|Tb|mB|gB|tB|mb|gb|tb))?$`), + "input data is not a valid file size", + ), + setFunc: fieldSetSizeInMB("max_wal_size", "1024"), + skipFunc: EmptySkipFunc, }, { label: "Enable WAL compression?", @@ -223,6 +246,7 @@ func NewProcessor(configDir string) *processor { helpMessage: "e.g ANY(s1, s2, s3) or FIRST(s1, s2) or '*' to include all standby servers", validation: validateRegExp( regexp.MustCompile(`(?i)(?:(?:^(?:first|any)\(.+\)$)|(?:^(?:\*)?$))`), + "input data is not a valid replica/standbys", ), setFunc: fieldSetFunctionDefault("synchronous_standby_names", "*"), skipFunc: EmptySkipFunc, @@ -230,44 +254,88 @@ func NewProcessor(configDir string) *processor { { label: "Set temp_file_limit to prevent large temporary files (recommended: 1GB-2GB). This prevents disk full errors and other issues. Default: 1GB. For more information, visit: https://klouddb.io/temporary-files-in-postgresql-steps-to-identify-and-fix-temp-file-issues/", helpMessage: "Specify a value (e.g., 1GB or 2GB) to limit temporary file size.", - validation: validateRegExp(regexp.MustCompile(`^(?:[1-9]\d*(?:GB|TB))?$`)), - setFunc: fieldSetFunctionDefault("temp_file_limit", "1GB"), - skipFunc: EmptySkipFunc, + validation: validateRegExp( + regexp.MustCompile(`^(?:\s*[1-9]\d*\s*(?:GB|TB|Gb|Tb|gb|tb))?$`), + "input data is not a valid file size", + ), + setFunc: fieldSetSizeDefault("temp_file_limit", "1GB"), + skipFunc: EmptySkipFunc, }, { - label: "Specify the minimum delay between autovacuum runs (default: 1 minute). Decrease to 30s or 15s for large number of tables.", - helpMessage: "Value in milliseconds (ms). Default: 60", - validation: validateIntRangeAllowEmpty(1, 2147483), - setFunc: fieldSetFunctionDefault("autovacuum_naptime", "60"), - skipFunc: EmptySkipFunc, + label: "Would you like to fine-tune your autovacuum settings?", + helpMessage: "y/N", + validation: validateBool, + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - For critical environments, especially if you’re experiencing autovacuum issues or anticipate them, it’s recommended to fine-tune the settings"), + ), + setFunc: func(m map[string]string, val string) error { + if simplifyBoolVal(val) == "yes" { + return nil + } + + // Set defaults + m["autovacuum_naptime"] = "60" + m["autovacuum_vacuum_cost_limit"] = "-1" + m["autovacuum_vacuum_cost_delay"] = "2" + m["autovacuum_max_workers"] = "3" + return nil + }, + skipFunc: EmptySkipFunc, }, { - label: "Set autovacuum_vacuum_cost_limit to control vacuum resource usage. Too high may slow queries, too low may not reclaim space.", - helpMessage: "Accumulated cost causing vacuum to sleep for cost_delay time. Default: -1 (ms)", - validation: validateIntRangeAllowEmpty(-1, 10000), - setFunc: fieldSetFunctionDefault("autovacuum_vacuum_cost_limit", "-1"), - skipFunc: EmptySkipFunc, + label: "Specify the minimum delay between autovacuum runs", + helpMessage: "Value in milliseconds (ms). Default: 1 minute", + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - This parameter specifies the minimum delay between autovacuum runs on any given database. Default is 1 minute , decrease this to 30s or 15s if you have a large number (100's) of tables.. If you want to override the default please specify a value or else hit enter"), + ), + validation: validateIntRangeAllowEmpty(1, 2147483), + setFunc: fieldSetFunctionDefault("autovacuum_naptime", "60"), + skipFunc: skipIfSetFieldFunction("autovacuum_naptime"), }, { - label: "Specify autovacuum_vacuum_cost_delay (in ms) to pause autovacuum job when autovacuum_vacuum_cost_limit is reached. Default is 2 ms. Override only if you have vacuuming issues.", - helpMessage: "Enter a value in milliseconds (ms). Default: 2 ms", - validation: validateIntRangeAllowEmpty(-1, 100), - setFunc: fieldSetFunctionDefault("autovacuum_vacuum_cost_delay", "2"), - skipFunc: EmptySkipFunc, + label: "Set autovacuum_vacuum_cost_limit to control vacuum resource usage", + helpMessage: "Value in milliseconds (ms). Default: -1 (ms)", + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - This is the accumulated cost that will cause the vacuuming process to sleep for cost_delay time .If you set the value of autovacuum_vacuum_cost_limit too high, the autovacuum process might consume too many resources and slow down other queries. If you set it too low, the autovacuum process might not reclaim enough space, which causes the table to become larger over time. If you want to override the default please specify a value or else hit enter"), + ), + validation: validateIntRangeAllowEmpty(-1, 10000), + setFunc: fieldSetFunctionDefault("autovacuum_vacuum_cost_limit", "-1"), + skipFunc: skipIfSetFieldFunction("autovacuum_vacuum_cost_limit"), }, { - label: "Increase this value if you have a large number of tables and sufficient CPU cores", - helpMessage: "Note: Stick with the default unless you have vacuuming issues. Value in milliseconds (ms). Default: 3", - validation: validateIntRangeAllowEmpty(1, 262143), - setFunc: fieldSetFunctionDefault("autovacuum_max_workers", "3"), - skipFunc: EmptySkipFunc, + label: "Specify autovacuum_vacuum_cost_delay", + helpMessage: "Value in milliseconds (ms) Default: 2 ms", + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - As soon as autovacuum_vacuum_cost_limit is hit autovacuum job is paused for autovacuum_vacuum_cost_delay. If you want to override the default please specify a value or else hit enter. NOTE - It’s best to stick with the default value unless you have some vacuuming issues"), + ), + validation: validateIntRangeAllowEmpty(-1, 100), + setFunc: fieldSetFunctionDefault("autovacuum_vacuum_cost_delay", "2"), + skipFunc: skipIfSetFieldFunction("autovacuum_vacuum_cost_delay"), + }, + { + label: "Specify autovacuum_max_workers", + helpMessage: "Default: 3", + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - If you have hundreds and thousands of tables it is better to increase this to a bigger number .. But make sure you have lot of cores to support the increase .If you want to override the default please specify a value or else hit enter. NOTE - It’s best to stick with the default value unless you have some vacuuming issues"), + ), + validation: validateIntRangeAllowEmpty(1, 262143), + setFunc: fieldSetFunctionDefault("autovacuum_max_workers", "3"), + skipFunc: skipIfSetFieldFunction("autovacuum_max_workers"), }, { label: "Recommended JIT setting: 'off' for OLTP/web apps, 'on' for data warehouses. Benchmark critical apps to find the best setting.", helpMessage: "Default: 'off'", - validation: validateRegExp(regexp.MustCompile(`(?i)^(?:on|off)?$`)), - setFunc: fieldSetFunctionDefault("jit", "off"), - skipFunc: EmptySkipFunc, + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:on|off)?$`), + "input data is not valid. Enter on/off", + ), + setFunc: fieldSetFunctionDefault("jit", "off"), + skipFunc: EmptySkipFunc, }, { label: "Set statement_timeout (in milliseconds) to abort long-running queries. For OLTP apps, 60000 ms (60s) is recommended", @@ -283,6 +351,206 @@ func NewProcessor(configDir string) *processor { setFunc: fieldSetFunctionDefault("idle_in_transaction_session_timeout", "1800000"), skipFunc: EmptySkipFunc, }, + { + heading: "SECURITY SETTINGS", + label: "For enhanced security, we recommend the following settings.\n\t• To proceed with these settings, press Y.\n\t• To customize them, press N, and we'll guide you through the process step by step.", + helpMessage: "y/N", + validation: validateBool, + extraMessage: fmt.Sprintf( + "\n\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n\n%s", + fmt.Sprintf("%-30s = %s", text.FgHiCyan.Sprint("log_connections"), text.FgHiGreen.Sprint("'on'")), + fmt.Sprintf("%-30s = %s", text.FgHiCyan.Sprint("log_disconnections"), text.FgHiGreen.Sprint("'on'")), + fmt.Sprintf("%-30s = %s", text.FgHiCyan.Sprint("log_statement"), text.FgHiGreen.Sprint("'all'")), + fmt.Sprintf("%-30s = %s", text.FgHiCyan.Sprint("ssl"), text.FgHiGreen.Sprint("'on'")), + fmt.Sprintf("%-30s = %s", text.FgHiCyan.Sprint("log_line_prefix"), text.FgHiGreen.Sprint("'%m [%p]: [%l-1] db=%d,user=%u,app=%a,client=%h'")), //nolint:govet + fmt.Sprintf("%-30s = %s", text.FgHiCyan.Sprint("logging_collector"), text.FgHiGreen.Sprint("'on'")), + fmt.Sprintf("%-30s = %s", text.FgHiCyan.Sprint("log_destination"), text.FgHiGreen.Sprint("'stderr'")), + text.FgHiGreen.Sprint(""), + ), + setFunc: func(m map[string]string, val string) error { + if simplifyBoolVal(val) == "no" { + return nil + } + + // Set defaults + m["log_connections"] = "on" + m["log_disconnections"] = "on" + m["log_statement"] = "all" + m["ssl"] = "on" + m["log_line_prefix"] = "%m [%p]: [%l-1] db=%d,user=%u,app=%a,client=%h" + m["logging_collector"] = "on" + m["log_destination"] = "stderr" + return nil + }, + skipFunc: EmptySkipFunc, + }, + { + heading: "SECURITY SETTINGS", + label: "Specify log_connections", + helpMessage: "on|off. Default: on", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:on|off)?$`), + "input data is not valid. Enter on/off", + ), + setFunc: fieldSetFunctionDefault("log_connections", "on"), + skipFunc: skipIfSetFieldFunction("log_connections"), + }, + { + heading: "SECURITY SETTINGS", + label: "Specify log_disconnections", + helpMessage: "on|off. Default: on", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:on|off)?$`), + "input data is not valid. Enter on/off", + ), + setFunc: fieldSetFunctionDefault("log_disconnections", "on"), + skipFunc: skipIfSetFieldFunction("log_disconnections"), + }, + { + heading: "SECURITY SETTINGS", + label: "Specify log_statement", + helpMessage: "none|ddl|mod|all. Default: all", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:none|ddl|mod|all)?$`), + "input data is not valid. Enter one of the [none, ddl, mod, all]", + ), + setFunc: fieldSetFunctionDefault("log_statement", "all"), + skipFunc: skipIfSetFieldFunction("log_statement"), + }, + { + heading: "SECURITY SETTINGS", + label: "Specify ssl", + helpMessage: "on|off. Default: on", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:on|off)?$`), + "input data is not valid. Enter on/off", + ), + setFunc: fieldSetFunctionDefault("ssl", "on"), + skipFunc: skipIfSetFieldFunction("ssl"), + }, + { + heading: "SECURITY SETTINGS", + label: "Specify log_line_prefix", + helpMessage: "e.g : %m [%p], Default: '%m [%p]: [%l-1] db=%d,user=%u,app=%a,client=%h'", + extraMessage: text.FgHiGreen.Sprint(""), + validation: EmptyValidationfunc, + setFunc: fieldSetFunctionDefault("log_line_prefix", "%m [%p]: [%l-1] db=%d,user=%u,app=%a,client=%h"), + skipFunc: skipIfSetFieldFunction("log_line_prefix"), + }, + { + heading: "SECURITY SETTINGS", + label: "Specify logging_collector", + helpMessage: "on|off. Default: on", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:on|off)?$`), + "input data is not valid. Enter on/off", + ), + setFunc: fieldSetFunctionDefault("logging_collector", "on"), + skipFunc: skipIfSetFieldFunction("logging_collector"), + }, + { + heading: "SECURITY SETTINGS", + label: "Specify log_destination", + helpMessage: "stderr|csv|syslog , Default: stderr", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:stderr|csv|syslog)?$`), + "input data is not valid. Enter one of the [stderr, csv, syslog]", + ), + setFunc: fieldSetFunctionDefault("log_destination", "stderr"), + skipFunc: skipIfSetFieldFunction("log_destination"), + }, + { + heading: "LOGGING SETTINGS", + label: "Logging settings are crucial for monitoring and troubleshooting.\n\t• To proceed with the recommended settings, press Y.\n\t• To customize them, press N, and we'll guide you through the process step by step.", + helpMessage: "y/N", + validation: validateBool, + extraMessage: fmt.Sprintf( + "\n\n%s\n%s\n%s\n%s\n%s\n\n%s", + fmt.Sprintf("%-38s = %s", text.FgHiCyan.Sprint("log_checkpoints"), text.FgHiGreen.Sprint("'on'")), + fmt.Sprintf("%-38s = %s", text.FgHiCyan.Sprint("log_lock_waits"), text.FgHiGreen.Sprint("'on'")), + fmt.Sprintf("%-38s = %s", text.FgHiCyan.Sprint("log_temp_files"), text.FgHiGreen.Sprint("'1KB'")), + fmt.Sprintf("%-38s = %s", text.FgHiCyan.Sprint("log_autovacuum_min_duration"), text.FgHiGreen.Sprint("'600000(ms)'")), + fmt.Sprintf("%-38s = %s", text.FgHiCyan.Sprint("log_min_duration_statement"), text.FgHiGreen.Sprint("'1s'")), + text.FgHiGreen.Sprint(""), + ), + setFunc: func(m map[string]string, val string) error { + if simplifyBoolVal(val) == "no" { + return nil + } + + // Set defaults + m["log_checkpoints"] = "on" + m["log_lock_waits"] = "on" + m["log_temp_files"] = "1" + m["log_autovacuum_min_duration"] = "600000" + m["log_min_duration_statement"] = "1000" + return nil + }, + skipFunc: EmptySkipFunc, + }, + { + heading: "LOGGING SETTINGS", + label: "Specify log_checkpoints", + helpMessage: "on|off. Default: on", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:on|off)?$`), + "input data is not valid. Enter on/off", + ), + setFunc: fieldSetFunctionDefault("log_checkpoints", "on"), + skipFunc: skipIfSetFieldFunction("log_checkpoints"), + }, + { + heading: "LOGGING SETTINGS", + label: "Specify log_lock_waits", + helpMessage: "on|off. Default: on", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`(?i)^(?:on|off)?$`), + "input data is not valid. Enter on/off", + ), + setFunc: fieldSetFunctionDefault("log_lock_waits", "on"), + skipFunc: skipIfSetFieldFunction("log_lock_waits"), + }, + { + heading: "LOGGING SETTINGS", + label: "Specify log_temp_files", + helpMessage: "Enter a value in KB, MB or GB, Default: 1KB", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateRegExp( + regexp.MustCompile(`^(?:\s*[1-9]\d*\s*(?:KB|MB|GB|K|M|G|Kb|Mb|Gb|kB|mB|gB|lb|mb|gb))?$`), + "input data is not a valid file size", + ), + setFunc: fieldSetSizeInKB("log_temp_files", "1"), + skipFunc: skipIfSetFieldFunction("log_temp_files"), + }, + { + heading: "LOGGING SETTINGS", + label: "Specify log_autovacuum_min_duration", + helpMessage: "Value in milliseconds (ms). Default: 600000(ms)", + extraMessage: text.FgHiGreen.Sprint(""), + validation: validateIntRangeAllowEmpty(1, 2147483647), + setFunc: fieldSetFunctionDefault("log_autovacuum_min_duration", "600000"), + skipFunc: skipIfSetFieldFunction("log_autovacuum_min_duration"), + }, + { + heading: "LOGGING SETTINGS", + label: "Specify log_min_duration_statement", + helpMessage: "Value in milliseconds (ms). Default: 1s", + extraMessage: fmt.Sprintf("%s\n\n%s", + text.FgHiGreen.Sprint(""), + text.FgHiYellow.Sprint("NOTE - log_min_duration_statement - Logs the duration of any completed statement that takes at least the specified amount of time to execute. For example, if set to 250ms, all SQL statements running for 250ms or longer will be logged. The recommended value is typically 1s or 2s, depending on your specific use case"), + ), + validation: validateIntRangeAllowEmpty(1, 2147483647), + setFunc: fieldSetFunctionDefault("log_min_duration_statement", "1000"), + skipFunc: skipIfSetFieldFunction("log_min_duration_statement"), + }, }, configDir: configDir, } @@ -329,6 +597,7 @@ func (p *processor) Run(ctx context.Context) error { } func (p *processor) GetUserInput(ctx context.Context) error { + reader := bufio.NewReader(os.Stdin) for _, n := range p.nodes { skip, err := n.skipFunc(p.data) if err != nil { @@ -340,16 +609,42 @@ func (p *processor) GetUserInput(ctx context.Context) error { } fmt.Print("\033[H\033[2J") // Clear terminal + + // Get the terminal width dynamically + width, _, err := term.GetSize(int(os.Stdout.Fd())) + if err != nil { + width = 80 // Fallback to 80 if unable to get terminal width + } + + termProfile := termenv.ColorProfile() + + // Print center-aligned heading if its length > 0 + if len(n.heading) > 0 { + heading := termenv.String(n.heading).Bold().Foreground(termProfile.Color("#FFA500")).String() // Orange and bold + padding := (width - len(stripAnsiCodes(n.heading))) / 2 + if padding < 0 { + padding = 0 // Prevent negative padding + } + fmt.Printf("%s%s\n\n", strings.Repeat(" ", padding), heading) + } fmt.Printf("> %s: ", n.label) - if len(n.helpMessage) != 0 { - fmt.Printf("[%s] : ", text.FgHiCyan.Sprint(n.helpMessage)) + if len(n.helpMessage) > 0 { + fmt.Printf("[%s] ", text.FgHiCyan.Sprint(n.helpMessage)) } + if len(n.extraMessage) > 0 { + fmt.Printf("%s ", n.extraMessage) + } + if len(n.helpMessage) > 0 || len(n.extraMessage) > 0 { + fmt.Print(": ") + } + var userInput string i := 0 for ; i < 3; i++ { userInput = "" if len(n.options) == 0 { - fmt.Scanln(&userInput) //nolint:errcheck + userInput, _ = reader.ReadString('\n') + userInput = strings.TrimSpace(userInput) } else { fmt.Println() for i, v := range n.options { @@ -358,12 +653,24 @@ func (p *processor) GetUserInput(ctx context.Context) error { // wait for user input fmt.Print("Select one from the list :") - var val int - fmt.Scanf("%d", &val) //nolint:errcheck + + userInput, _ = reader.ReadString('\n') + userInput = strings.TrimSpace(userInput) + + if userInput == "" { + fmt.Print(text.FgRed.Sprintf("You have not selected any option. Please enter an integer to select.")) + continue + } + + val, err := strconv.Atoi(userInput) + if err != nil { + fmt.Print(text.FgRed.Sprintf("Error: '%s' is not a valid integer.", userInput) + " Please retry.\n") + continue + } if val > len(n.options) || val <= 0 { // invalid value - fmt.Println("invalid option selected. please retry") + fmt.Println(text.FgRed.Sprint("invalid option selected. please retry")) continue } userInput = n.options[val-1] @@ -374,13 +681,11 @@ func (p *processor) GetUserInput(ctx context.Context) error { break } - if i < 2 { - fmt.Printf( - "You have added invalid input. Error: %v\nTry again (available retries: %d) :", - text.FgRed.Sprint(err), - 2-i, - ) - } + fmt.Printf( + "Invalid input detected. Error: %v\nTry again (available retries: %d) :", + text.FgRed.Sprint(err), + 2-i, + ) } if i == 3 { return fmt.Errorf("reached maximum retry limit") @@ -397,8 +702,21 @@ func (p *processor) GetUserInput(ctx context.Context) error { return nil } -func (p *processor) GenerateConfigFile(filepath string) error { +func (p *processor) GenerateConfigFile(configPath string) error { configString := ConfigGenerator(p.GetData()) + err := WriteToFile(configString, configPath) + if err != nil { + fmt.Printf("\n%s Error: %v ❌\n", text.FgRed.Sprintf("Error:"), err) + return err + } + absolutePath, _ := filepath.Abs(configPath) + fmt.Printf("\n%s Config file has been generated at %s ✅\n", text.FgGreen.Sprintf("Success:"), absolutePath) + + return nil +} - return WriteToFile(configString, filepath) +// stripAnsiCodes removes ANSI escape codes to correctly calculate string length +func stripAnsiCodes(input string) string { + ansiEscape := "\033\\[[0-9;]*m" + return strings.ReplaceAll(strings.ReplaceAll(input, ansiEscape, ""), "\033[0m", "") }