From 16dd084a3004680db8bcaf2d64f349ce9ca88fb5 Mon Sep 17 00:00:00 2001 From: Nestor Date: Fri, 12 Apr 2024 18:34:29 +0200 Subject: [PATCH 01/11] feat: add basic support for mysql --- Dockerfile | 2 +- go.mod | 2 +- go.sum | 41 --------------------------------------- main.go | 9 ++++++--- pgdump.go | 56 ++++++++++++++++++++++++++++++++++++++---------------- 5 files changed, 48 insertions(+), 62 deletions(-) diff --git a/Dockerfile b/Dockerfile index 51e4273..6fa743e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o main . FROM alpine:latest WORKDIR /root/ -RUN apk --no-cache add ca-certificates postgresql-client +RUN apk --no-cache add ca-certificates postgresql-client mysql-client COPY --from=builder /app/main /bin/postgres-s3-backup diff --git a/go.mod b/go.mod index 48c7f60..6ae3af6 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/rs/xid v1.5.0 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/stretchr/testify v1.7.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.17.0 // indirect diff --git a/go.sum b/go.sum index f33e5b2..949ccd0 100644 --- a/go.sum +++ b/go.sum @@ -4,41 +4,19 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= -github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= -github.com/minio/minio-go/v7 v7.0.64 h1:Zdza8HwOzkld0ZG/og50w56fKi6AAyfqfifmasD9n2Q= -github.com/minio/minio-go/v7 v7.0.64/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc= -github.com/minio/minio-go/v7 v7.0.65 h1:sOlB8T3nQK+TApTpuN3k4WD5KasvZIE3vVFzyyCa0go= -github.com/minio/minio-go/v7 v7.0.65/go.mod h1:R4WVUR6ZTedlCcGwZRauLMIKjgyaWxhs4Mqi/OMPmEc= -github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= -github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= -github.com/minio/minio-go/v7 v7.0.67 h1:BeBvZWAS+kRJm1vGTMJYVjKUNoo0FoEt/wUWdUtfmh8= -github.com/minio/minio-go/v7 v7.0.67/go.mod h1:+UXocnUeZ3wHvVh5s95gcrA4YjMIbccT6ubB+1m054A= -github.com/minio/minio-go/v7 v7.0.68 h1:hTqSIfLlpXaKuNy4baAp4Jjy2sqZEN9hRxD0M4aOfrQ= -github.com/minio/minio-go/v7 v7.0.68/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ= github.com/minio/minio-go/v7 v7.0.69 h1:l8AnsQFyY1xiwa/DaQskY4NXSLA2yrGsW5iD9nRPVS0= github.com/minio/minio-go/v7 v7.0.69/go.mod h1:XAvOPJQ5Xlzk5o3o/ArO2NMbhSGkimC+bpW/ngRKDmQ= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= @@ -52,36 +30,17 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go index 7a315b3..851d00c 100644 --- a/main.go +++ b/main.go @@ -3,15 +3,16 @@ package main import ( "context" "flag" - "github.com/joho/godotenv" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "log" "net/url" "os" "strconv" "strings" "time" + + "github.com/joho/godotenv" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" ) func main() { @@ -25,6 +26,7 @@ func main() { s3Bucket := flag.String("s3-bucket", lookupEnvOrString("S3_BUCKET", "postgres-backups"), "S3 bucket") s3AccessKey := flag.String("s3-access-key", lookupEnvOrString("S3_ACCESS_KEY", "minio"), "S3 access key") s3SecretKey := flag.String("s3-secret-key", lookupEnvOrString("S3_SECRET_KEY", "minioadmin"), "S3 secret key") + dbType := flag.String("db-type", lookupEnvOrString("DB_TYPE", "postgres"), "Type of database: postgres or mariadb") interval := flag.Duration("interval", lookupEnvOrDuration("INTERVAL", 24*time.Hour), "How often to run the backup") @@ -62,6 +64,7 @@ func main() { Database: strings.TrimPrefix(parsedUrl.Path, "/"), Username: parsedUrl.User.Username(), Password: password, + DbType: *dbType, } } if len(urls) == 0 { diff --git a/pgdump.go b/pgdump.go index 3dc70ce..be743f3 100644 --- a/pgdump.go +++ b/pgdump.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "strconv" "time" ) @@ -15,6 +16,7 @@ type connectionOptions struct { Database string Username string Password string + DbType string } var ( @@ -22,26 +24,48 @@ var ( pgDumpStdOpts = []string{"--no-owner", "--no-acl", "--clean", "--blobs", "-v"} pgDumpDefaultFormat = "c" - ErrPgDumpNotFound = errors.New("pg_dump not found") + ErrPgDumpNotFound = errors.New("pg_dump not found") + ErrMySqlDumpNotFound = errors.New("mysqldump not found") ) -func RunDump(pg *connectionOptions, outFile string) error { - if !commandExist(PGDumpCmd) { - return ErrPgDumpNotFound +func RunDump(connectionOpts *connectionOptions, outFile string) error { + var cmd *exec.Cmd + + if connectionOpts.DbType == "postgres" { + if !commandExist(PGDumpCmd) { + return ErrPgDumpNotFound + } + + options := append( + pgDumpStdOpts, + fmt.Sprintf(`-f%s`, outFile), + fmt.Sprintf(`--dbname=%v`, connectionOpts.Database), + fmt.Sprintf(`--host=%v`, connectionOpts.Host), + fmt.Sprintf(`--port=%v`, connectionOpts.Port), + fmt.Sprintf(`--username=%v`, connectionOpts.Username), + fmt.Sprintf(`--format=%v`, pgDumpDefaultFormat), + ) + cmd = exec.Command(PGDumpCmd, options...) + } else if connectionOpts.DbType == "mariadb" { + mysqldumpCmd := "mysqldump" + if !commandExist(mysqldumpCmd) { + return ErrMySqlDumpNotFound + } + + options := []string{ + "-h", connectionOpts.Host, + "-P", strconv.Itoa(connectionOpts.Port), + "-u", connectionOpts.Username, + fmt.Sprintf(`--password=%s`, connectionOpts.Password), + "--databases", connectionOpts.Database, + "-r", outFile, + } + cmd = exec.Command(mysqldumpCmd, options...) + } else { + return errors.New("unsupported database type") } - options := append( - pgDumpStdOpts, - fmt.Sprintf(`-f%s`, outFile), - fmt.Sprintf(`--dbname=%v`, pg.Database), - fmt.Sprintf(`--host=%v`, pg.Host), - fmt.Sprintf(`--port=%v`, pg.Port), - fmt.Sprintf(`--username=%v`, pg.Username), - fmt.Sprintf(`--format=%v`, pgDumpDefaultFormat), - ) - - cmd := exec.Command(PGDumpCmd, options...) - cmd.Env = append(os.Environ(), fmt.Sprintf(`PGPASSWORD=%v`, pg.Password)) + cmd.Env = append(os.Environ(), fmt.Sprintf(`PGPASSWORD=%v`, connectionOpts.Password)) stderr, err := cmd.StderrPipe() if err != nil { From 4e6eeabdc04ea43059b063d3c018afe3f26db772 Mon Sep 17 00:00:00 2001 From: Nestor Date: Sat, 13 Apr 2024 18:52:03 +0200 Subject: [PATCH 02/11] feat: grab database type based on url --- main.go | 2 -- pgdump.go | 92 +++++++++++++++++++++++++++++-------------------------- 2 files changed, 48 insertions(+), 46 deletions(-) diff --git a/main.go b/main.go index 851d00c..e31158b 100644 --- a/main.go +++ b/main.go @@ -26,7 +26,6 @@ func main() { s3Bucket := flag.String("s3-bucket", lookupEnvOrString("S3_BUCKET", "postgres-backups"), "S3 bucket") s3AccessKey := flag.String("s3-access-key", lookupEnvOrString("S3_ACCESS_KEY", "minio"), "S3 access key") s3SecretKey := flag.String("s3-secret-key", lookupEnvOrString("S3_SECRET_KEY", "minioadmin"), "S3 secret key") - dbType := flag.String("db-type", lookupEnvOrString("DB_TYPE", "postgres"), "Type of database: postgres or mariadb") interval := flag.Duration("interval", lookupEnvOrDuration("INTERVAL", 24*time.Hour), "How often to run the backup") @@ -64,7 +63,6 @@ func main() { Database: strings.TrimPrefix(parsedUrl.Path, "/"), Username: parsedUrl.User.Username(), Password: password, - DbType: *dbType, } } if len(urls) == 0 { diff --git a/pgdump.go b/pgdump.go index be743f3..d316801 100644 --- a/pgdump.go +++ b/pgdump.go @@ -1,9 +1,9 @@ package main import ( - "bufio" "errors" "fmt" + "net/url" "os" "os/exec" "strconv" @@ -16,78 +16,82 @@ type connectionOptions struct { Database string Username string Password string - DbType string } var ( PGDumpCmd = "pg_dump" pgDumpStdOpts = []string{"--no-owner", "--no-acl", "--clean", "--blobs", "-v"} pgDumpDefaultFormat = "c" + ErrPgDumpNotFound = errors.New("pg_dump not found") - ErrPgDumpNotFound = errors.New("pg_dump not found") + MysqlDumpCmd = "mysqldump" + mysqlDumpStdOpts = []string{"--compact", "--skip-add-drop-table", "--skip-add-locks", "--skip-disable-keys", "--skip-set-charset", "-v"} ErrMySqlDumpNotFound = errors.New("mysqldump not found") + + ErrUnsupportedType = errors.New("unsupported database type") ) func RunDump(connectionOpts *connectionOptions, outFile string) error { - var cmd *exec.Cmd + parsedUrl, err := url.Parse(connectionOpts.Host) + if err != nil { + return err + } + + cmd, err := buildDumpCommand(parsedUrl.Scheme, connectionOpts, outFile) + if err != nil { + return err + } + + return executeCommand(cmd) +} - if connectionOpts.DbType == "postgres" { +func buildDumpCommand(scheme string, opts *connectionOptions, outFile string) (*exec.Cmd, error) { + switch scheme { + case "postgres": if !commandExist(PGDumpCmd) { - return ErrPgDumpNotFound + return nil, ErrPgDumpNotFound } - options := append( pgDumpStdOpts, - fmt.Sprintf(`-f%s`, outFile), - fmt.Sprintf(`--dbname=%v`, connectionOpts.Database), - fmt.Sprintf(`--host=%v`, connectionOpts.Host), - fmt.Sprintf(`--port=%v`, connectionOpts.Port), - fmt.Sprintf(`--username=%v`, connectionOpts.Username), - fmt.Sprintf(`--format=%v`, pgDumpDefaultFormat), + fmt.Sprintf("-f%s", outFile), + fmt.Sprintf("--dbname=%s", opts.Database), + fmt.Sprintf("--host=%s", opts.Host), + fmt.Sprintf("--port=%d", opts.Port), + fmt.Sprintf("--username=%s", opts.Username), + fmt.Sprintf("--format=%s", pgDumpDefaultFormat), ) - cmd = exec.Command(PGDumpCmd, options...) - } else if connectionOpts.DbType == "mariadb" { + return exec.Command(PGDumpCmd, options...), nil + + case "mysql": mysqldumpCmd := "mysqldump" if !commandExist(mysqldumpCmd) { - return ErrMySqlDumpNotFound + return nil, ErrMySqlDumpNotFound } - - options := []string{ - "-h", connectionOpts.Host, - "-P", strconv.Itoa(connectionOpts.Port), - "-u", connectionOpts.Username, - fmt.Sprintf(`--password=%s`, connectionOpts.Password), - "--databases", connectionOpts.Database, + options := append( + mysqlDumpStdOpts, + "-h", opts.Host, + "-P", strconv.Itoa(opts.Port), + "-u", opts.Username, + fmt.Sprintf("--password=%s", opts.Password), + "--databases", opts.Database, "-r", outFile, - } - cmd = exec.Command(mysqldumpCmd, options...) - } else { - return errors.New("unsupported database type") - } + ) - cmd.Env = append(os.Environ(), fmt.Sprintf(`PGPASSWORD=%v`, connectionOpts.Password)) + return exec.Command(mysqldumpCmd, options...), nil - stderr, err := cmd.StderrPipe() - if err != nil { - return err + default: + return nil, ErrUnsupportedType } +} - if err = cmd.Start(); err != nil { +func executeCommand(cmd *exec.Cmd) error { + cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", cmd.Args[9])) // Assumes password is at index 9 + if err := cmd.Start(); err != nil { return err } - - go func() { - scanner := bufio.NewScanner(stderr) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - fmt.Println(scanner.Text()) - } - }() - - if err = cmd.Wait(); err != nil { + if err := cmd.Wait(); err != nil { return err } - return nil } From 07d4d8c434e65cd9e54b2a6f78555c44b4a9a6b7 Mon Sep 17 00:00:00 2001 From: Nestor Date: Sat, 13 Apr 2024 18:55:58 +0200 Subject: [PATCH 03/11] feat: add logs back --- pgdump.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pgdump.go b/pgdump.go index d316801..346e61a 100644 --- a/pgdump.go +++ b/pgdump.go @@ -1,10 +1,10 @@ package main import ( + "bufio" "errors" "fmt" "net/url" - "os" "os/exec" "strconv" "time" @@ -85,10 +85,22 @@ func buildDumpCommand(scheme string, opts *connectionOptions, outFile string) (* } func executeCommand(cmd *exec.Cmd) error { - cmd.Env = append(os.Environ(), fmt.Sprintf("PGPASSWORD=%s", cmd.Args[9])) // Assumes password is at index 9 + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { return err } + + go func() { + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + fmt.Println(scanner.Text()) + } + }() + if err := cmd.Wait(); err != nil { return err } From 516b4e5b72db8f61918b2a2e39101ee9b94ba754 Mon Sep 17 00:00:00 2001 From: Nestor Date: Sat, 13 Apr 2024 18:58:34 +0200 Subject: [PATCH 04/11] chore: rename file to dump.go --- pgdump.go => dump.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pgdump.go => dump.go (100%) diff --git a/pgdump.go b/dump.go similarity index 100% rename from pgdump.go rename to dump.go From fb4e7e85aae75f19a7e0aaace05bdcaaed72f8bd Mon Sep 17 00:00:00 2001 From: Nestor Date: Sat, 13 Apr 2024 19:01:50 +0200 Subject: [PATCH 05/11] chore: update readme to account for mysql --- .env.dist | 18 ++++++------------ README.md | 16 ++++++++-------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.env.dist b/.env.dist index 84241a1..238a749 100644 --- a/.env.dist +++ b/.env.dist @@ -1,12 +1,6 @@ -S3_ENDPOINT= -S3_BUCKET=postgres-backups -S3_ACCESS_KEY= -S3_SECRET_KEY= - -POSTGRES_HOST=postgres -POSTGRES_PORT=5432 -POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres -POSTGRES_DB=postgres - -EVERY=24h \ No newline at end of file +URLS="postgres://user:password@host:port/dbname,mysql://user:password@host:port/dbname" +S3_ENDPOINT="your_s3_endpoint" +S3_BUCKET="your_s3_bucket" +S3_ACCESS_KEY="your_s3_access_key" +S3_SECRET_KEY="your_s3_secret_key" +INTERVAL="24h" diff --git a/README.md b/README.md index dadb930..5dcabb6 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ -# PostgreSQL Backup to S3 with Docker +# Database Backup to S3 with Docker -This application automates the process of backing up PostgreSQL databases and uploading them to an S3-compatible storage service, utilizing Docker for easy deployment and scheduling. +This application automates the process of backing up PostgreSQL and MySQL databases and uploading them to an S3-compatible storage service, utilizing Docker for easy deployment and scheduling. ## Features - Easy deployment with Docker and Docker Compose. -- Support for multiple PostgreSQL databases. +- Support for multiple PostgreSQL and MySQL databases. - Customizable backup intervals. - Direct upload of backups to an S3-compatible storage bucket. - Environment variable and command-line configuration for flexibility. @@ -14,7 +14,7 @@ This application automates the process of backing up PostgreSQL databases and up ## Prerequisites - Docker and Docker Compose installed on your system. -- Access to a PostgreSQL database. +- Access to PostgreSQL and/or MySQL databases. - Access to an S3-compatible storage service. ## Configuration @@ -25,7 +25,7 @@ Before running the application, you need to configure it either by setting envir Create a `.env` file in the project directory with the following variables: -- `URLS`: Comma-separated list of PostgreSQL database URLs to backup. Format: `postgres://:@[:]/` +- `URLS`: Comma-separated list of database URLs to backup. Format for PostgreSQL: `postgres://:@[:]/` and for MySQL: `mysql://:@[:]/` - `S3_ENDPOINT`: The endpoint URL of your S3-compatible storage service. - `S3_BUCKET`: The name of the bucket where backups will be stored. - `S3_ACCESS_KEY`: Your S3 access key. @@ -41,7 +41,7 @@ services: app: build: . environment: - URLS: "postgres://user:password@host:port/dbname" + URLS: "postgres://user:password@host:port/dbname,mysql://user:password@host:port/dbname" S3_ENDPOINT: "your_s3_endpoint" S3_BUCKET: "your_s3_bucket" S3_ACCESS_KEY: "your_s3_access_key" @@ -51,7 +51,7 @@ services: ## Running the Application with Docker -There is an image available on `ghcr.io/thedevminertv/postgres_s3_backup` that you can use. +There is an image available on `ghcr.io/thedevminertv/database_s3_backup` that you can use. Alternatively, you can build the image yourself: @@ -67,7 +67,7 @@ Alternatively, you can build the image yourself: docker compose up -d ``` -This will start the application in the background. It will automatically perform backups based on the configured interval and upload them to the specified S3 bucket. +This will start the application in the background. It will automatically perform backups for both PostgreSQL and MySQL databases based on the configured interval and upload them to the specified S3 bucket. ## Monitoring and Logs From c92e0afa7a3ac20490520404466fba2cc0d72766 Mon Sep 17 00:00:00 2001 From: Nestor Date: Sat, 13 Apr 2024 19:02:45 +0200 Subject: [PATCH 06/11] chore: update image name to be generic --- .github/workflows/ghcr.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index 6f1919b..423fd9f 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -12,7 +12,7 @@ on: env: REGISTRY: ghcr.io - FQDN: "ghcr.io/thedevminertv/postgres-s3-backup" + FQDN: "ghcr.io/thedevminertv/database-s3-backup" jobs: build: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 59d446e..2250ca4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: env: REGISTRY: ghcr.io - FQDN: "ghcr.io/thedevminertv/postgres-s3-backup" + FQDN: "ghcr.io/thedevminertv/database-s3-backup" jobs: build: From a219d6242c83884b6d381b273edba12bff95b6ff Mon Sep 17 00:00:00 2001 From: Nestor Date: Sat, 13 Apr 2024 19:46:26 +0200 Subject: [PATCH 07/11] feat: properly parse db type based on url --- dump.go | 13 ++++--------- main.go | 4 ++-- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/dump.go b/dump.go index 346e61a..29ab1d1 100644 --- a/dump.go +++ b/dump.go @@ -4,7 +4,6 @@ import ( "bufio" "errors" "fmt" - "net/url" "os/exec" "strconv" "time" @@ -12,6 +11,7 @@ import ( type connectionOptions struct { Host string + DbType string Port int Database string Username string @@ -32,12 +32,7 @@ var ( ) func RunDump(connectionOpts *connectionOptions, outFile string) error { - parsedUrl, err := url.Parse(connectionOpts.Host) - if err != nil { - return err - } - - cmd, err := buildDumpCommand(parsedUrl.Scheme, connectionOpts, outFile) + cmd, err := buildDumpCommand(connectionOpts, outFile) if err != nil { return err } @@ -45,8 +40,8 @@ func RunDump(connectionOpts *connectionOptions, outFile string) error { return executeCommand(cmd) } -func buildDumpCommand(scheme string, opts *connectionOptions, outFile string) (*exec.Cmd, error) { - switch scheme { +func buildDumpCommand(opts *connectionOptions, outFile string) (*exec.Cmd, error) { + switch opts.DbType { case "postgres": if !commandExist(PGDumpCmd) { return nil, ErrPgDumpNotFound diff --git a/main.go b/main.go index e31158b..6691f44 100644 --- a/main.go +++ b/main.go @@ -43,7 +43,7 @@ func main() { log.Fatalf("Failed to parse url %s: %s", rawUrl, err) } - port := 5432 + port := 0 rawPort := parsedUrl.Port() if rawPort != "" { port, err = strconv.Atoi(rawPort) @@ -59,6 +59,7 @@ func main() { urls[i] = connectionOptions{ Host: parsedUrl.Hostname(), + DbType: parsedUrl.Scheme, Port: port, Database: strings.TrimPrefix(parsedUrl.Path, "/"), Username: parsedUrl.User.Username(), @@ -80,7 +81,6 @@ func main() { for { for _, u := range urls { log.Printf("Backing up %s", u.Database) - file := newFileName(u.Database) if err = RunDump(&u, file); err != nil { From e963f1b5853389fb44a067e41c48dcadef099d87 Mon Sep 17 00:00:00 2001 From: Nestor Date: Sat, 13 Apr 2024 19:58:56 +0200 Subject: [PATCH 08/11] feat: add different dbTypes file format --- dump.go | 10 ++++++++-- main.go | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/dump.go b/dump.go index 29ab1d1..a3d2ced 100644 --- a/dump.go +++ b/dump.go @@ -107,6 +107,12 @@ func commandExist(command string) bool { return err == nil } -func newFileName(db string) string { - return fmt.Sprintf(`%v_%v.pgdump`, db, time.Now().Unix()) +func newFileName(db string, dbType string) string { + switch dbType { + case "postgres": + return fmt.Sprintf(`%v_%v.pgdump`, db, time.Now().Unix()) + case "mysql": + return fmt.Sprintf(`%v_%v.sql`, db, time.Now().Unix()) + } + return fmt.Sprintf(`%v_%v`, db, time.Now().Unix()) } diff --git a/main.go b/main.go index 6691f44..b005e00 100644 --- a/main.go +++ b/main.go @@ -81,7 +81,7 @@ func main() { for { for _, u := range urls { log.Printf("Backing up %s", u.Database) - file := newFileName(u.Database) + file := newFileName(u.Database, u.DbType) if err = RunDump(&u, file); err != nil { log.Printf("WARNING: Failed to dump database: %s", err) From d18c651dad50fbddad3c6af8e0438536bfd3263a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N=C3=A9stor?= Date: Sun, 14 Apr 2024 21:02:58 +0200 Subject: [PATCH 09/11] chore: ypdate README.md Co-authored-by: DevMiner --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5dcabb6..ecb2bfd 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ services: ## Running the Application with Docker -There is an image available on `ghcr.io/thedevminertv/database_s3_backup` that you can use. +There is an image available on `ghcr.io/thedevminertv/database-s3-backup` that you can use. Alternatively, you can build the image yourself: From 4ac531c69f9d1981b9afdd46882293f6ec60eede Mon Sep 17 00:00:00 2001 From: Nestor Date: Sun, 14 Apr 2024 21:03:53 +0200 Subject: [PATCH 10/11] chore: rename binary --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6fa743e..337cf74 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,6 @@ WORKDIR /root/ RUN apk --no-cache add ca-certificates postgresql-client mysql-client -COPY --from=builder /app/main /bin/postgres-s3-backup +COPY --from=builder /app/main /bin/database-s3-backup -CMD ["/bin/postgres-s3-backup"] +CMD ["/bin/database-s3-backup"] From ce2cf993e84f3e778e20e86265faa5c5fd60e921 Mon Sep 17 00:00:00 2001 From: Nestor Date: Sun, 14 Apr 2024 21:11:25 +0200 Subject: [PATCH 11/11] feat: parse default port from database type --- main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/main.go b/main.go index b005e00..d573bf5 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,12 @@ func main() { } port := 0 + switch parsedUrl.Scheme { + case "postgres": + port = 5432 + case "mysql": + port = 3306 + } rawPort := parsedUrl.Port() if rawPort != "" { port, err = strconv.Atoi(rawPort)