diff --git a/README.md b/README.md index ec5b1f0..175fdda 100644 --- a/README.md +++ b/README.md @@ -60,22 +60,30 @@ This will upload `file1.txt` and `file2.txt` to S3 bucket and then perform S3 ob ## Usage -1. Ensure you have AWS credentials configured. You can set them up using the AWS CLI(ignore if already set): +1. Ensure you have AWS credentials configured. You can set them up using the AWS CLI (ignore if already set): ```sh aws configure --profile ``` -2. Setup Environment variables +2. Run the s3-cleaner cli + ```sh + s3-cleaner -p pratikkumar-mohite-aws -r us-east-1 -b pratikkumar-mohite-test + ``` +3. Alternatively, Setup Environment variables and then run s3-cleaner cli ```sh export AWS_REGION=us-east-1 export AWS_DELETE_S3_BUCKET=pratikkumar-mohite-test export AWS_PROFILE=pratikkumar-mohite-aws - ``` -3. Run the application: - ```sh - s3-cleaner + + $ s3-cleaner ``` ![Usage](docs/gif/s3-cleaner-usage.gif) +## Note +1. In case you get following issue, there might problem with bucket region. +```sh +FATA[0002] Unable to list objects, %!v(MISSING)operation error S3: ListObjectsV2, https response error StatusCode: 301, RequestID: FJFNV6SB70432CZT, HostID: testB9w==, api error PermanentRedirect: The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint. +``` + ## Contributing -Contributions are greatly appreciated. We actively manage the issues list, and try to highlight issues suitable for newcomers. The project follows the typical GitHub pull request model. Before starting any work, please either comment on an existing issue, or file a new one. +Contributions are greatly appreciated. We actively manage the issues list, and try to highlight issues suitable for newcomers. The project follows the typical GitHub pull request model. Before starting any work, please either comment on an existing issue, or file a new one. \ No newline at end of file diff --git a/cmd/s3-cleaner/cleanup.go b/cmd/s3-cleaner/cleanup.go index afc9b3a..326c9dc 100644 --- a/cmd/s3-cleaner/cleanup.go +++ b/cmd/s3-cleaner/cleanup.go @@ -1,18 +1,18 @@ package main import ( - "time" "sync" + "time" - log "github.com/sirupsen/logrus" "github.com/pratikkumar-mohite/s3-cleaner/pkg/aws" + log "github.com/sirupsen/logrus" ) -func setup() aws.S3Client { - config := aws.AWSConnection(getFromEnv("AWS_PROFILE")) +func setup(profile, region, bucket string) aws.S3Client { + + config := aws.AWSConnection(profile, region) client := aws.S3Connection(config) - client.Bucket = getFromEnv("AWS_DELETE_S3_BUCKET") - _ = getFromEnv("AWS_REGION") + client.Bucket = bucket return client } @@ -24,12 +24,15 @@ func s3Upload(s3Client aws.S3Client) { s3Client.DeleteS3BucketObjectVersion("file1.txt", object1) } -func s3Cleanup() { - s3Client := setup() - - bucket := s3Client.GetS3Bucket(s3Client.Bucket) +func s3Cleanup(profile, region, bucket *string) { + var s3Client aws.S3Client + if *profile != "" && *region != "" && *bucket != "" { + s3Client = setup(*profile, *region, *bucket) + } else { + s3Client = setup(getFromEnv("AWS_PROFILE"), getFromEnv("AWS_REGION"), getFromEnv("AWS_DELETE_S3_BUCKET")) + } - if bucket == "" { + if s3Client.GetS3Bucket(s3Client.Bucket) == "" { log.Fatalf("Bucket %s not found\n", s3Client.Bucket) } @@ -61,7 +64,7 @@ func s3Cleanup() { if len(object.ObjectVersion) == 0 { versionWG.Add(1) - go func(object_name string){ + go func(object_name string) { defer versionWG.Done() s3Client.DeleteS3BucketObject(object_name) }(object.ObjectName) @@ -69,7 +72,7 @@ func s3Cleanup() { for _, version := range object.ObjectVersion { versionWG.Add(1) - go func(version string){ + go func(version string) { defer versionWG.Done() s3Client.DeleteS3BucketObjectVersion(object.ObjectName, version) }(version) @@ -80,7 +83,7 @@ func s3Cleanup() { } }() - for _, object := range objects{ + for _, object := range objects { wg.Add(1) objectChan <- object } diff --git a/cmd/s3-cleaner/main.go b/cmd/s3-cleaner/main.go index aabbc73..4ec6d0c 100644 --- a/cmd/s3-cleaner/main.go +++ b/cmd/s3-cleaner/main.go @@ -1,5 +1,24 @@ package main +import ( + "github.com/spf13/cobra" +) + func main() { - s3Cleanup() + var profile, region, bucket string + var rootCmd = &cobra.Command{ + Use: "s3-cleaner", + Short: "A cli to clean and destroy s3 bucket", + Long: `A cli to clean and destroy s3 bucket.`, + Run: func(cmd *cobra.Command, args []string) { + s3Cleanup(&profile, ®ion, &bucket) + }, + } + + rootCmd.PersistentFlags().StringVarP(&profile, "profile", "p", "", "AWS Profile (Required)") + rootCmd.PersistentFlags().StringVarP(®ion, "region", "r", "", "AWS Region (Required)") + rootCmd.PersistentFlags().StringVarP(&bucket, "bucket", "b", "", "AWS S3 bucket (Required)") + rootCmd.MarkFlagsRequiredTogether("profile", "region", "bucket") + + rootCmd.Execute() } diff --git a/go.mod b/go.mod index 0194c0d..eebadc8 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.27.27 github.com/aws/aws-sdk-go-v2/service/s3 v1.58.3 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/cobra v1.8.1 ) require ( @@ -25,5 +26,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect github.com/aws/smithy-go v1.20.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect ) diff --git a/go.sum b/go.sum index a5c85e0..c39f236 100644 --- a/go.sum +++ b/go.sum @@ -34,18 +34,27 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 h1:ZsDKRLXGWHk8WdtyYMoGNO7bTudr github.com/aws/aws-sdk-go-v2/service/sts v1.30.3/go.mod h1:zwySh8fpFyXp9yOr/KVzxOl8SRqgf/IDw5aUt9UKFcQ= github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/aws/connection.go b/pkg/aws/connection.go index 16dedd9..7b8c44a 100644 --- a/pkg/aws/connection.go +++ b/pkg/aws/connection.go @@ -3,17 +3,18 @@ package aws import ( "context" - log "github.com/sirupsen/logrus" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" + log "github.com/sirupsen/logrus" ) -func AWSConnection(profile string) aws.Config { +func AWSConnection(profile, region string) aws.Config { cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithSharedConfigProfile(profile)) if err != nil { log.Fatalf("Unable to load SDK config, %v", err) } + cfg.Region = region return cfg }