diff --git a/helpers/helpers.go b/helpers/helpers.go index 86678185..64a2bb94 100644 --- a/helpers/helpers.go +++ b/helpers/helpers.go @@ -433,7 +433,7 @@ func CheckTokenExpiration(accessToken string) error { return nil } -func ListFiles(config Config, prefix string) (result *s3.ListObjectsV2Output, err error) { +func ListFiles(config Config, prefix string) ([]*s3.Object, error) { sess := session.Must(session.NewSession(&aws.Config{ // The region for the backend is always the specified one // and not present in the configuration from auth - hardcoded @@ -450,15 +450,31 @@ func ListFiles(config Config, prefix string) (result *s3.ListObjectsV2Output, er svc := s3.New(sess) - result, err = svc.ListObjectsV2(&s3.ListObjectsV2Input{ - Bucket: aws.String(config.AccessKey + "/"), - Prefix: aws.String(config.AccessKey + "/" + prefix), - }) - if err != nil { - return nil, fmt.Errorf("failed to list objects, reason: %v", err) + var continuationToken *string + var fullResult []*s3.Object + + for { + result, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(config.AccessKey + "/"), + Prefix: aws.String(config.AccessKey + "/" + prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, fmt.Errorf("failed to list objects, reason: %v", err) + } + + if result.Contents != nil { + fullResult = append(fullResult, result.Contents...) + } + + if result.NextContinuationToken == nil { + break + } + + continuationToken = result.NextContinuationToken } - return result, nil + return fullResult, nil } // Check for invalid characters diff --git a/list/list.go b/list/list.go index f9c392d2..13f838a7 100644 --- a/list/list.go +++ b/list/list.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "strconv" - "strings" "github.com/NBISweden/sda-cli/download" @@ -100,9 +99,9 @@ func List(args []string, configPath string) error { return err } - for i := range result.Contents { - file := *result.Contents[i].Key - fmt.Printf("%s \t %s \n", bytesize.New(float64((*result.Contents[i].Size))), file[strings.Index(file, "/")+1:]) + for i := range result { + file := *result[i].Key + fmt.Printf("%s \t %s \n", bytesize.New(float64((*result[i].Size))), file[strings.Index(file, "/")+1:]) } return nil diff --git a/upload/upload.go b/upload/upload.go index 07c893b2..01eb120e 100644 --- a/upload/upload.go +++ b/upload/upload.go @@ -156,9 +156,9 @@ func uploadFiles(files, outFiles []string, targetDir string, config *helpers.Con if err != nil { return fmt.Errorf("listing uploaded files: %s", err.Error()) } - if len(fileExists.Contents) > 0 { + if len(fileExists) > 0 { if aws.StringValue( - fileExists.Contents[0].Key, + fileExists[0].Key, ) == filepath.Clean( config.AccessKey+"/"+targetDir+"/"+outFiles[k], ) {