add list pagination logic

- adjust upload.go accordingly

helpers/helpers.go

@@ -433,7 +433,7 @@ func CheckTokenExpiration(accessToken string) error {
 	return nil
 }
 
-func ListFiles(config Config, prefix string) (result *s3.ListObjectsV2Output, err error) {
+func ListFiles(config Config, prefix string) ([]*s3.Object, error) {
 	sess := session.Must(session.NewSession(&aws.Config{
 		// The region for the backend is always the specified one
 		// and not present in the configuration from auth - hardcoded
@@ -450,15 +450,31 @@ func ListFiles(config Config, prefix string) (result *s3.ListObjectsV2Output, er
 
 	svc := s3.New(sess)
 
-	result, err = svc.ListObjectsV2(&s3.ListObjectsV2Input{
-		Bucket: aws.String(config.AccessKey + "/"),
-		Prefix: aws.String(config.AccessKey + "/" + prefix),
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to list objects, reason: %v", err)
+	var continuationToken *string
+	var fullResult []*s3.Object
+
+	for {
+		result, err := svc.ListObjectsV2(&s3.ListObjectsV2Input{
+			Bucket:            aws.String(config.AccessKey + "/"),
+			Prefix:            aws.String(config.AccessKey + "/" + prefix),
+			ContinuationToken: continuationToken,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to list objects, reason: %v", err)
+		}
+
+		if result.Contents != nil {
+			fullResult = append(fullResult, result.Contents...)
+		}
+
+		if result.NextContinuationToken == nil {
+			break
+		}
+
+		continuationToken = result.NextContinuationToken
 	}
 
-	return result, nil
+	return fullResult, nil
 }
 
 // Check for invalid characters

list/list.go

@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"strconv"
-
 	"strings"
 
 	"github.com/NBISweden/sda-cli/download"
@@ -100,9 +99,9 @@ func List(args []string, configPath string) error {
 		return err
 	}
 
-	for i := range result.Contents {
-		file := *result.Contents[i].Key
-		fmt.Printf("%s \t %s \n", bytesize.New(float64((*result.Contents[i].Size))), file[strings.Index(file, "/")+1:])
+	for i := range result {
+		file := *result[i].Key
+		fmt.Printf("%s \t %s \n", bytesize.New(float64((*result[i].Size))), file[strings.Index(file, "/")+1:])
 	}
 
 	return nil

upload/upload.go

@@ -156,9 +156,9 @@ func uploadFiles(files, outFiles []string, targetDir string, config *helpers.Con
 		if err != nil {
 			return fmt.Errorf("listing uploaded files: %s", err.Error())
 		}
-		if len(fileExists.Contents) > 0 {
+		if len(fileExists) > 0 {
 			if aws.StringValue(
-				fileExists.Contents[0].Key,
+				fileExists[0].Key,
 			) == filepath.Clean(
 				config.AccessKey+"/"+targetDir+"/"+outFiles[k],
 			) {