parallel upload

Author: zhangliang

Diff for: README.md

@@ -1,5 +1,5 @@
 # drone-qiniu
-上传文件到七牛的drone插件
+上传文件到七牛的 drone 插件
 
 ## Build
 ```
@@ -12,9 +12,9 @@ GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o qiniu
 SET ACCESS_KEY=xxx
 SET SECRET_KEY=xxx
 SET ZONE=huadong
-SET BUCKET=nameimtest
+SET BUCKET=mybucket
 SET PREFIX=drone/
-SET PATH=e:\drone-qiniu
+SET dir=./dist
 ```
 
 ## Usage

Diff for: go.mod

@@ -3,6 +3,7 @@ module github.com/zbysir/drone-qiniu
 go 1.15
 
 require (
+	github.com/dustin/go-humanize v1.0.1
 	github.com/qiniu/go-sdk/v7 v7.21.1
 	github.com/urfave/cli/v2 v2.3.0
 )

Diff for: go.sum

@@ -9,6 +9,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/dave/jennifer v1.6.1/go.mod h1:nXbxhEmQfOZhWml3D1cDK5M1FLnMSozpbFN/m3RmGZc=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=

Diff for: lib/qiniu/upload.go

@@ -3,13 +3,14 @@ package qiniu
 import (
 	"context"
 	"fmt"
+	"github.com/dustin/go-humanize"
+	"github.com/qiniu/go-sdk/v7/auth/qbox"
+	"github.com/qiniu/go-sdk/v7/storage"
 	"io"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
-
-	"github.com/qiniu/go-sdk/v7/auth/qbox"
-	"github.com/qiniu/go-sdk/v7/storage"
 )
 
 type Uploader struct {
@@ -24,9 +25,40 @@ type PutRsp struct {
 	Bucket string
 }
 
+type Assets struct {
+	Path string
+	Key  string
+}
+
 // UploadDir 上传一个文件夹
-func (u Uploader) UploadDir(zone *storage.Zone, bucket string, keyPrefix string, dirPath string) (err error) {
-	fmt.Printf("upload dir: '%s' to bucket '%s', prefix: '%s'\n", dirPath, bucket, keyPrefix)
+func (u Uploader) UploadDir(zone *storage.Zone, bucket string, keyPrefix string, dirPath string, parallel int) (err error) {
+	if parallel == 0 {
+		parallel = 5
+	}
+	fmt.Printf("upload dir: '%s' to bucket '%s', prefix: '%s', parallel: '%d'\n", dirPath, bucket, keyPrefix, parallel)
+
+	as := make(chan Assets, 10)
+
+	// 上传
+	var wg sync.WaitGroup
+	for i := 0; i < parallel; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for item := range as {
+				if err != nil {
+					return
+				}
+				start := time.Now()
+				rsp, e := u.UploadFile(zone, bucket, item.Key, item.Path)
+				if e != nil {
+					err = fmt.Errorf("upload file '%s' error: %w", item.Key, err)
+					return
+				}
+				fmt.Printf("uploaded file '%s' success, size: %s, spend time: %v\n", item.Key, humanize.IBytes(uint64(rsp.Fsize)), time.Since(start))
+			}
+		}()
+	}
 
 	err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
@@ -41,19 +73,19 @@ func (u Uploader) UploadDir(zone *storage.Zone, bucket string, keyPrefix string,
 			return err
 		}
 
-		start := time.Now()
-		_, err = u.UploadFile(zone, bucket, keyPrefix+reaPath, path)
-		if err != nil {
-			return fmt.Errorf("upload file '%s' error: %w", keyPrefix+reaPath, err)
+		as <- Assets{
+			Path: path,
+			Key:  keyPrefix + reaPath,
 		}
-		fmt.Printf("uploaded file '%s' success, spend time: %v\n", keyPrefix+reaPath, time.Since(start))
 
 		return nil
 	})
 	if err != nil {
 		return
 	}
-	return
+
+	wg.Wait()
+	return err
 }
 
 // UploadFile 服务端表单直传 + 自定义回  JSON

Diff for: lib/qiniu/upload_test.go

@@ -7,7 +7,7 @@ import (
 
 func TestUploadDir(t *testing.T) {
 	u := NewQiniu("", "").Uploader()
-	err := u.UploadDir(&storage.ZoneHuadong, "nameimtest", "test/", `../qiniu/`)
+	err := u.UploadDir(&storage.ZoneHuadong, "nameimtest", "test/", `../qiniu/`, 1)
 	if err != nil {
 		t.Fatal(err)
 	}

Diff for: main.go

@@ -7,7 +7,7 @@ import (
 
 func main() {
 	c := cli.NewApp()
-	c.Name = "qiniu plugin"
+	c.Name = "qiniu drone plugin"
 	c.Usage = ""
 	c.Version = ""
 
@@ -19,6 +19,7 @@ func main() {
 			Zone:      ctx.String("zone"),
 			Prefix:    ctx.String("prefix"),
 			Dir:       ctx.String("dir"),
+			Parallel:  ctx.Int("parallel"),
 		}
 
 		return p.Exec()
@@ -55,6 +56,11 @@ func main() {
 			Usage:   "local dir",
 			EnvVars: []string{"DIR", "PLUGIN_DIR"},
 		},
+		&cli.StringFlag{
+			Name:    "parallel",
+			Usage:   "parallel num",
+			EnvVars: []string{"PARALLEL", "PLUGIN_PARALLEL"},
+		},
 	}
 
 	err := c.Run(os.Args)

Diff for: plugin.go

@@ -12,6 +12,7 @@ type Plugin struct {
 	Bucket    string
 	Prefix    string
 	Dir       string
+	Parallel  int
 }
 
 func (p Plugin) Exec() error {
@@ -22,5 +23,5 @@ func (p Plugin) Exec() error {
 		return errors.New("bad zone")
 	}
 
-	return u.UploadDir(zone, p.Bucket, p.Prefix, p.Dir)
+	return u.UploadDir(zone, p.Bucket, p.Prefix, p.Dir, p.Parallel)
 }