diff --git a/pkg/.golangci.yaml b/pkg/.golangci.yaml new file mode 100644 index 000000000..dc19bb965 --- /dev/null +++ b/pkg/.golangci.yaml @@ -0,0 +1,9 @@ +linters-settings: + gocyclo: + # Minimal code complexity to report. + # Default: 30 (but we recommend 10-20) + min-complexity: 100 + staticcheck: + # SAxxxx checks in https://staticcheck.io/docs/configuration/options/#checks + # Default: ["*"] + checks: ["all"] diff --git a/pkg/Makefile b/pkg/Makefile index 2af25b9a0..a55d047b2 100644 --- a/pkg/Makefile +++ b/pkg/Makefile @@ -5,13 +5,11 @@ all: build getdeps: @echo "Installing golint" && go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.55.0 - @echo "Installing gocyclo" && go install github.com/fzipp/gocyclo/cmd/gocyclo@latest @echo "Installing deadcode" && go install github.com/remyoudompheng/go-misc/deadcode@latest @echo "Installing misspell" && go install github.com/client9/misspell/cmd/misspell@latest @echo "Installing ineffassign" && go install github.com/gordonklaus/ineffassign@latest - @echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest -verifiers: vet fmt lint cyclo spelling static #deadcode +verifiers: vet fmt lint spelling#deadcode vet: @echo "Running $@" @@ -29,10 +27,6 @@ ineffassign: @echo "Running $@" @${GOPATH}/bin/ineffassign . -cyclo: - @echo "Running $@" - @${GOPATH}/bin/gocyclo -over 100 . - deadcode: @echo "Running $@" @${GOPATH}/bin/deadcode -test $(shell go list ./...) || true @@ -40,9 +34,6 @@ deadcode: spelling: @${GOPATH}/bin/misspell -i "monitord,forumla,etherent" -error `find .` -static: - @${GOPATH}/bin/staticcheck -- ./... - check: test test: verifiers build go test -vet=off -v $(shell go list ./... | grep -Ev "stubs|network" ) diff --git a/pkg/storage/disk.go b/pkg/storage/disk.go index f9083717c..a2a78cd17 100644 --- a/pkg/storage/disk.go +++ b/pkg/storage/disk.go @@ -14,6 +14,8 @@ import ( log "github.com/rs/zerolog/log" "github.com/threefoldtech/zos/pkg" "github.com/threefoldtech/zos/pkg/gridtypes" + "github.com/threefoldtech/zos/pkg/gridtypes/zos" + "golang.org/x/sync/errgroup" ) const ( @@ -145,7 +147,15 @@ func (s *Module) DiskWrite(name string, image string) error { return fmt.Errorf("image size is bigger than disk") } - _, err = io.Copy(file, source) + // do sequential copy on HDD only node + // otherwise do concurrent copy + if n, _ := s.Total(zos.SSDDevice); n == uint64(0) { + _, err = io.Copy(file, source) + + } else { + err = s.concurrentCopy(file, source, imgStat) + } + if err != nil { return errors.Wrap(err, "failed to write disk image") } @@ -153,6 +163,39 @@ func (s *Module) DiskWrite(name string, image string) error { return nil } +func (s *Module) concurrentCopy(dst io.WriterAt, src io.ReaderAt, imgStat os.FileInfo) error { + // writing the image concurrently to speedup the previous sequential write. + // the sequential write is slow because the data source is from the remote server. + var ( + // use errgroup because there is no point in continuing if one of the goroutines failed + group = new(errgroup.Group) + numWorkers int = 5 + imgSize int64 = imgStat.Size() + chunkSize = imgSize / int64(numWorkers) + ) + + log.Info().Int("numWorkers", numWorkers).Msg("writing image concurrently") + for i := 0; i < numWorkers; i++ { + index := i + group.Go(func() error { + start := chunkSize * int64(index) + len := chunkSize + if index == numWorkers-1 { //last chunk + len = imgSize - start + } + wr := io.NewOffsetWriter(dst, start) + rd := io.NewSectionReader(src, start, len) + _, err := io.Copy(wr, rd) + return err + }) + } + if err := group.Wait(); err != nil { + return err + } + log.Info().Msg("writing image concurrently finished") + return nil +} + // DiskCreate with given size, return path to virtual disk (size in MB) func (s *Module) DiskCreate(name string, size gridtypes.Unit) (disk pkg.VDisk, err error) { path, err := s.findDisk(name)