From 92ea5f8a2709758f9f551811b510009f63adf40d Mon Sep 17 00:00:00 2001 From: Lucas Pinheiro Date: Fri, 1 Apr 2022 14:56:35 -0300 Subject: [PATCH] Windows cache CLI support (#313) --- .github/workflows/test.yml | 24 ++++++ .semaphore/release.yml | 1 + .semaphore/semaphore.yml | 6 +- cache-cli/Makefile | 10 ++- cache-cli/cmd/clear_test.go | 3 +- cache-cli/cmd/delete_test.go | 5 +- cache-cli/cmd/has_key_test.go | 5 +- cache-cli/cmd/is_not_empty_test.go | 3 +- cache-cli/cmd/list_test.go | 3 +- cache-cli/cmd/restore_test.go | 38 +++++---- cache-cli/cmd/root_test.go | 41 ++++++---- cache-cli/cmd/store.go | 4 +- cache-cli/cmd/store_test.go | 16 ++-- cache-cli/docker-compose.yml | 2 + cache-cli/pkg/files/checksum_test.go | 2 +- cache-cli/pkg/files/compress.go | 7 +- cache-cli/pkg/files/compress_test.go | 25 +++--- cache-cli/pkg/files/lookup.go | 4 +- cache-cli/pkg/files/lookup_test.go | 90 ++++++++++++++-------- cache-cli/pkg/files/unpack.go | 4 +- cache-cli/pkg/files/unpack_test.go | 6 +- cache-cli/pkg/metrics/local.go | 10 ++- cache-cli/pkg/storage/clear_test.go | 4 +- cache-cli/pkg/storage/delete_test.go | 2 +- cache-cli/pkg/storage/has_key_test.go | 2 +- cache-cli/pkg/storage/is_not_empty_test.go | 2 +- cache-cli/pkg/storage/list_test.go | 4 +- cache-cli/pkg/storage/restore_test.go | 2 +- cache-cli/pkg/storage/s3_restore.go | 2 +- cache-cli/pkg/storage/s3_store.go | 7 +- cache-cli/pkg/storage/sftp_restore.go | 2 +- cache-cli/pkg/storage/storage_test.go | 55 ++++++++----- cache-cli/pkg/storage/store_test.go | 63 ++++++++------- cache-cli/pkg/storage/usage_test.go | 2 +- install-self-hosted-toolbox.ps1 | 79 ++++++++++++------- release/create.sh | 24 +++++- 36 files changed, 371 insertions(+), 188 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..27b50a6d --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,24 @@ +on: [push] +name: Unit tests +jobs: + unit-testing: + runs-on: windows-latest + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: 1.16.x + - name: Check out repository code + uses: actions/checkout@v2 + - name: Install gotestsum + run: go install gotest.tools/gotestsum@latest + - name: Run tests + env: + SEMAPHORE_CACHE_S3_URL: "http://127.0.0.1:9000" + run: | + New-Item C:\minio -ItemType Directory > $null + Invoke-WebRequest "https://dl.min.io/server/minio/release/windows-amd64/minio.exe" -OutFile C:\minio\minio.exe + New-Item C:\minio\data\semaphore-cache -ItemType Directory > $null + Start-Process C:\minio\minio.exe -ArgumentList 'server','C:\minio\data' -RedirectStandardOutput C:\minio\logs -RedirectStandardError C:\minio\errors + cd cache-cli + gotestsum --format short-verbose --packages="./..." -- -p 1 diff --git a/.semaphore/release.yml b/.semaphore/release.yml index 9778ed27..26b398a5 100644 --- a/.semaphore/release.yml +++ b/.semaphore/release.yml @@ -16,5 +16,6 @@ blocks: - checkout - artifact pull workflow bin/linux/cache -d cache-cli/bin/linux/cache - artifact pull workflow bin/darwin/cache -d cache-cli/bin/darwin/cache + - artifact pull workflow bin/windows/cache.exe -d cache-cli/bin/windows/cache.exe - bash release/create.sh -a - bash release/upload.sh diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 1d9b7935..f29c0082 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -22,10 +22,12 @@ blocks: - name: Build cache CLI commands: - cd cache-cli - - make build OS=linux - - make build OS=darwin + - make build.linux + - make build.darwin + - make build.windows - artifact push workflow bin/linux/cache -d bin/linux/cache - artifact push workflow bin/darwin/cache -d bin/darwin/cache + - artifact push workflow bin/windows/cache.exe -d bin/windows/cache.exe - name: Static Code Analysis dependencies: [] diff --git a/cache-cli/Makefile b/cache-cli/Makefile index 342fe865..719ed13a 100644 --- a/cache-cli/Makefile +++ b/cache-cli/Makefile @@ -29,8 +29,14 @@ test: test.watch: docker-compose run --rm cli gotestsum --watch --format short-verbose --junitfile junit-report.xml --packages="./..." -- -p 1 -build: - CGO_ENABLED=0 GOOS=$(OS) go build -o bin/$(OS)/cache main.go +build.darwin: + CGO_ENABLED=0 GOOS=darwin go build -o bin/darwin/cache main.go + +build.linux: + CGO_ENABLED=0 GOOS=linux go build -o bin/linux/cache main.go + +build.windows: + CGO_ENABLED=0 GOOS=windows go build -o bin/windows/cache.exe main.go lint: revive -formatter friendly -config lint.toml ./... diff --git a/cache-cli/cmd/clear_test.go b/cache-cli/cmd/clear_test.go index aed58e43..c669bb13 100644 --- a/cache-cli/cmd/clear_test.go +++ b/cache-cli/cmd/clear_test.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "io/ioutil" + "os" "testing" "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" @@ -27,7 +28,7 @@ func Test__Clear(t *testing.T) { err := storage.Clear() assert.Nil(t, err) - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") storage.Store("abc001", tempFile.Name()) capturer := utils.CreateOutputCapturer() diff --git a/cache-cli/cmd/delete_test.go b/cache-cli/cmd/delete_test.go index b72f87b4..39327c06 100644 --- a/cache-cli/cmd/delete_test.go +++ b/cache-cli/cmd/delete_test.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "io/ioutil" + "os" "testing" "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" @@ -22,7 +23,7 @@ func Test__Delete(t *testing.T) { t.Run(fmt.Sprintf("%s key is present", backend), func(*testing.T) { storage.Clear() - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") storage.Store("abc001", tempFile.Name()) capturer := utils.CreateOutputCapturer() @@ -34,7 +35,7 @@ func Test__Delete(t *testing.T) { t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { storage.Clear() - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") RunStore(storeCmd, []string{"abc/00/33", tempFile.Name()}) capturer := utils.CreateOutputCapturer() diff --git a/cache-cli/cmd/has_key_test.go b/cache-cli/cmd/has_key_test.go index abe99e1c..e80c8f4b 100644 --- a/cache-cli/cmd/has_key_test.go +++ b/cache-cli/cmd/has_key_test.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "io/ioutil" + "os" "testing" "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" @@ -22,7 +23,7 @@ func Test__HasKey(t *testing.T) { t.Run(fmt.Sprintf("%s key is present", backend), func(*testing.T) { storage.Clear() - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") storage.Store("abc001", tempFile.Name()) capturer := utils.CreateOutputCapturer() @@ -34,7 +35,7 @@ func Test__HasKey(t *testing.T) { t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { storage.Clear() - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") RunStore(storeCmd, []string{"abc/00/33", tempFile.Name()}) capturer := utils.CreateOutputCapturer() diff --git a/cache-cli/cmd/is_not_empty_test.go b/cache-cli/cmd/is_not_empty_test.go index 866ea1bb..c2310615 100644 --- a/cache-cli/cmd/is_not_empty_test.go +++ b/cache-cli/cmd/is_not_empty_test.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "io/ioutil" + "os" "testing" "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" @@ -18,7 +19,7 @@ func Test__IsNotEmpty(t *testing.T) { t.Run(fmt.Sprintf("%s cache is not empty", backend), func(*testing.T) { storage.Clear() - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") storage.Store("abc001", tempFile.Name()) assert.True(t, RunIsNotEmpty(isNotEmptyCmd, []string{})) diff --git a/cache-cli/cmd/list_test.go b/cache-cli/cmd/list_test.go index 8344ea16..a31bf4e4 100644 --- a/cache-cli/cmd/list_test.go +++ b/cache-cli/cmd/list_test.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "io/ioutil" + "os" "testing" "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" @@ -24,7 +25,7 @@ func Test__List(t *testing.T) { t.Run(fmt.Sprintf("%s with keys", backend), func(*testing.T) { storage.Clear() - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") storage.Store("abc001", tempFile.Name()) storage.Store("abc002", tempFile.Name()) storage.Store("abc003", tempFile.Name()) diff --git a/cache-cli/cmd/restore_test.go b/cache-cli/cmd/restore_test.go index 043abe9e..c06eb232 100644 --- a/cache-cli/cmd/restore_test.go +++ b/cache-cli/cmd/restore_test.go @@ -37,8 +37,9 @@ func Test__Restore(t *testing.T) { t.Run(fmt.Sprintf("%s using single exact key", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") + _ = tempFile.Close() compressAndStore(storage, "abc-001", tempDir) @@ -46,8 +47,9 @@ func Test__Restore(t *testing.T) { RunRestore(restoreCmd, []string{"abc-001"}) output := capturer.Done() + restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir)) assert.Contains(t, output, "HIT: 'abc-001', using key 'abc-001'.") - assert.Contains(t, output, fmt.Sprintf("Restored: %s/.", tempDir)) + assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath)) os.Remove(tempFile.Name()) os.Remove(tempDir) @@ -56,8 +58,9 @@ func Test__Restore(t *testing.T) { t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") + _ = tempFile.Close() compressAndStore(storage, "abc/00/22", tempDir) @@ -65,9 +68,10 @@ func Test__Restore(t *testing.T) { RunRestore(restoreCmd, []string{"abc/00/22"}) output := capturer.Done() + restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir)) assert.Contains(t, output, "Key 'abc/00/22' is normalized to 'abc-00-22'") assert.Contains(t, output, "HIT: 'abc-00-22', using key 'abc-00-22'.") - assert.Contains(t, output, fmt.Sprintf("Restored: %s/.", tempDir)) + assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath)) os.Remove(tempFile.Name()) os.Remove(tempDir) @@ -76,8 +80,9 @@ func Test__Restore(t *testing.T) { t.Run(fmt.Sprintf("%s using single matching key", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") + _ = tempFile.Close() compressAndStore(storage, "abc-001", tempDir) @@ -85,8 +90,9 @@ func Test__Restore(t *testing.T) { RunRestore(restoreCmd, []string{"abc"}) output := capturer.Done() + restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir)) assert.Contains(t, output, "HIT: 'abc', using key 'abc-001'.") - assert.Contains(t, output, fmt.Sprintf("Restored: %s/.", tempDir)) + assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath)) os.Remove(tempFile.Name()) os.Remove(tempDir) @@ -95,8 +101,9 @@ func Test__Restore(t *testing.T) { t.Run(fmt.Sprintf("%s only first matching key is used", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") + _ = tempFile.Close() compressAndStore(storage, "abc-001", tempDir) compressAndStore(storage, "abc-002", tempDir) @@ -105,8 +112,9 @@ func Test__Restore(t *testing.T) { RunRestore(restoreCmd, []string{"abc-001,abc-002"}) output := capturer.Done() + restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir)) assert.Contains(t, output, "HIT: 'abc-001', using key 'abc-001'.") - assert.Contains(t, output, fmt.Sprintf("Restored: %s/.", tempDir)) + assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath)) assert.NotContains(t, output, "HIT: 'abc-002', using key 'abc-002'.") os.Remove(tempFile.Name()) @@ -116,8 +124,9 @@ func Test__Restore(t *testing.T) { t.Run(fmt.Sprintf("%s using fallback key", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") + _ = tempFile.Close() compressAndStore(storage, "abc", tempDir) @@ -125,9 +134,10 @@ func Test__Restore(t *testing.T) { RunRestore(restoreCmd, []string{"abc-001,abc"}) output := capturer.Done() + restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir)) assert.Contains(t, output, "MISS: 'abc-001'.") assert.Contains(t, output, "HIT: 'abc', using key 'abc'.") - assert.Contains(t, output, fmt.Sprintf("Restored: %s/.", tempDir)) + assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath)) os.Remove(tempFile.Name()) os.Remove(tempDir) @@ -136,8 +146,9 @@ func Test__Restore(t *testing.T) { t.Run(fmt.Sprintf("%s using regex key", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") + _ = tempFile.Close() compressAndStore(storage, "abc", tempDir) @@ -145,8 +156,9 @@ func Test__Restore(t *testing.T) { RunRestore(restoreCmd, []string{"^abc"}) output := capturer.Done() + restoredPath := filepath.FromSlash(fmt.Sprintf("%s/", tempDir)) assert.Contains(t, output, "HIT: '^abc', using key 'abc'.") - assert.Contains(t, output, fmt.Sprintf("Restored: %s/.", tempDir)) + assert.Contains(t, output, fmt.Sprintf("Restored: %s.", restoredPath)) os.Remove(tempFile.Name()) os.Remove(tempDir) @@ -191,7 +203,7 @@ func Test__AutomaticRestore(t *testing.T) { assert.Contains(t, output, "Detected Gemfile.lock") assert.Contains(t, output, fmt.Sprintf("Downloading key '%s'", key)) - assert.Contains(t, output, "Restored: vendor/bundle") + assert.Contains(t, output, fmt.Sprintf("Restored: %s", filepath.FromSlash("vendor/bundle"))) os.RemoveAll("vendor") os.Remove(compressedFile) diff --git a/cache-cli/cmd/root_test.go b/cache-cli/cmd/root_test.go index 4bf55a98..70af138c 100644 --- a/cache-cli/cmd/root_test.go +++ b/cache-cli/cmd/root_test.go @@ -2,36 +2,51 @@ package cmd import ( "os" + "runtime" "testing" "github.com/semaphoreci/toolbox/cache-cli/pkg/storage" assert "github.com/stretchr/testify/assert" ) -var testBackend = map[string]map[string]string{ +type TestBackend struct { + envVars map[string]string + runInWindows bool +} + +var testBackends = map[string]TestBackend{ "s3": { - "SEMAPHORE_PROJECT_NAME": "cache-cli", - "SEMAPHORE_CACHE_BACKEND": "s3", - "SEMAPHORE_CACHE_S3_URL": "http://s3:9000", - "SEMAPHORE_CACHE_S3_BUCKET": "semaphore-cache", + runInWindows: true, + envVars: map[string]string{ + "SEMAPHORE_PROJECT_NAME": "cache-cli", + "SEMAPHORE_CACHE_BACKEND": "s3", + "SEMAPHORE_CACHE_S3_URL": os.Getenv("SEMAPHORE_CACHE_S3_URL"), + "SEMAPHORE_CACHE_S3_BUCKET": "semaphore-cache", + }, }, "sftp": { - "SEMAPHORE_CACHE_BACKEND": "sftp", - "SEMAPHORE_CACHE_URL": "sftp-server:22", - "SEMAPHORE_CACHE_USERNAME": "tester", - "SEMAPHORE_CACHE_PRIVATE_KEY_PATH": "/root/.ssh/semaphore_cache_key", + runInWindows: false, + envVars: map[string]string{ + "SEMAPHORE_CACHE_BACKEND": "sftp", + "SEMAPHORE_CACHE_URL": "sftp-server:22", + "SEMAPHORE_CACHE_USERNAME": "tester", + "SEMAPHORE_CACHE_PRIVATE_KEY_PATH": "/root/.ssh/semaphore_cache_key", + }, }, } func runTestForAllBackends(t *testing.T, test func(string, storage.Storage)) { - for backend, envVars := range testBackend { - for envVarName, envVarValue := range envVars { + for backendType, testBackend := range testBackends { + if runtime.GOOS == "windows" && !testBackend.runInWindows { + continue + } + + for envVarName, envVarValue := range testBackend.envVars { os.Setenv(envVarName, envVarValue) } storage, err := storage.InitStorage() assert.Nil(t, err) - - test(backend, storage) + test(backendType, storage) } } diff --git a/cache-cli/cmd/store.go b/cache-cli/cmd/store.go index f9ae626e..dfed4160 100644 --- a/cache-cli/cmd/store.go +++ b/cache-cli/cmd/store.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" "os" + "path/filepath" "strings" "time" @@ -52,7 +53,8 @@ func RunStore(cmd *cobra.Command, args []string) { } } } else { - compressAndStore(storage, args[0], args[1]) + path := filepath.FromSlash(args[1]) + compressAndStore(storage, args[0], path) } } diff --git a/cache-cli/cmd/store_test.go b/cache-cli/cmd/store_test.go index ef8a275f..55c2f672 100644 --- a/cache-cli/cmd/store_test.go +++ b/cache-cli/cmd/store_test.go @@ -29,12 +29,12 @@ func Test__Store(t *testing.T) { RunStore(storeCmd, []string{"abc001", "/tmp/this-path-does-not-exist"}) output := capturer.Done() - assert.Contains(t, output, "'/tmp/this-path-does-not-exist' doesn't exist locally.") + assert.Contains(t, output, fmt.Sprintf("'%s' doesn't exist locally.", filepath.FromSlash("/tmp/this-path-does-not-exist"))) }) t.Run(fmt.Sprintf("%s using key and valid path", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") ioutil.TempFile(tempDir, "*") capturer := utils.CreateOutputCapturer() @@ -47,7 +47,7 @@ func Test__Store(t *testing.T) { t.Run(fmt.Sprintf("%s normalizes key", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") ioutil.TempFile(tempDir, "*") capturer := utils.CreateOutputCapturer() @@ -61,7 +61,7 @@ func Test__Store(t *testing.T) { t.Run(fmt.Sprintf("%s using duplicate key", backend), func(*testing.T) { storage.Clear() - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") ioutil.TempFile(tempDir, "*") // Storing key for the first time @@ -103,7 +103,7 @@ func Test__AutomaticStore(t *testing.T) { RunStore(storeCmd, []string{}) output := capturer.Done() - assert.Contains(t, output, "'vendor/bundle' doesn't exist locally.") + assert.Contains(t, output, fmt.Sprintf("'%s' doesn't exist locally.", filepath.FromSlash("vendor/bundle"))) }) t.Run(fmt.Sprintf("%s detects and stores", backend), func(t *testing.T) { @@ -122,8 +122,8 @@ func Test__AutomaticStore(t *testing.T) { output := capturer.Done() assert.Contains(t, output, "Detected Gemfile.lock") - assert.Contains(t, output, "Compressing vendor/bundle") - assert.Contains(t, output, fmt.Sprintf("Uploading 'vendor/bundle' with cache key '%s'", key)) + assert.Contains(t, output, fmt.Sprintf("Compressing %s", filepath.FromSlash("vendor/bundle"))) + assert.Contains(t, output, fmt.Sprintf("Uploading '%s' with cache key '%s'", filepath.FromSlash("vendor/bundle"), key)) assert.Contains(t, output, "Upload complete") os.RemoveAll("vendor") @@ -138,7 +138,7 @@ func Test__AutomaticStore(t *testing.T) { checksum, _ := files.GenerateChecksum("Gemfile.lock") - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") key := fmt.Sprintf("gems-master-%s", checksum) err := storage.Store(key, tempFile.Name()) assert.Nil(t, err) diff --git a/cache-cli/docker-compose.yml b/cache-cli/docker-compose.yml index bb99fea5..b36a853f 100644 --- a/cache-cli/docker-compose.yml +++ b/cache-cli/docker-compose.yml @@ -13,6 +13,8 @@ services: volumes: - go-pkg-cache:/go - .:/app + environment: + SEMAPHORE_CACHE_S3_URL: "http://s3:9000" s3: image: quay.io/minio/minio:RELEASE.2021-09-15T04-54-25Z container_name: 's3' diff --git a/cache-cli/pkg/files/checksum_test.go b/cache-cli/pkg/files/checksum_test.go index 366f28b9..72db69d5 100644 --- a/cache-cli/pkg/files/checksum_test.go +++ b/cache-cli/pkg/files/checksum_test.go @@ -10,7 +10,7 @@ import ( func Test__GeneratesChecksum(t *testing.T) { t.Run("file is present", func(t *testing.T) { - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") tempFile.WriteString("hello, hello\n") checksum, err := GenerateChecksum(tempFile.Name()) diff --git a/cache-cli/pkg/files/compress.go b/cache-cli/pkg/files/compress.go index 87e255ab..903a9e09 100644 --- a/cache-cli/pkg/files/compress.go +++ b/cache-cli/pkg/files/compress.go @@ -2,6 +2,7 @@ package files import ( "fmt" + "os" "os/exec" "path/filepath" "time" @@ -9,11 +10,13 @@ import ( func Compress(key, path string) (string, error) { epochNanos := time.Now().Nanosecond() - tempFileName := fmt.Sprintf("/tmp/%s-%d", key, epochNanos) + tempFileName := filepath.Join(os.TempDir(), fmt.Sprintf("%s-%d", key, epochNanos)) cmd := compressionCommand(path, tempFileName) - _, err := cmd.Output() + output, err := cmd.CombinedOutput() if err != nil { + fmt.Printf("Error compressing %s: %s\n", path, output) + fmt.Printf("Error: %v\n", err) return tempFileName, err } diff --git a/cache-cli/pkg/files/compress_test.go b/cache-cli/pkg/files/compress_test.go index 509996cf..623c672b 100644 --- a/cache-cli/pkg/files/compress_test.go +++ b/cache-cli/pkg/files/compress_test.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "os" "path/filepath" - "strings" "testing" "github.com/semaphoreci/toolbox/cache-cli/pkg/metrics" @@ -28,9 +27,9 @@ func Test__CompressAndUnpack(t *testing.T) { }) t.Run("using absolute paths", func(t *testing.T) { - tempDir, _ := ioutil.TempDir("/tmp", "*") + tempDir, _ := ioutil.TempDir(os.TempDir(), "*") tempFile, _ := ioutil.TempFile(tempDir, "*") - assertCompressAndUnpack(t, metricsManager, tempDir, tempFile.Name()) + assertCompressAndUnpack(t, metricsManager, tempDir, tempFile) }) t.Run("using relative paths", func(t *testing.T) { @@ -38,17 +37,18 @@ func Test__CompressAndUnpack(t *testing.T) { tempDir, _ := ioutil.TempDir(cwd, "*") tempFile, _ := ioutil.TempFile(tempDir, "*") tempDirBase := filepath.Base(tempDir) - assertCompressAndUnpack(t, metricsManager, tempDirBase, tempFile.Name()) + assertCompressAndUnpack(t, metricsManager, tempDirBase, tempFile) }) t.Run("using single file", func(t *testing.T) { cwd, _ := os.Getwd() tempFile, _ := ioutil.TempFile(cwd, "*") + _ = tempFile.Close() // compressing compressedFileName, err := Compress("abc0003", tempFile.Name()) assert.Nil(t, err) - assert.True(t, strings.HasPrefix(compressedFileName, "/tmp/abc0003")) + assert.Contains(t, compressedFileName, filepath.FromSlash(fmt.Sprintf("%s/abc0003", os.TempDir()))) _, err = os.Stat(compressedFileName) assert.Nil(t, err) @@ -71,16 +71,19 @@ func Test__CompressAndUnpack(t *testing.T) { }) } -func assertCompressAndUnpack(t *testing.T, metricsManager metrics.MetricsManager, tempDirectory, tempFile string) { +func assertCompressAndUnpack(t *testing.T, metricsManager metrics.MetricsManager, tempDirectory string, tempFile *os.File) { // compressing compressedFileName, err := Compress("abc0003", tempDirectory) assert.Nil(t, err) - assert.True(t, strings.HasPrefix(compressedFileName, "/tmp/abc0003")) + assert.Contains(t, compressedFileName, filepath.FromSlash(fmt.Sprintf("%s/abc0003", os.TempDir()))) _, err = os.Stat(compressedFileName) assert.Nil(t, err) - err = os.Remove(tempFile) + // make sure file and directory are deleted + // before trying to unpack. + _ = tempFile.Close() + err = os.Remove(tempFile.Name()) assert.Nil(t, err) err = os.Remove(tempDirectory) assert.Nil(t, err) @@ -88,14 +91,14 @@ func assertCompressAndUnpack(t *testing.T, metricsManager metrics.MetricsManager // unpacking unpackedAt, err := Unpack(metricsManager, compressedFileName) assert.Nil(t, err) - assert.Equal(t, fmt.Sprintf("%s/", tempDirectory), unpackedAt) + assert.Equal(t, filepath.FromSlash(fmt.Sprintf("%s/", tempDirectory)), unpackedAt) files, _ := ioutil.ReadDir(unpackedAt) assert.Len(t, files, 1) file := files[0] - assert.Equal(t, filepath.Base(tempFile), file.Name()) + assert.Equal(t, filepath.Base(tempFile.Name()), file.Name()) - err = os.Remove(tempFile) + err = os.Remove(tempFile.Name()) assert.Nil(t, err) err = os.Remove(unpackedAt) assert.Nil(t, err) diff --git a/cache-cli/pkg/files/lookup.go b/cache-cli/pkg/files/lookup.go index d9531e92..6b5dfc27 100644 --- a/cache-cli/pkg/files/lookup.go +++ b/cache-cli/pkg/files/lookup.go @@ -133,14 +133,14 @@ func buildResult(filePath string, options LookupOptions, entries []buildResultRe for _, entry := range entries { if options.Restore { newEntries = append(newEntries, LookupResultEntry{ - Path: entry.Path, + Path: filepath.FromSlash(entry.Path), Keys: keysForRestore(entry.KeyPrefix, gitBranch, checksum), }) } else { key := fmt.Sprintf("%s-%s-%s", entry.KeyPrefix, gitBranch, checksum) newEntries = append(newEntries, LookupResultEntry{ Keys: []string{key}, - Path: entry.Path, + Path: filepath.FromSlash(entry.Path), }) } } diff --git a/cache-cli/pkg/files/lookup_test.go b/cache-cli/pkg/files/lookup_test.go index 8bf79199..ad1333b4 100644 --- a/cache-cli/pkg/files/lookup_test.go +++ b/cache-cli/pkg/files/lookup_test.go @@ -33,7 +33,10 @@ func Test__LookupStore(t *testing.T) { { DetectedFile: ".nvmrc", Entries: []LookupResultEntry{ - {Path: fmt.Sprintf("%s/.nvm", homedir), Keys: []string{fmt.Sprintf("nvm-master-%s", checksum)}}, + { + Path: filepath.FromSlash(fmt.Sprintf("%s/.nvm", homedir)), + Keys: []string{fmt.Sprintf("nvm-master-%s", checksum)}, + }, }, }, }) @@ -48,7 +51,10 @@ func Test__LookupStore(t *testing.T) { { DetectedFile: "Gemfile.lock", Entries: []LookupResultEntry{ - {Path: "vendor/bundle", Keys: []string{fmt.Sprintf("gems-master-%s", checksum)}}, + { + Path: filepath.FromSlash("vendor/bundle"), + Keys: []string{fmt.Sprintf("gems-master-%s", checksum)}, + }, }, }, }) @@ -123,7 +129,10 @@ func Test__LookupStore(t *testing.T) { { DetectedFile: "go.sum", Entries: []LookupResultEntry{ - {Path: fmt.Sprintf("%s/go/pkg/mod", homedir), Keys: []string{fmt.Sprintf("go-master-%s", checksum)}}, + { + Path: filepath.FromSlash(fmt.Sprintf("%s/go/pkg/mod", homedir)), + Keys: []string{fmt.Sprintf("go-master-%s", checksum)}, + }, }, }, }) @@ -138,8 +147,14 @@ func Test__LookupStore(t *testing.T) { { DetectedFile: "yarn.lock", Entries: []LookupResultEntry{ - {Path: fmt.Sprintf("%s/.cache/yarn", homedir), Keys: []string{fmt.Sprintf("yarn-cache-master-%s", checksum)}}, - {Path: "node_modules", Keys: []string{fmt.Sprintf("node-modules-master-%s", checksum)}}, + { + Path: filepath.FromSlash(fmt.Sprintf("%s/.cache/yarn", homedir)), + Keys: []string{fmt.Sprintf("yarn-cache-master-%s", checksum)}, + }, + { + Path: "node_modules", + Keys: []string{fmt.Sprintf("node-modules-master-%s", checksum)}, + }, }, }, }) @@ -225,11 +240,14 @@ func Test__LookupRestore(t *testing.T) { { DetectedFile: ".nvmrc", Entries: []LookupResultEntry{ - {Path: fmt.Sprintf("%s/.nvm", homedir), Keys: []string{ - fmt.Sprintf("nvm-some-branch-%s", checksum), - "nvm-some-branch", - "nvm-master", - }}, + { + Path: filepath.FromSlash(fmt.Sprintf("%s/.nvm", homedir)), + Keys: []string{ + fmt.Sprintf("nvm-some-branch-%s", checksum), + "nvm-some-branch", + "nvm-master", + }, + }, }, }, }) @@ -244,11 +262,14 @@ func Test__LookupRestore(t *testing.T) { { DetectedFile: "Gemfile.lock", Entries: []LookupResultEntry{ - {Path: "vendor/bundle", Keys: []string{ - fmt.Sprintf("gems-some-branch-%s", checksum), - "gems-some-branch", - "gems-master", - }}, + { + Path: filepath.FromSlash("vendor/bundle"), + Keys: []string{ + fmt.Sprintf("gems-some-branch-%s", checksum), + "gems-some-branch", + "gems-master", + }, + }, }, }, }) @@ -339,11 +360,14 @@ func Test__LookupRestore(t *testing.T) { { DetectedFile: "go.sum", Entries: []LookupResultEntry{ - {Path: fmt.Sprintf("%s/go/pkg/mod", homedir), Keys: []string{ - fmt.Sprintf("go-some-branch-%s", checksum), - "go-some-branch", - "go-master", - }}, + { + Path: filepath.FromSlash(fmt.Sprintf("%s/go/pkg/mod", homedir)), + Keys: []string{ + fmt.Sprintf("go-some-branch-%s", checksum), + "go-some-branch", + "go-master", + }, + }, }, }, }) @@ -358,16 +382,22 @@ func Test__LookupRestore(t *testing.T) { { DetectedFile: "yarn.lock", Entries: []LookupResultEntry{ - {Path: fmt.Sprintf("%s/.cache/yarn", homedir), Keys: []string{ - fmt.Sprintf("yarn-cache-some-branch-%s", checksum), - "yarn-cache-some-branch", - "yarn-cache-master", - }}, - {Path: "node_modules", Keys: []string{ - fmt.Sprintf("node-modules-some-branch-%s", checksum), - "node-modules-some-branch", - "node-modules-master", - }}, + { + Path: filepath.FromSlash(fmt.Sprintf("%s/.cache/yarn", homedir)), + Keys: []string{ + fmt.Sprintf("yarn-cache-some-branch-%s", checksum), + "yarn-cache-some-branch", + "yarn-cache-master", + }, + }, + { + Path: "node_modules", + Keys: []string{ + fmt.Sprintf("node-modules-some-branch-%s", checksum), + "node-modules-some-branch", + "node-modules-master", + }, + }, }, }, }) diff --git a/cache-cli/pkg/files/unpack.go b/cache-cli/pkg/files/unpack.go index bdab8017..09f8cf2d 100644 --- a/cache-cli/pkg/files/unpack.go +++ b/cache-cli/pkg/files/unpack.go @@ -24,7 +24,7 @@ func Unpack(metricsManager metrics.MetricsManager, path string) (string, error) } cmd := unpackCommand(restorationPath, path) - output, err := cmd.Output() + output, err := cmd.CombinedOutput() if err != nil { fmt.Printf("Unpacking command failed: %s\n", string(output)) if metricErr := metricsManager.Publish(metrics.Metric{Name: metrics.CacheCorruptionRate, Value: "1"}); metricErr != nil { @@ -76,5 +76,5 @@ func findRestorationPath(path string) (string, error) { return "", err } - return header.Name, gzipReader.Close() + return filepath.FromSlash(header.Name), gzipReader.Close() } diff --git a/cache-cli/pkg/files/unpack_test.go b/cache-cli/pkg/files/unpack_test.go index 1019f20d..ed67f46d 100644 --- a/cache-cli/pkg/files/unpack_test.go +++ b/cache-cli/pkg/files/unpack_test.go @@ -15,16 +15,16 @@ func Test__UnpackSendsMetricsOnFailure(t *testing.T) { metricsManager, err := metrics.InitMetricsManager(metrics.LocalBackend) assert.Nil(t, err) - tempFile, _ := ioutil.TempFile("/tmp", "*") + tempFile, _ := ioutil.TempFile(os.TempDir(), "*") tempFile.WriteString("this is not a proper archive") _, err = Unpack(metricsManager, tempFile.Name()) assert.NotNil(t, err) - bytes, err := ioutil.ReadFile("/tmp/toolbox_metrics") + bytes, err := ioutil.ReadFile(fmt.Sprintf("%s/toolbox_metrics", os.TempDir())) assert.Nil(t, err) assert.Contains(t, string(bytes), fmt.Sprintf("%s 1", metrics.CacheCorruptionRate)) os.Remove(tempFile.Name()) - os.Remove("/tmp/toolbox_metrics") + os.Remove(fmt.Sprintf("%s/toolbox_metrics", os.TempDir())) } diff --git a/cache-cli/pkg/metrics/local.go b/cache-cli/pkg/metrics/local.go index 397cc0cc..29b42068 100644 --- a/cache-cli/pkg/metrics/local.go +++ b/cache-cli/pkg/metrics/local.go @@ -3,6 +3,7 @@ package metrics import ( "fmt" "os" + "runtime" ) type LocalMetricsManager struct { @@ -11,9 +12,14 @@ type LocalMetricsManager struct { } func NewLocalMetricsBackend() (*LocalMetricsManager, error) { + basePath := "/tmp" + if runtime.GOOS == "windows" { + basePath = os.TempDir() + } + return &LocalMetricsManager{ - ToolboxMetricsPath: "/tmp/toolbox_metrics", - CacheMetricsPath: "/tmp/cache_metrics", + ToolboxMetricsPath: fmt.Sprintf("%s/toolbox_metrics", basePath), + CacheMetricsPath: fmt.Sprintf("%s/cache_metrics", basePath), }, nil } diff --git a/cache-cli/pkg/storage/clear_test.go b/cache-cli/pkg/storage/clear_test.go index fe7379b2..889bf7d9 100644 --- a/cache-cli/pkg/storage/clear_test.go +++ b/cache-cli/pkg/storage/clear_test.go @@ -14,10 +14,10 @@ func Test__Clear(t *testing.T) { setup := func(storage Storage) []string { _ = storage.Clear() - file1, _ := ioutil.TempFile("/tmp", "*") + file1, _ := ioutil.TempFile(os.TempDir(), "*") file1.WriteString("something, something") - file2, _ := ioutil.TempFile("/tmp", "*") + file2, _ := ioutil.TempFile(os.TempDir(), "*") file2.WriteString("else, else") _ = storage.Store("abc001", file1.Name()) diff --git a/cache-cli/pkg/storage/delete_test.go b/cache-cli/pkg/storage/delete_test.go index 17f59372..a116f572 100644 --- a/cache-cli/pkg/storage/delete_test.go +++ b/cache-cli/pkg/storage/delete_test.go @@ -20,7 +20,7 @@ func Test__Delete(t *testing.T) { t.Run(fmt.Sprintf("%s existing key", storageType), func(t *testing.T) { _ = storage.Clear() - file, _ := ioutil.TempFile("/tmp", "*") + file, _ := ioutil.TempFile(os.TempDir(), "*") _ = storage.Store("abc001", file.Name()) keys, err := storage.List() diff --git a/cache-cli/pkg/storage/has_key_test.go b/cache-cli/pkg/storage/has_key_test.go index 14073d9b..91f84ec4 100644 --- a/cache-cli/pkg/storage/has_key_test.go +++ b/cache-cli/pkg/storage/has_key_test.go @@ -21,7 +21,7 @@ func Test__HasKey(t *testing.T) { t.Run(fmt.Sprintf("%s existing key", storageType), func(t *testing.T) { _ = storage.Clear() - file, _ := ioutil.TempFile("/tmp", "*") + file, _ := ioutil.TempFile(os.TempDir(), "*") _ = storage.Store("abc001", file.Name()) exists, err := storage.HasKey("abc001") diff --git a/cache-cli/pkg/storage/is_not_empty_test.go b/cache-cli/pkg/storage/is_not_empty_test.go index 1d55d485..7c0a1028 100644 --- a/cache-cli/pkg/storage/is_not_empty_test.go +++ b/cache-cli/pkg/storage/is_not_empty_test.go @@ -21,7 +21,7 @@ func Test__IsNotEmpty(t *testing.T) { t.Run(fmt.Sprintf("%s non-empty cache", storageType), func(t *testing.T) { _ = storage.Clear() - file, _ := ioutil.TempFile("/tmp", "*") + file, _ := ioutil.TempFile(os.TempDir(), "*") _ = storage.Store("abc001", file.Name()) isNotEmpty, err := storage.IsNotEmpty() diff --git a/cache-cli/pkg/storage/list_test.go b/cache-cli/pkg/storage/list_test.go index 2e36d974..9c9993c0 100644 --- a/cache-cli/pkg/storage/list_test.go +++ b/cache-cli/pkg/storage/list_test.go @@ -23,13 +23,13 @@ func Test__List(t *testing.T) { err := storage.Clear() assert.Nil(t, err) - file1, _ := ioutil.TempFile("/tmp", "*") + file1, _ := ioutil.TempFile(os.TempDir(), "*") err = storage.Store("abc001", file1.Name()) assert.Nil(t, err) time.Sleep(time.Second) - file2, _ := ioutil.TempFile("/tmp", "*") + file2, _ := ioutil.TempFile(os.TempDir(), "*") err = storage.Store("abc002", file2.Name()) assert.Nil(t, err) diff --git a/cache-cli/pkg/storage/restore_test.go b/cache-cli/pkg/storage/restore_test.go index 4348d321..8641775a 100644 --- a/cache-cli/pkg/storage/restore_test.go +++ b/cache-cli/pkg/storage/restore_test.go @@ -14,7 +14,7 @@ func Test__Restore(t *testing.T) { t.Run(fmt.Sprintf("%s key exists", storageType), func(t *testing.T) { _ = storage.Clear() - file, _ := ioutil.TempFile("/tmp", "*") + file, _ := ioutil.TempFile(os.TempDir(), "*") file.WriteString("restore - key exists") err := storage.Store("abc001", file.Name()) diff --git a/cache-cli/pkg/storage/s3_restore.go b/cache-cli/pkg/storage/s3_restore.go index 52be4d14..37938bf4 100644 --- a/cache-cli/pkg/storage/s3_restore.go +++ b/cache-cli/pkg/storage/s3_restore.go @@ -11,7 +11,7 @@ import ( ) func (s *S3Storage) Restore(key string) (*os.File, error) { - tempFile, err := ioutil.TempFile("/tmp", fmt.Sprintf("%s-*", key)) + tempFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) if err != nil { return nil, err } diff --git a/cache-cli/pkg/storage/s3_store.go b/cache-cli/pkg/storage/s3_store.go index 096d3e4d..e40530a8 100644 --- a/cache-cli/pkg/storage/s3_store.go +++ b/cache-cli/pkg/storage/s3_store.go @@ -24,5 +24,10 @@ func (s *S3Storage) Store(key, path string) error { Body: file, }) - return err + if err != nil { + fmt.Printf("Error uploading: %v\n", err) + return err + } + + return file.Close() } diff --git a/cache-cli/pkg/storage/sftp_restore.go b/cache-cli/pkg/storage/sftp_restore.go index 7040087d..d037b805 100644 --- a/cache-cli/pkg/storage/sftp_restore.go +++ b/cache-cli/pkg/storage/sftp_restore.go @@ -7,7 +7,7 @@ import ( ) func (s *SFTPStorage) Restore(key string) (*os.File, error) { - localFile, err := ioutil.TempFile("/tmp", fmt.Sprintf("%s-*", key)) + localFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-*", key)) if err != nil { return nil, err } diff --git a/cache-cli/pkg/storage/storage_test.go b/cache-cli/pkg/storage/storage_test.go index 722778ea..cffb7c36 100644 --- a/cache-cli/pkg/storage/storage_test.go +++ b/cache-cli/pkg/storage/storage_test.go @@ -2,33 +2,50 @@ package storage import ( "math" + "os" + "runtime" "testing" assert "github.com/stretchr/testify/assert" ) -var storageTypes = map[string]func(int64) (Storage, error){ - "s3": func(storageSize int64) (Storage, error) { - return NewS3Storage(S3StorageOptions{ - URL: "http://s3:9000", - Bucket: "semaphore-cache", - Project: "cache-cli", - Config: StorageConfig{MaxSpace: math.MaxInt64}, - }) +type TestStorageType struct { + runInWindows bool + initializer func(storageSize int64) (Storage, error) +} + +var testStorageTypes = map[string]TestStorageType{ + "s3": { + runInWindows: true, + initializer: func(storageSize int64) (Storage, error) { + return NewS3Storage(S3StorageOptions{ + URL: os.Getenv("SEMAPHORE_CACHE_S3_URL"), + Bucket: "semaphore-cache", + Project: "cache-cli", + Config: StorageConfig{MaxSpace: math.MaxInt64}, + }) + }, }, - "sftp": func(storageSize int64) (Storage, error) { - return NewSFTPStorage(SFTPStorageOptions{ - URL: "sftp-server:22", - Username: "tester", - PrivateKeyPath: "/root/.ssh/semaphore_cache_key", - Config: StorageConfig{MaxSpace: storageSize}, - }) + "sftp": { + runInWindows: false, + initializer: func(storageSize int64) (Storage, error) { + return NewSFTPStorage(SFTPStorageOptions{ + URL: "sftp-server:22", + Username: "tester", + PrivateKeyPath: "/root/.ssh/semaphore_cache_key", + Config: StorageConfig{MaxSpace: storageSize}, + }) + }, }, } func runTestForAllStorageTypes(t *testing.T, test func(string, Storage)) { - for storageType, storageProvider := range storageTypes { - storage, err := storageProvider(9 * 1024 * 1024 * 1024) + for storageType, testStorage := range testStorageTypes { + if runtime.GOOS == "windows" && !testStorage.runInWindows { + continue + } + + storage, err := testStorage.initializer(9 * 1024 * 1024 * 1024) if assert.Nil(t, err) { test(storageType, storage) } @@ -36,8 +53,8 @@ func runTestForAllStorageTypes(t *testing.T, test func(string, Storage)) { } func runTestForSingleStorageType(storageType string, storageSize int64, t *testing.T, test func(Storage)) { - storageProvider := storageTypes[storageType] - storage, err := storageProvider(storageSize) + storageProvider := testStorageTypes[storageType] + storage, err := storageProvider.initializer(storageSize) if assert.Nil(t, err) { test(storage) } diff --git a/cache-cli/pkg/storage/store_test.go b/cache-cli/pkg/storage/store_test.go index d2934fa3..9529f3ff 100644 --- a/cache-cli/pkg/storage/store_test.go +++ b/cache-cli/pkg/storage/store_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strconv" "strings" "testing" @@ -18,7 +19,7 @@ func Test__Store(t *testing.T) { t.Run(fmt.Sprintf("%s stored objects can be listed", storageType), func(t *testing.T) { _ = storage.Clear() - file, _ := ioutil.TempFile("/tmp", "*") + file, _ := ioutil.TempFile(os.TempDir(), "*") err := storage.Store("abc001", file.Name()) assert.Nil(t, err) @@ -38,7 +39,7 @@ func Test__Store(t *testing.T) { t.Run(fmt.Sprintf("%s stored objects can be restored", storageType), func(t *testing.T) { _ = storage.Clear() - file, _ := ioutil.TempFile("/tmp", "*") + file, _ := ioutil.TempFile(os.TempDir(), "*") file.WriteString("stored objects can be restored") err := storage.Store("abc002", file.Name()) @@ -65,14 +66,18 @@ func Test__Store(t *testing.T) { * have any lines from the smaller file. */ t.Run(fmt.Sprintf("%s concurrent writes keep the file that finished writing last", storageType), func(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + _ = storage.Clear() - smallerFile := "/tmp/smaller.tmp" + smallerFile := fmt.Sprintf("%s/smaller.tmp", os.TempDir()) err := createBigTempFile(smallerFile, 300*1000*1000) // 300M assert.Nil(t, err) // this one is bigger so it will take longer to finish - biggerFile := "/tmp/bigger.tmp" + biggerFile := fmt.Sprintf("%s/bigger.tmp", os.TempDir()) err = createBigTempFile(biggerFile, 600*1000*1000) // 600M assert.Nil(t, err) @@ -92,39 +97,41 @@ func Test__Store(t *testing.T) { }) }) - runTestForSingleStorageType("sftp", 1024, t, func(storage Storage) { - t.Run("sftp storage deletes old keys if no space left to store", func(t *testing.T) { - _ = storage.Clear() + if runtime.GOOS != "windows" { + runTestForSingleStorageType("sftp", 1024, t, func(storage Storage) { + t.Run("sftp storage deletes old keys if no space left to store", func(t *testing.T) { + _ = storage.Clear() - file1, _ := ioutil.TempFile("/tmp", "*") - file1.WriteString(strings.Repeat("x", 400)) - storage.Store("abc001", file1.Name()) + file1, _ := ioutil.TempFile(os.TempDir(), "*") + file1.WriteString(strings.Repeat("x", 400)) + storage.Store("abc001", file1.Name()) - time.Sleep(time.Second) + time.Sleep(time.Second) - file2, _ := ioutil.TempFile("/tmp", "*") - file2.WriteString(strings.Repeat("x", 400)) - storage.Store("abc002", file2.Name()) + file2, _ := ioutil.TempFile(os.TempDir(), "*") + file2.WriteString(strings.Repeat("x", 400)) + storage.Store("abc002", file2.Name()) - time.Sleep(time.Second) + time.Sleep(time.Second) - file3, _ := ioutil.TempFile("/tmp", "*") - file3.WriteString(strings.Repeat("x", 400)) - storage.Store("abc003", file3.Name()) + file3, _ := ioutil.TempFile(os.TempDir(), "*") + file3.WriteString(strings.Repeat("x", 400)) + storage.Store("abc003", file3.Name()) - keys, _ := storage.List() - assert.Len(t, keys, 2) + keys, _ := storage.List() + assert.Len(t, keys, 2) - firstKey := keys[0] - assert.Equal(t, "abc003", firstKey.Name) - secondKey := keys[1] - assert.Equal(t, "abc002", secondKey.Name) + firstKey := keys[0] + assert.Equal(t, "abc003", firstKey.Name) + secondKey := keys[1] + assert.Equal(t, "abc002", secondKey.Name) - os.Remove(file1.Name()) - os.Remove(file2.Name()) - os.Remove(file3.Name()) + os.Remove(file1.Name()) + os.Remove(file2.Name()) + os.Remove(file3.Name()) + }) }) - }) + } } func createBigTempFile(fileName string, size int64) error { diff --git a/cache-cli/pkg/storage/usage_test.go b/cache-cli/pkg/storage/usage_test.go index b011ea32..f2112cc3 100644 --- a/cache-cli/pkg/storage/usage_test.go +++ b/cache-cli/pkg/storage/usage_test.go @@ -29,7 +29,7 @@ func Test__Usage(t *testing.T) { _ = storage.Clear() fileContents := "usage - some usage" - file, _ := ioutil.TempFile("/tmp", "*") + file, _ := ioutil.TempFile(os.TempDir(), "*") file.WriteString(fileContents) _ = storage.Store("abc001", file.Name()) diff --git a/install-self-hosted-toolbox.ps1 b/install-self-hosted-toolbox.ps1 index 9dc853ef..825bf462 100644 --- a/install-self-hosted-toolbox.ps1 +++ b/install-self-hosted-toolbox.ps1 @@ -1,43 +1,66 @@ -$ErrorActionPreference = "Stop" +function Install-PSModule { + param ( + [string] $ModuleLocation, + [string] $ModuleName + ) -# PowerShell Core 6.0+ has $isWindows set, but older versions don't. -# For those versions, we use the $env:OS variable, which is only set in Windows. -if ($IsWindows -or $env:OS) { - $ModulePath = $Env:PSModulePath.Split(";")[0] -} else { - $ModulePath = $Env:PSModulePath.Split(":")[0] -} + Write-Output "Installing $ModuleName module in $ModuleLocation..." + if (-not (Test-Path $ModuleLocation)) { + Write-Output "No $ModuleLocation directory found. Creating it..." + New-Item -ItemType Directory -Path $ModuleLocation > $null + if (-not (Test-Path $ModuleLocation)) { + Write-Output "Error creating $ModuleLocation" + Exit 1 + } + } + + $ModulePath = Join-Path -Path $ModuleLocation -ChildPath $ModuleName + if (Test-Path $ModulePath) { + Write-Output "$ModuleName module directory already exists. Overriding it..." + Remove-Item -Path $ModulePath -Force -Recurse + } -Write-Output "Installing Checkout module in $ModulePath..." -if (-not (Test-Path $ModulePath)) { - Write-Output "No $ModulePath directory found. Creating it..." + Write-Output "Creating $ModuleName module directory at $ModulePath..." New-Item -ItemType Directory -Path $ModulePath > $null if (-not (Test-Path $ModulePath)) { Write-Output "Error creating $ModulePath" Exit 1 } -} -$CheckoutModulePath = Join-Path -Path $ModulePath -ChildPath "Checkout" -if (Test-Path $CheckoutModulePath) { - Write-Output "Checkout module directory already exists. Overriding it..." - Remove-Item -Path $CheckoutModulePath -Force -Recurse + Write-Output "Copying .psm1 file to $ModuleName module directory..." + + # The .psm1 file name needs to match the module directory name, otherwise powershell will ignore it + Copy-Item "$HOME\.toolbox\$ModuleName.psm1" -Destination "$ModulePath\$ModuleName.psm1" + if (-not $?) { + Write-Output "Error copying .psm1 module to $ModulePath" + Exit 1 + } } -Write-Output "Creating Checkout module directory at $CheckoutModulePath..." -New-Item -ItemType Directory -Path $CheckoutModulePath > $null -if (-not (Test-Path $CheckoutModulePath)) { - Write-Output "Error creating $CheckoutModulePath" - Exit 1 +# To make the binaries available, +# we include the $HOME\.toolbox\bin directory in the user's PATH. +function Install-BinaryFolder { + $toolboxPath = Join-Path $HOME ".toolbox" | Join-Path -ChildPath "bin" + Write-Output "Adding $toolboxPath to the PATH..." + + $currentPaths = [Environment]::GetEnvironmentVariable('PATH', 'User') -split ';' + $updatePaths = @($currentPaths | Where-Object { $_ -ne $toolboxPath }) + $updatePaths += $toolboxPath + + [Environment]::SetEnvironmentVariable('PATH', ($updatePaths -join ';'), 'User') } -Write-Output "Copying .psm1 file to checkout module directory..." +$ErrorActionPreference = "Stop" -# The .psm1 file name needs to match the module directory name, otherwise powershell will ignore it -Copy-Item $HOME\.toolbox\Checkout.psm1 -Destination "$CheckoutModulePath\Checkout.psm1" -if (-not $?) { - Write-Output "Error copying .psm1 module to $CheckoutModulePath" - Exit 1 +# PowerShell Core 6.0+ has $isWindows set, but older versions don't. +# For those versions, we use the $env:OS variable, which is only set in Windows. +if ($IsWindows -or $env:OS) { + $ModulePath = $Env:PSModulePath.Split(";")[0] +} else { + $ModulePath = $Env:PSModulePath.Split(":")[0] } -Write-Output "Installation completed successfully." \ No newline at end of file +Install-PSModule -ModuleLocation $ModulePath -ModuleName 'Checkout' +Install-BinaryFolder + +Write-Output "Installation completed successfully." diff --git a/release/create.sh b/release/create.sh index 69673e19..12ad9f83 100755 --- a/release/create.sh +++ b/release/create.sh @@ -56,7 +56,20 @@ hosted::create_initial_content() { cp -R ~/$SEMAPHORE_GIT_DIR/* /tmp/Linux/toolbox cp -R ~/$SEMAPHORE_GIT_DIR/* /tmp/Darwin/toolbox - exclusions=(.git .gitignore Makefile release tests cache-cli install-self-hosted-toolbox self-hosted-toolbox) + exclusions=( + .git + .gitignore + Makefile + release + scripts + tests + docker-compose.yml + cache-cli + install-self-hosted-toolbox + install-self-hosted-toolbox.ps1 + Checkout.psm1 + self-hosted-toolbox + ) for exclusion in "${exclusions[@]}"; do rm -rf /tmp/Linux/toolbox/${exclusion} rm -rf /tmp/Darwin/toolbox/${exclusion} @@ -87,8 +100,15 @@ self_hosted::create_initial_content() { cp ~/$SEMAPHORE_GIT_DIR/${inclusion} /tmp/self-hosted-Darwin/toolbox/ done - # Windows inclusions + # Windows PowerShell module inclusions cp ~/$SEMAPHORE_GIT_DIR/Checkout.psm1 /tmp/self-hosted-Windows/toolbox/ + + # For the windows toolbox, we put all the binaries in a bin folder. + # The reason for that is in Windows we don't have a location like /usr/local/bin + # to use to place all the binaries in. Instead, we add the '$HOME/.toolbox/bin' + # folder to the user's PATH. + mkdir -p /tmp/self-hosted-Windows/toolbox/bin + cp ~/$SEMAPHORE_GIT_DIR/cache-cli/bin/windows/cache.exe /tmp/self-hosted-Windows/toolbox/bin/cache.exe } self_hosted::pack() {