Skip to content

Commit

Permalink
New cache CLI: publish metrics (#280)
Browse files Browse the repository at this point in the history
  • Loading branch information
lucaspin authored Oct 25, 2021
1 parent d5e706f commit 6c85b98
Show file tree
Hide file tree
Showing 12 changed files with 420 additions and 29 deletions.
2 changes: 1 addition & 1 deletion .semaphore/semaphore.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ blocks:
- 'shellcheck sem-service -f gcc | wc -l && [[ "$(shellcheck sem-service -f gcc | wc -l)" -le 76 ]]'
- 'shellcheck sem-version -f gcc | wc -l && [[ "$(shellcheck sem-version -f gcc | wc -l)" -le 21 ]]'
- 'shellcheck cache -f gcc | wc -l && [[ "$(shellcheck cache -f gcc | wc -l)" -le 152 ]]'
- 'shellcheck libcheckout -f gcc | wc -l && [[ "$(shellcheck libcheckout -f gcc | wc -l)" -le 89 ]]'
- 'shellcheck libcheckout -f gcc | wc -l && [[ "$(shellcheck libcheckout -f gcc | wc -l)" -le 85 ]]'
- shellcheck install-package

- name: Sem Version Tests bionic
Expand Down
56 changes: 49 additions & 7 deletions cache-cli/cmd/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,13 @@ package cmd

import (
"fmt"
"io/fs"
"os"
"strings"
"time"

"github.com/semaphoreci/toolbox/cache-cli/pkg/files"
"github.com/semaphoreci/toolbox/cache-cli/pkg/metrics"
"github.com/semaphoreci/toolbox/cache-cli/pkg/storage"
"github.com/semaphoreci/toolbox/cache-cli/pkg/utils"
"github.com/spf13/cobra"
Expand All @@ -32,6 +34,9 @@ func RunRestore(cmd *cobra.Command, args []string) {
storage, err := storage.InitStorage()
utils.Check(err)

metricsManager, err := metrics.InitMetricsManager(metrics.LocalBackend)
utils.Check(err)

if len(args) == 0 {
lookupResults := files.Lookup(files.LookupOptions{
GitBranch: os.Getenv("SEMAPHORE_GIT_BRANCH"),
Expand All @@ -47,21 +52,21 @@ func RunRestore(cmd *cobra.Command, args []string) {
fmt.Printf("Detected %s.\n", lookupResult.DetectedFile)
for _, entry := range lookupResult.Entries {
fmt.Printf("Fetching '%s' directory with cache keys '%s'...\n", entry.Path, strings.Join(entry.Keys, ","))
downloadAndUnpack(storage, entry.Keys)
downloadAndUnpack(storage, metricsManager, entry.Keys)
}
}
} else {
keys := strings.Split(args[0], ",")
downloadAndUnpack(storage, keys)
downloadAndUnpack(storage, metricsManager, keys)
}
}

func downloadAndUnpack(storage storage.Storage, keys []string) {
func downloadAndUnpack(storage storage.Storage, metricsManager metrics.MetricsManager, keys []string) {
for _, rawKey := range keys {
key := NormalizeKey(rawKey)
if ok, _ := storage.HasKey(key); ok {
fmt.Printf("HIT: '%s', using key '%s'.\n", key, key)
downloadAndUnpackKey(storage, key)
downloadAndUnpackKey(storage, metricsManager, key)
break
}

Expand All @@ -71,7 +76,7 @@ func downloadAndUnpack(storage storage.Storage, keys []string) {
matchingKey := findMatchingKey(availableKeys, key)
if matchingKey != "" {
fmt.Printf("HIT: '%s', using key '%s'.\n", key, matchingKey)
downloadAndUnpackKey(storage, matchingKey)
downloadAndUnpackKey(storage, metricsManager, matchingKey)
break
} else {
fmt.Printf("MISS: '%s'.\n", key)
Expand All @@ -89,19 +94,21 @@ func findMatchingKey(availableKeys []storage.CacheKey, match string) string {
return ""
}

func downloadAndUnpackKey(storage storage.Storage, key string) {
func downloadAndUnpackKey(storage storage.Storage, metricsManager metrics.MetricsManager, key string) {
downloadStart := time.Now()
fmt.Printf("Downloading key '%s'...\n", key)
compressed, err := storage.Restore(key)
utils.Check(err)

downloadDuration := time.Since(downloadStart)
info, _ := os.Stat(compressed.Name())

fmt.Printf("Download complete. Duration: %v. Size: %v bytes.\n", downloadDuration.String(), files.HumanReadableSize(info.Size()))
publishMetrics(metricsManager, info, downloadDuration)

unpackStart := time.Now()
fmt.Printf("Unpacking '%s'...\n", compressed.Name())
restorationPath, err := files.Unpack(compressed.Name())
restorationPath, err := files.Unpack(metricsManager, compressed.Name())
utils.Check(err)

unpackDuration := time.Since(unpackStart)
Expand All @@ -110,6 +117,41 @@ func downloadAndUnpackKey(storage storage.Storage, key string) {
os.Remove(compressed.Name())
}

func publishMetrics(metricsManager metrics.MetricsManager, fileInfo fs.FileInfo, downloadDuration time.Duration) error {
metricsToPublish := []metrics.Metric{
{Name: metrics.CacheDownloadSize, Value: fmt.Sprintf("%d", fileInfo.Size())},
{Name: metrics.CacheDownloadTime, Value: downloadDuration.String()},
}

username := os.Getenv("SEMAPHORE_CACHE_USERNAME")
if username != "" {
metricsToPublish = append(metricsToPublish, metrics.Metric{Name: metrics.CacheUser, Value: username})
}

cacheServerIP := getCacheServerIP()
if cacheServerIP != "" {
metricsToPublish = append(metricsToPublish, metrics.Metric{Name: metrics.CacheServer, Value: cacheServerIP})
}

metricsToPublish = append(metricsToPublish, metrics.Metric{Name: metrics.CacheTotalRate, Value: "1"})

return metricsManager.PublishBatch(metricsToPublish)
}

func getCacheServerIP() string {
cacheURL := os.Getenv("SEMAPHORE_CACHE_URL")
if cacheURL != "" {
ipAndPort := strings.Split(cacheURL, ":")
if len(ipAndPort) != 2 {
return ""
}

return ipAndPort[0]
}

return ""
}

func init() {
RootCmd.AddCommand(restoreCmd)
}
16 changes: 10 additions & 6 deletions cache-cli/pkg/files/compress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,37 @@ import (
"strings"
"testing"

"github.com/semaphoreci/toolbox/cache-cli/pkg/metrics"
assert "github.com/stretchr/testify/assert"
)

func Test__CompressAndUnpack(t *testing.T) {
metricsManager, err := metrics.InitMetricsManager(metrics.LocalBackend)
assert.Nil(t, err)

t.Run("file to compress is not present", func(t *testing.T) {
compressedFileName, err := Compress("abc0001", "/tmp/this-file-does-not-exist")
assert.NotNil(t, err)
os.Remove(compressedFileName)
})

t.Run("file to unpack is not present", func(t *testing.T) {
_, err := Unpack("/tmp/this-file-does-not-exist")
_, err := Unpack(metricsManager, "/tmp/this-file-does-not-exist")
assert.NotNil(t, err)
})

t.Run("using absolute paths", func(t *testing.T) {
tempDir, _ := ioutil.TempDir("/tmp", "*")
tempFile, _ := ioutil.TempFile(tempDir, "*")
assertCompressAndUnpack(t, tempDir, tempFile.Name())
assertCompressAndUnpack(t, metricsManager, tempDir, tempFile.Name())
})

t.Run("using relative paths", func(t *testing.T) {
cwd, _ := os.Getwd()
tempDir, _ := ioutil.TempDir(cwd, "*")
tempFile, _ := ioutil.TempFile(tempDir, "*")
tempDirBase := filepath.Base(tempDir)
assertCompressAndUnpack(t, tempDirBase, tempFile.Name())
assertCompressAndUnpack(t, metricsManager, tempDirBase, tempFile.Name())
})

t.Run("using single file", func(t *testing.T) {
Expand All @@ -53,7 +57,7 @@ func Test__CompressAndUnpack(t *testing.T) {
assert.Nil(t, err)

// unpacking
unpackedAt, err := Unpack(compressedFileName)
unpackedAt, err := Unpack(metricsManager, compressedFileName)
assert.Nil(t, err)
assert.Equal(t, tempFile.Name(), unpackedAt)

Expand All @@ -67,7 +71,7 @@ func Test__CompressAndUnpack(t *testing.T) {
})
}

func assertCompressAndUnpack(t *testing.T, tempDirectory, tempFile string) {
func assertCompressAndUnpack(t *testing.T, metricsManager metrics.MetricsManager, tempDirectory, tempFile string) {
// compressing
compressedFileName, err := Compress("abc0003", tempDirectory)
assert.Nil(t, err)
Expand All @@ -82,7 +86,7 @@ func assertCompressAndUnpack(t *testing.T, tempDirectory, tempFile string) {
assert.Nil(t, err)

// unpacking
unpackedAt, err := Unpack(compressedFileName)
unpackedAt, err := Unpack(metricsManager, compressedFileName)
assert.Nil(t, err)
assert.Equal(t, fmt.Sprintf("%s/", tempDirectory), unpackedAt)

Expand Down
18 changes: 9 additions & 9 deletions cache-cli/pkg/files/unpack.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,33 +8,33 @@ import (
"os"
"os/exec"
"path/filepath"

"github.com/semaphoreci/toolbox/cache-cli/pkg/metrics"
)

func Unpack(path string) (string, error) {
func Unpack(metricsManager metrics.MetricsManager, path string) (string, error) {
restorationPath, err := findRestorationPath(path)
if err != nil {
metricsManager.Publish(metrics.Metric{Name: metrics.CacheCorruptionRate, Value: "1"})
return "", err
}

cmd, err := unpackCommand(restorationPath, path)
if err != nil {
return "", err
}

cmd := unpackCommand(restorationPath, path)
_, err = cmd.Output()
if err != nil {
metricsManager.Publish(metrics.Metric{Name: metrics.CacheCorruptionRate, Value: "1"})
return "", err
}

return restorationPath, nil
}

func unpackCommand(restorationPath, tempFile string) (*exec.Cmd, error) {
func unpackCommand(restorationPath, tempFile string) *exec.Cmd {
if filepath.IsAbs(restorationPath) {
return exec.Command("tar", "xzPf", tempFile, "-C", "."), nil
return exec.Command("tar", "xzPf", tempFile, "-C", ".")
}

return exec.Command("tar", "xzf", tempFile, "-C", "."), nil
return exec.Command("tar", "xzf", tempFile, "-C", ".")
}

func findRestorationPath(path string) (string, error) {
Expand Down
30 changes: 30 additions & 0 deletions cache-cli/pkg/files/unpack_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package files

import (
"fmt"
"io/ioutil"
"os"
"testing"

"github.com/semaphoreci/toolbox/cache-cli/pkg/metrics"
assert "github.com/stretchr/testify/assert"
)

func Test__UnpackSendsMetricsOnFailure(t *testing.T) {
os.Setenv("SEMAPHORE_TOOLBOX_METRICS_ENABLED", "true")
metricsManager, err := metrics.InitMetricsManager(metrics.LocalBackend)
assert.Nil(t, err)

tempFile, _ := ioutil.TempFile("/tmp", "*")
tempFile.WriteString("this is not a proper archive")

_, err = Unpack(metricsManager, tempFile.Name())
assert.NotNil(t, err)

bytes, err := ioutil.ReadFile("/tmp/toolbox_metrics")
assert.Nil(t, err)
assert.Contains(t, string(bytes), fmt.Sprintf("%s 1", metrics.CacheCorruptionRate))

os.Remove(tempFile.Name())
os.Remove("/tmp/toolbox_metrics")
}
67 changes: 67 additions & 0 deletions cache-cli/pkg/metrics/local.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package metrics

import (
"fmt"
"os"
)

type LocalMetricsManager struct {
ToolboxMetricsPath string
CacheMetricsPath string
}

func NewLocalMetricsBackend() (*LocalMetricsManager, error) {
return &LocalMetricsManager{
ToolboxMetricsPath: "/tmp/toolbox_metrics",
CacheMetricsPath: "/tmp/cache_metrics",
}, nil
}

func (b *LocalMetricsManager) Enabled() bool {
return os.Getenv("SEMAPHORE_TOOLBOX_METRICS_ENABLED") == "true"
}

func (b *LocalMetricsManager) PublishBatch(metrics []Metric) error {
if !b.Enabled() {
return nil
}

for _, metric := range metrics {
err := b.Publish(metric)
if err != nil {
return err
}
}

return nil
}

func (b *LocalMetricsManager) Publish(metric Metric) error {
if !b.Enabled() {
return nil
}

switch metric.Name {
case CacheDownloadSize, CacheDownloadTime, CacheUser, CacheServer:
return publishMetricToFile(b.CacheMetricsPath, metric.Name, metric.Value)
case CacheTotalRate, CacheCorruptionRate:
return publishMetricToFile(b.ToolboxMetricsPath, metric.Name, metric.Value)
}

fmt.Printf("Ignoring metric %s\n", metric.Name)
return nil
}

func publishMetricToFile(file, metricName, metricValue string) error {
f, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
return err
}

defer f.Close()

line := fmt.Sprintf("%s %s\n", metricName, metricValue)

_, err = f.WriteString(line)
return err
}
Loading

0 comments on commit 6c85b98

Please sign in to comment.