From 3e7cfa5db88c38e72265f5278de03607743f1e02 Mon Sep 17 00:00:00 2001 From: karlderkaefer Date: Sat, 14 Dec 2024 11:52:00 +0100 Subject: [PATCH] feat: use random helm cache dir --- README.md | 6 +++++- helm.go | 26 ++++++++++++++++---------- kube.go | 39 +++++++++++++++++++++++++++++++++++---- main.go | 3 +++ 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ebf07e2..202755b 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ repoServer: | HELM_DEPS_SKIP_REFRESH | false | Skips repository update for sub-dependencies, providing a significant performance boost. | | HELM_DEPS_SKIP_START_LOGIN | false | Skips login to all available registries at the start if combined with HELM_DEPS_SKIP_REFRESH=true. | | HELM_DEPS_FETCH_ARGOCD_REPO_SECRETS | false | Adds registries that are registered in the argocd repository secrets. | +| HELM_DEPS_RANDOM_CACHE_DIR | false | Uses a random cache directory for the helm plugin. Useful for parallel executions when using in ArgoCD. This give additional speed boost of ~30% | ### Local @@ -123,6 +124,9 @@ helm template . -f deps://dummy -f env/qa/values.yaml 200.83s user 19.00s syste export HELM_DEPS_SKIP_REFRESH=true export HELM_DEPS_SKIP_REPO_OVERWRITE=true export HELM_DEPS_SKIP_START_LOGIN=true -helm template . -f deps://dummy -f env/qa/values.yaml 129.04s user 8.81s system 286% cpu 48.064 total # V2 max performance +helm template . -f deps://dummy -f env/qa/values.yaml 129.04s user 8.81s system 286% cpu 48.064 total # V2 Performance + +export HELM_DEPS_RANDOM_CACHE_DIR=true +helm template -f env/qa/values.yaml -f deps://dummy . > app.yaml 41.07s user 6.11s system 184% cpu 25.541 total # V2 Performance with Random Cache Dir ``` diff --git a/helm.go b/helm.go index dadd1a7..4208e72 100644 --- a/helm.go +++ b/helm.go @@ -35,6 +35,7 @@ type HelmUpdateConfig struct { SkipRepoOverwrite bool // ENV: HELM_DEPS_SKIP_REPO_OVERWRITE SkipDepdencyRefresh bool // ENV: HELM_DEPS_SKIP_REFRESH FetchArgocdRepoSecrets bool // ENV: HELM_DEPS_FETCH_ARGOCD_REPO_SECRETS + UseRandomHelmCacheDir bool // ENV: HELM_DEPS_RANDOM_CACHE_DIR } func (c *ChartInfo) AddDependencyUrl(depdencyUrl string) error { @@ -183,17 +184,22 @@ func helmRepoExists(registry *RegistryInfo, config *HelmUpdateConfig) (bool, err cmd := exec.Command("helm", "repo", "ls") var out bytes.Buffer cmd.Stdout = &out + cmd.Stderr = &out err := cmd.Run() - if err != nil { - return false, fmt.Errorf("failed to run helm repo ls: %w", err) - } - output := out.String() - lines := strings.Split(output, "\n") - for _, line := range lines { - if strings.Contains(line, registry.Hostname) { - return true, nil - } - } + if err != nil { + // Check if the error is due to no repositories existing + if strings.Contains(out.String(), "no repositories to show") { + return false, nil + } + return false, fmt.Errorf("failed to run helm repo ls: %w", err) + } + output := out.String() + lines := strings.Split(output, "\n") + for _, line := range lines { + if strings.Contains(line, registry.Hostname) { + return true, nil + } + } return false, nil } diff --git a/kube.go b/kube.go index b2bc796..9ecf8e9 100644 --- a/kube.go +++ b/kube.go @@ -8,8 +8,14 @@ package main import ( "context" "errors" + "fmt" "log" + "os" + "path/filepath" "strings" + "time" + + "math/rand" "helm.sh/helm/v3/pkg/kube" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -30,7 +36,7 @@ type RegistryHelper struct { Registries map[string]*RegistryInfo Namespace string kubernetesClient KubeClientInterface - config *HelmUpdateConfig + config *HelmUpdateConfig } type RegistryInfo struct { @@ -83,14 +89,39 @@ func NewRegistryHelper(secretNames string, namespace string, config *HelmUpdateC registryMap[registry] = &RegistryInfo{} } } - return &RegistryHelper{ + r := &RegistryHelper{ Registries: registryMap, Namespace: namespace, - config: config, + config: config, + } + if config.UseRandomHelmCacheDir { + r.SetRandomHelmCacheDir() } + return r } +func (r *RegistryHelper) SetRandomHelmCacheDir() error { + // Create a local rand generator + seed := rand.New(rand.NewSource(time.Now().UnixNano())) + // Create unique temporary directories + cacheDir := fmt.Sprintf("/tmp/helm_cache_%d", seed.Int()) + configDir := fmt.Sprintf("/tmp/helm_config_%d", seed.Int()) + + // Ensure the directories exist + err := os.MkdirAll(cacheDir, 0755) + if err != nil { + return fmt.Errorf("failed to create cache directory: %w", err) + } + err = os.MkdirAll(configDir, 0755) + if err != nil { + return fmt.Errorf("failed to create config directory: %w", err) + } + // Set the environment variables for Helm + os.Setenv("HELM_REPOSITORY_CACHE", cacheDir) + os.Setenv("HELM_REPOSITORY_CONFIG", filepath.Join(configDir, "repositories.yaml")) + return nil +} // GetRegistry returns the credential protected registry by hostname func (r *RegistryHelper) GetRegistryByHostname(registry string) *RegistryInfo { @@ -122,7 +153,7 @@ func (r *RegistryHelper) LoginIfExists(registry *RegistryInfo) error { return action.Login() } if !registry.EnableOCI { - log.Printf("Registry %s not found in secrets, adding it as a repo", registry.SecretName) + log.Printf("Registry %s ≈", registry.SecretName) action := GetRegistryAction(registry) return action.Login() } diff --git a/main.go b/main.go index bd38c94..fe51cb6 100644 --- a/main.go +++ b/main.go @@ -33,6 +33,8 @@ func main() { skipLoginAtStartFlag := flag.Bool("skip-login-at-start", skipLoginAtStart, "Env: HELM_DEPS_SKIP_START_LOGIN (default false). This will skip login to all available registry at the start in combination with HELM_DEPS_SKIP_REFRESH=true") fetchArgocdRepoSecrets := parseBoolEnv("HELM_DEPS_FETCH_ARGOCD_REPO_SECRETS", false) fetchArgocdRepoSecretsFlag := flag.Bool("fetch-argocd-repo-secrets", fetchArgocdRepoSecrets, "Env: HELM_DEPS_FETCH_ARGOCD_REPO_SECRETS (default false). Fetch the argocd repository secrets as registries") + useRandomHelmCacheDir := parseBoolEnv("HELM_DEPS_RANDOM_CACHE_DIR", false) + useRandomHelmCacheDirFlag := flag.Bool("use-random-helm-cache-dir", useRandomHelmCacheDir, "Env: HELM_DEPS_RANDOM_CACHE_DIR (default false). Use a random cache directory for helm") flag.Parse() @@ -40,6 +42,7 @@ func main() { FetchArgocdRepoSecrets: *fetchArgocdRepoSecretsFlag, SkipRepoOverwrite: *skipRepoOverwriteFlag, SkipDepdencyRefresh: *skipDepRefreshFlag, + UseRandomHelmCacheDir: *useRandomHelmCacheDirFlag, } // just for backward compatibility comnbinedSecretNames := strings.Join([]string{*secretNames, *addRegistries}, ",")