From 1d6d768fe8c709ed3a56dcb3014ba3d9428c0b53 Mon Sep 17 00:00:00 2001 From: Jake Runzer Date: Mon, 27 Jan 2025 17:23:56 -0500 Subject: [PATCH] Closes PRO-3675. Exec commands can reference multiple caches --- buildkit/graph.go | 39 +++++++++++++++++--------- core/generate/cache_context.go | 11 ++++++-- core/generate/context.go | 4 +-- core/generate/mise_step_builder.go | 4 +-- core/plan/command.go | 10 +++---- core/plan/command_test.go | 4 +-- core/providers/golang/golang.go | 4 +-- core/providers/node/package_manager.go | 12 ++++---- 8 files changed, 54 insertions(+), 34 deletions(-) diff --git a/buildkit/graph.go b/buildkit/graph.go index 23f2695..4b0926c 100644 --- a/buildkit/graph.go +++ b/buildkit/graph.go @@ -318,20 +318,12 @@ func (g *BuildGraph) convertCommandToLLB(node *Node, cmd plan.Command, state llb opts = append(opts, llb.WithCustomName(cmd.CustomName)) } - if cmd.CacheKey != "" { - if planCache, ok := g.Plan.Caches[cmd.CacheKey]; ok { - cache := g.CacheStore.GetCache(cmd.CacheKey, planCache) - cacheType := llb.CacheMountShared - if planCache.Type == plan.CacheTypeLocked { - cacheType = llb.CacheMountLocked - } - - opts = append(opts, - llb.AddMount(planCache.Directory, *cache.cacheState, llb.AsPersistentCacheDir(cache.cacheKey, cacheType)), - ) - } else { - return state, fmt.Errorf("cache with key %q not found", cmd.CacheKey) + if len(cmd.Caches) > 0 { + cacheOpts, err := g.getCacheMountOptions(cmd.Caches) + if err != nil { + return state, err } + opts = append(opts, cacheOpts...) } s := state.Run(opts...).Root() @@ -526,3 +518,24 @@ func (g *BuildGraph) PrintGraph() { } fmt.Println("\n=====================") } + +// getCacheMountOptions returns the llb.RunOption slice for the given cache keys +func (g *BuildGraph) getCacheMountOptions(cacheKeys []string) ([]llb.RunOption, error) { + var opts []llb.RunOption + for _, cacheKey := range cacheKeys { + if planCache, ok := g.Plan.Caches[cacheKey]; ok { + cache := g.CacheStore.GetCache(cacheKey, planCache) + cacheType := llb.CacheMountShared + if planCache.Type == plan.CacheTypeLocked { + cacheType = llb.CacheMountLocked + } + + opts = append(opts, + llb.AddMount(planCache.Directory, *cache.cacheState, llb.AsPersistentCacheDir(cache.cacheKey, cacheType)), + ) + } else { + return nil, fmt.Errorf("cache with key %q not found", cacheKey) + } + } + return opts, nil +} diff --git a/core/generate/cache_context.go b/core/generate/cache_context.go index 495273a..ef93169 100644 --- a/core/generate/cache_context.go +++ b/core/generate/cache_context.go @@ -26,12 +26,19 @@ func (c *CacheContext) GetCache(name string) *plan.Cache { return c.Caches[name] } -func (c *CacheContext) GetAptCache() string { +func (c *CacheContext) GetAptCaches() []string { if _, ok := c.Caches[APT_CACHE_KEY]; !ok { aptCache := plan.NewCache("/var/cache/apt") aptCache.Type = plan.CacheTypeLocked c.Caches[APT_CACHE_KEY] = aptCache } - return APT_CACHE_KEY + aptListsKey := "apt-lists" + if _, ok := c.Caches[aptListsKey]; !ok { + aptListsCache := plan.NewCache("/var/lib/apt/lists") + aptListsCache.Type = plan.CacheTypeLocked + c.Caches[aptListsKey] = aptListsCache + } + + return []string{APT_CACHE_KEY, aptListsKey} } diff --git a/core/generate/context.go b/core/generate/context.go index e08dd8d..c1b391e 100644 --- a/core/generate/context.go +++ b/core/generate/context.go @@ -84,8 +84,8 @@ func (c *GenerateContext) ResolvePackages() (map[string]*resolver.ResolvedPackag } func (o *BuildStepOptions) NewAptInstallCommand(pkgs []string) plan.Command { - return plan.NewExecCommand("sh -c 'apt-get update && apt-get install -y "+strings.Join(pkgs, " ")+" && rm -rf /var/lib/apt/lists/*'", plan.ExecOptions{ + return plan.NewExecCommand("sh -c 'apt-get update && apt-get install -y "+strings.Join(pkgs, " ")+"'", plan.ExecOptions{ CustomName: "install apt packages: " + strings.Join(pkgs, " "), - CacheKey: o.Caches.GetAptCache(), + Caches: o.Caches.GetAptCaches(), }) } diff --git a/core/generate/mise_step_builder.go b/core/generate/mise_step_builder.go index 5afdd3d..a702a96 100644 --- a/core/generate/mise_step_builder.go +++ b/core/generate/mise_step_builder.go @@ -85,7 +85,7 @@ func (b *MiseStepBuilder) Build(options *BuildStepOptions) (*plan.Step, error) { plan.NewExecCommand("sh -c 'curl -fsSL https://mise.run | sh'", plan.ExecOptions{ CustomName: "install mise", - CacheKey: miseCache, + Caches: []string{miseCache}, }), }) @@ -133,7 +133,7 @@ func (b *MiseStepBuilder) Build(options *BuildStepOptions) (*plan.Step, error) { }), plan.NewExecCommand("sh -c 'mise trust -a && mise install'", plan.ExecOptions{ CustomName: "install mise packages: " + strings.Join(pkgNames, ", "), - CacheKey: miseCache, + Caches: []string{miseCache}, }), }) } diff --git a/core/plan/command.go b/core/plan/command.go index b2f6cb1..04106d2 100644 --- a/core/plan/command.go +++ b/core/plan/command.go @@ -13,15 +13,15 @@ type Command interface { } type ExecOptions struct { - CacheKey string + Caches []string CustomName string } // ExecCommand represents a command to be executed type ExecCommand struct { - Cmd string `json:"cmd"` - CacheKey string `json:"cacheKey,omitempty"` - CustomName string `json:"customName,omitempty"` + Cmd string `json:"cmd"` + Caches []string `json:"caches,omitempty"` + CustomName string `json:"customName,omitempty"` } // PathCommand represents a global path addition @@ -65,7 +65,7 @@ func NewExecCommand(cmd string, options ...ExecOptions) Command { exec := ExecCommand{Cmd: cmd} if len(options) > 0 { exec.CustomName = options[0].CustomName - exec.CacheKey = options[0].CacheKey + exec.Caches = options[0].Caches } return exec } diff --git a/core/plan/command_test.go b/core/plan/command_test.go index ec01b46..6266db5 100644 --- a/core/plan/command_test.go +++ b/core/plan/command_test.go @@ -29,8 +29,8 @@ func TestCommandMarshalUnmarshal(t *testing.T) { }, { name: "exec command with cache key", - command: ExecCommand{Cmd: "npm install", CacheKey: "v1"}, - expectedJSON: `{"cmd":"npm install","cacheKey":"v1"}`, + command: ExecCommand{Cmd: "npm install", Caches: []string{"v1", "v2"}}, + expectedJSON: `{"cmd":"npm install","caches":["v1","v2"]}`, unmarshalString: "", }, diff --git a/core/providers/golang/golang.go b/core/providers/golang/golang.go index 7e71d23..58059d2 100644 --- a/core/providers/golang/golang.go +++ b/core/providers/golang/golang.go @@ -90,7 +90,7 @@ func (p *GoProvider) Build(ctx *generate.GenerateContext, packages *generate.Mis build.AddCommands([]plan.Command{ plan.NewCopyCommand("."), plan.NewExecCommand(buildCmd, plan.ExecOptions{ - CacheKey: p.goBuildCacheKey(ctx), + Caches: []string{p.goBuildCacheKey(ctx)}, }), }) @@ -115,7 +115,7 @@ func (p *GoProvider) Install(ctx *generate.GenerateContext, packages *generate.M plan.NewCopyCommand("go.mod"), plan.NewCopyCommand("go.sum"), plan.NewExecCommand("go mod download", plan.ExecOptions{ - CacheKey: p.goBuildCacheKey(ctx), + Caches: []string{p.goBuildCacheKey(ctx)}, }), }) diff --git a/core/providers/node/package_manager.go b/core/providers/node/package_manager.go index 7b0e696..b1b75a0 100644 --- a/core/providers/node/package_manager.go +++ b/core/providers/node/package_manager.go @@ -71,26 +71,26 @@ func (p PackageManager) InstallDeps(ctx *generate.GenerateContext, install *gene npmCache := ctx.Caches.AddCache("npm-install", "/root/.npm") if hasLockfile { - install.AddCommand(plan.NewExecCommand("npm ci", plan.ExecOptions{CacheKey: npmCache})) + install.AddCommand(plan.NewExecCommand("npm ci", plan.ExecOptions{Caches: []string{npmCache}})) } else { install.AddCommand(plan.NewExecCommand("npm install", - plan.ExecOptions{CacheKey: npmCache})) + plan.ExecOptions{Caches: []string{npmCache}})) } case PackageManagerPnpm: install.AddCommand(plan.NewExecCommand("pnpm install --frozen-lockfile", plan.ExecOptions{ - CacheKey: ctx.Caches.AddCache("pnpm-install", "/root/.local/share/pnpm/store/v3"), + Caches: []string{ctx.Caches.AddCache("pnpm-install", "/root/.local/share/pnpm/store/v3")}, })) case PackageManagerBun: install.AddCommand(plan.NewExecCommand("bun i --no-save", plan.ExecOptions{ - CacheKey: ctx.Caches.AddCache("bun-install", "/root/.bun/install/cache"), + Caches: []string{ctx.Caches.AddCache("bun-install", "/root/.bun/install/cache")}, })) case PackageManagerYarn1: install.AddCommand(plan.NewExecCommand("yarn install --frozen-lockfile", plan.ExecOptions{ - CacheKey: ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn"), + Caches: []string{ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn")}, })) case PackageManagerYarn2: install.AddCommand(plan.NewExecCommand("yarn install --check-cache", plan.ExecOptions{ - CacheKey: ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn"), + Caches: []string{ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn")}, })) } }