Closes PRO-3675. Exec commands can reference multiple caches

buildkit/graph.go

@@ -318,20 +318,12 @@ func (g *BuildGraph) convertCommandToLLB(node *Node, cmd plan.Command, state llb
 			opts = append(opts, llb.WithCustomName(cmd.CustomName))
 		}
 
-		if cmd.CacheKey != "" {
-			if planCache, ok := g.Plan.Caches[cmd.CacheKey]; ok {
-				cache := g.CacheStore.GetCache(cmd.CacheKey, planCache)
-				cacheType := llb.CacheMountShared
-				if planCache.Type == plan.CacheTypeLocked {
-					cacheType = llb.CacheMountLocked
-				}
-
-				opts = append(opts,
-					llb.AddMount(planCache.Directory, *cache.cacheState, llb.AsPersistentCacheDir(cache.cacheKey, cacheType)),
-				)
-			} else {
-				return state, fmt.Errorf("cache with key %q not found", cmd.CacheKey)
+		if len(cmd.Caches) > 0 {
+			cacheOpts, err := g.getCacheMountOptions(cmd.Caches)
+			if err != nil {
+				return state, err
 			}
+			opts = append(opts, cacheOpts...)
 		}
 
 		s := state.Run(opts...).Root()
@@ -526,3 +518,24 @@ func (g *BuildGraph) PrintGraph() {
 	}
 	fmt.Println("\n=====================")
 }
+
+// getCacheMountOptions returns the llb.RunOption slice for the given cache keys
+func (g *BuildGraph) getCacheMountOptions(cacheKeys []string) ([]llb.RunOption, error) {
+	var opts []llb.RunOption
+	for _, cacheKey := range cacheKeys {
+		if planCache, ok := g.Plan.Caches[cacheKey]; ok {
+			cache := g.CacheStore.GetCache(cacheKey, planCache)
+			cacheType := llb.CacheMountShared
+			if planCache.Type == plan.CacheTypeLocked {
+				cacheType = llb.CacheMountLocked
+			}
+
+			opts = append(opts,
+				llb.AddMount(planCache.Directory, *cache.cacheState, llb.AsPersistentCacheDir(cache.cacheKey, cacheType)),
+			)
+		} else {
+			return nil, fmt.Errorf("cache with key %q not found", cacheKey)
+		}
+	}
+	return opts, nil
+}

core/generate/cache_context.go

@@ -26,12 +26,19 @@ func (c *CacheContext) GetCache(name string) *plan.Cache {
 	return c.Caches[name]
 }
 
-func (c *CacheContext) GetAptCache() string {
+func (c *CacheContext) GetAptCaches() []string {
 	if _, ok := c.Caches[APT_CACHE_KEY]; !ok {
 		aptCache := plan.NewCache("/var/cache/apt")
 		aptCache.Type = plan.CacheTypeLocked
 		c.Caches[APT_CACHE_KEY] = aptCache
 	}
 
-	return APT_CACHE_KEY
+	aptListsKey := "apt-lists"
+	if _, ok := c.Caches[aptListsKey]; !ok {
+		aptListsCache := plan.NewCache("/var/lib/apt/lists")
+		aptListsCache.Type = plan.CacheTypeLocked
+		c.Caches[aptListsKey] = aptListsCache
+	}
+
+	return []string{APT_CACHE_KEY, aptListsKey}
 }

core/generate/context.go

@@ -84,8 +84,8 @@ func (c *GenerateContext) ResolvePackages() (map[string]*resolver.ResolvedPackag
 }
 
 func (o *BuildStepOptions) NewAptInstallCommand(pkgs []string) plan.Command {
-	return plan.NewExecCommand("sh -c 'apt-get update && apt-get install -y "+strings.Join(pkgs, " ")+" && rm -rf /var/lib/apt/lists/*'", plan.ExecOptions{
+	return plan.NewExecCommand("sh -c 'apt-get update && apt-get install -y "+strings.Join(pkgs, " ")+"'", plan.ExecOptions{
 		CustomName: "install apt packages: " + strings.Join(pkgs, " "),
-		CacheKey:   o.Caches.GetAptCache(),
+		Caches:     o.Caches.GetAptCaches(),
 	})
 }

core/generate/mise_step_builder.go

@@ -85,7 +85,7 @@ func (b *MiseStepBuilder) Build(options *BuildStepOptions) (*plan.Step, error) {
 			plan.NewExecCommand("sh -c 'curl -fsSL https://mise.run | sh'",
 				plan.ExecOptions{
 					CustomName: "install mise",
-					CacheKey:   miseCache,
+					Caches:     []string{miseCache},
 				}),
 		})
 
@@ -133,7 +133,7 @@ func (b *MiseStepBuilder) Build(options *BuildStepOptions) (*plan.Step, error) {
 			}),
 			plan.NewExecCommand("sh -c 'mise trust -a && mise install'", plan.ExecOptions{
 				CustomName: "install mise packages: " + strings.Join(pkgNames, ", "),
-				CacheKey:   miseCache,
+				Caches:     []string{miseCache},
 			}),
 		})
 	}

core/plan/command.go

@@ -13,15 +13,15 @@ type Command interface {
 }
 
 type ExecOptions struct {
-	CacheKey   string
+	Caches     []string
 	CustomName string
 }
 
 // ExecCommand represents a command to be executed
 type ExecCommand struct {
-	Cmd        string `json:"cmd"`
-	CacheKey   string `json:"cacheKey,omitempty"`
-	CustomName string `json:"customName,omitempty"`
+	Cmd        string   `json:"cmd"`
+	Caches     []string `json:"caches,omitempty"`
+	CustomName string   `json:"customName,omitempty"`
 }
 
 // PathCommand represents a global path addition
@@ -65,7 +65,7 @@ func NewExecCommand(cmd string, options ...ExecOptions) Command {
 	exec := ExecCommand{Cmd: cmd}
 	if len(options) > 0 {
 		exec.CustomName = options[0].CustomName
-		exec.CacheKey = options[0].CacheKey
+		exec.Caches = options[0].Caches
 	}
 	return exec
 }

core/plan/command_test.go

@@ -29,8 +29,8 @@ func TestCommandMarshalUnmarshal(t *testing.T) {
 		},
 		{
 			name:            "exec command with cache key",
-			command:         ExecCommand{Cmd: "npm install", CacheKey: "v1"},
-			expectedJSON:    `{"cmd":"npm install","cacheKey":"v1"}`,
+			command:         ExecCommand{Cmd: "npm install", Caches: []string{"v1", "v2"}},
+			expectedJSON:    `{"cmd":"npm install","caches":["v1","v2"]}`,
 			unmarshalString: "",
 		},
 

core/providers/golang/golang.go

@@ -90,7 +90,7 @@ func (p *GoProvider) Build(ctx *generate.GenerateContext, packages *generate.Mis
 	build.AddCommands([]plan.Command{
 		plan.NewCopyCommand("."),
 		plan.NewExecCommand(buildCmd, plan.ExecOptions{
-			CacheKey: p.goBuildCacheKey(ctx),
+			Caches: []string{p.goBuildCacheKey(ctx)},
 		}),
 	})
 
@@ -115,7 +115,7 @@ func (p *GoProvider) Install(ctx *generate.GenerateContext, packages *generate.M
 		plan.NewCopyCommand("go.mod"),
 		plan.NewCopyCommand("go.sum"),
 		plan.NewExecCommand("go mod download", plan.ExecOptions{
-			CacheKey: p.goBuildCacheKey(ctx),
+			Caches: []string{p.goBuildCacheKey(ctx)},
 		}),
 	})
 

core/providers/node/package_manager.go

@@ -71,26 +71,26 @@ func (p PackageManager) InstallDeps(ctx *generate.GenerateContext, install *gene
 		npmCache := ctx.Caches.AddCache("npm-install", "/root/.npm")
 
 		if hasLockfile {
-			install.AddCommand(plan.NewExecCommand("npm ci", plan.ExecOptions{CacheKey: npmCache}))
+			install.AddCommand(plan.NewExecCommand("npm ci", plan.ExecOptions{Caches: []string{npmCache}}))
 		} else {
 			install.AddCommand(plan.NewExecCommand("npm install",
-				plan.ExecOptions{CacheKey: npmCache}))
+				plan.ExecOptions{Caches: []string{npmCache}}))
 		}
 	case PackageManagerPnpm:
 		install.AddCommand(plan.NewExecCommand("pnpm install --frozen-lockfile", plan.ExecOptions{
-			CacheKey: ctx.Caches.AddCache("pnpm-install", "/root/.local/share/pnpm/store/v3"),
+			Caches: []string{ctx.Caches.AddCache("pnpm-install", "/root/.local/share/pnpm/store/v3")},
 		}))
 	case PackageManagerBun:
 		install.AddCommand(plan.NewExecCommand("bun i --no-save", plan.ExecOptions{
-			CacheKey: ctx.Caches.AddCache("bun-install", "/root/.bun/install/cache"),
+			Caches: []string{ctx.Caches.AddCache("bun-install", "/root/.bun/install/cache")},
 		}))
 	case PackageManagerYarn1:
 		install.AddCommand(plan.NewExecCommand("yarn install --frozen-lockfile", plan.ExecOptions{
-			CacheKey: ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn"),
+			Caches: []string{ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn")},
 		}))
 	case PackageManagerYarn2:
 		install.AddCommand(plan.NewExecCommand("yarn install --check-cache", plan.ExecOptions{
-			CacheKey: ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn"),
+			Caches: []string{ctx.Caches.AddCache("yarn-install", "/usr/local/share/.cache/yarn")},
 		}))
 	}
 }