Merge pull request #2 from railwayapp/jr/pro-3683-secret-and-variables

Secrets

.gitignore

@@ -24,6 +24,8 @@
 # Go workspace file
 go.work
 
+node_modules
+
 # End of https://www.toptal.com/developers/gitignore/api/go
 
 bin

buildkit/build.go

@@ -14,6 +14,8 @@ import (
 	_ "github.com/moby/buildkit/client/connhelper/dockercontainer"
 	_ "github.com/moby/buildkit/client/connhelper/nerdctlcontainer"
 	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/secrets/secretsprovider"
 	"github.com/moby/buildkit/util/appcontext"
 	_ "github.com/moby/buildkit/util/grpcutil/encoding/proto"
 	"github.com/moby/buildkit/util/progress/progressui"
@@ -26,6 +28,8 @@ type BuildWithBuildkitClientOptions struct {
 	DumpLLB      bool
 	OutputDir    string
 	ProgressMode string
+	SecretsHash  string
+	Secrets      map[string]string
 }
 
 func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWithBuildkitClientOptions) error {
@@ -60,6 +64,7 @@ func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWith
 
 	llbState, image, err := ConvertPlanToLLB(plan, ConvertPlanOptions{
 		BuildPlatform: buildPlatform,
+		SecretsHash:   opts.SecretsHash,
 	})
 	if err != nil {
 		return fmt.Errorf("error converting plan to LLB: %w", err)
@@ -142,10 +147,17 @@ func BuildWithBuildkitClient(appDir string, plan *plan.BuildPlan, opts BuildWith
 
 	log.Debugf("Building image for %s with BuildKit %s", buildPlatform.String(), info.BuildkitVersion.Version)
 
+	secretsMap := make(map[string][]byte)
+	for k, v := range opts.Secrets {
+		secretsMap[k] = []byte(v)
+	}
+	secrets := secretsprovider.FromMap(secretsMap)
+
 	solveOpts := client.SolveOpt{
 		LocalMounts: map[string]fsutil.FS{
 			"context": appFS,
 		},
+		Session: []session.Attachable{secrets},
 		Exports: []client.ExportEntry{
 			{
 				Type: client.ExporterDocker,

buildkit/cache_store.go

@@ -28,7 +28,7 @@ func NewBuildKitCacheStore(uniqueID string) *BuildKitCacheStore {
 
 func (c *BuildKitCacheStore) GetCache(key string, planCache *plan.Cache) BuildKitCache {
 	cacheKey := key
-	if cacheKey == "" {
+	if c.uniqueID != "" {
 		cacheKey = fmt.Sprintf("%s-%s", c.uniqueID, key)
 	}
 

buildkit/convert.go

@@ -12,6 +12,7 @@ import (
 
 type ConvertPlanOptions struct {
 	BuildPlatform BuildPlatform
+	SecretsHash   string
 	CacheKey      string
 }
 
@@ -24,14 +25,9 @@ func ConvertPlanToLLB(plan *p.BuildPlan, opts ConvertPlanOptions) (*llb.State, *
 
 	state := getBaseState(platform)
 
-	// Add all variables as environment variables
-	for name, value := range plan.Variables {
-		state = state.AddEnv(name, value)
-	}
-
 	cacheStore := NewBuildKitCacheStore(opts.CacheKey)
 
-	graph, err := NewBuildGraph(plan, &state, cacheStore, &platform)
+	graph, err := NewBuildGraph(plan, &state, cacheStore, opts.SecretsHash, &platform)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -124,10 +120,6 @@ func getImageEnv(graphOutput *BuildGraphOutput, plan *p.BuildPlan) []string {
 		imageEnv = append(imageEnv, fmt.Sprintf("%s=%s", k, v))
 	}
 
-	for k, v := range plan.Start.Env {
-		imageEnv = append(imageEnv, fmt.Sprintf("%s=%s", k, v))
-	}
-
 	return imageEnv
 }
 

buildkit/frontend.go

@@ -23,7 +23,11 @@ const (
 	configMountName = "dockerfile"
 
 	// The default filename for the serialized Railpack plan
-	defaultRailpackPlan = "rpk.json"
+	defaultRailpackPlan = "railpack-plan.json"
+
+	secretsHash = "secrets-hash"
+
+	cacheKey = "cache-key"
 )
 
 func StartFrontend() {
@@ -37,7 +41,11 @@ func StartFrontend() {
 }
 
 func Build(ctx context.Context, c client.Client) (*client.Result, error) {
-	buildPlatform, err := validatePlatform(c.BuildOpts().Opts)
+	opts := c.BuildOpts().Opts
+	cacheKey := opts[cacheKey]
+	secretsHash := opts[secretsHash]
+
+	buildPlatform, err := validatePlatform(opts)
 	if err != nil {
 		return nil, err
 	}
@@ -47,8 +55,15 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 		return nil, err
 	}
 
+	_, err = json.MarshalIndent(plan, "", "  ")
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling plan: %w", err)
+	}
+
 	llbState, image, err := ConvertPlanToLLB(plan, ConvertPlanOptions{
 		BuildPlatform: buildPlatform,
+		SecretsHash:   secretsHash,
+		CacheKey:      cacheKey,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error converting plan to LLB: %w", err)

buildkit/graph.go

@@ -13,11 +13,12 @@ import (
 )
 
 type BuildGraph struct {
-	Nodes      map[string]*Node
-	BaseState  *llb.State
-	CacheStore *BuildKitCacheStore
-	Plan       *plan.BuildPlan
-	Platform   *specs.Platform
+	Nodes       map[string]*Node
+	BaseState   *llb.State
+	CacheStore  *BuildKitCacheStore
+	SecretsHash string
+	Plan        *plan.BuildPlan
+	Platform    *specs.Platform
 }
 
 type BuildGraphOutput struct {
@@ -26,13 +27,14 @@ type BuildGraphOutput struct {
 	EnvVars  map[string]string
 }
 
-func NewBuildGraph(plan *plan.BuildPlan, baseState *llb.State, cacheStore *BuildKitCacheStore, platform *specs.Platform) (*BuildGraph, error) {
+func NewBuildGraph(plan *plan.BuildPlan, baseState *llb.State, cacheStore *BuildKitCacheStore, secretsHash string, platform *specs.Platform) (*BuildGraph, error) {
 	graph := &BuildGraph{
-		Nodes:      make(map[string]*Node),
-		BaseState:  baseState,
-		CacheStore: cacheStore,
-		Plan:       plan,
-		Platform:   platform,
+		Nodes:       make(map[string]*Node),
+		BaseState:   baseState,
+		CacheStore:  cacheStore,
+		SecretsHash: secretsHash,
+		Plan:        plan,
+		Platform:    platform,
 	}
 
 	// Create a node for each step
@@ -268,18 +270,20 @@ func (g *BuildGraph) convertStepToLLB(node *Node, baseState *llb.State) (*llb.St
 	}
 
 	// Process the step commands
-	for _, cmd := range step.Commands {
-		var err error
-		state, err = g.convertCommandToLLB(node, cmd, state, step)
-		if err != nil {
-			return nil, err
+	if step.Commands != nil {
+		for _, cmd := range *step.Commands {
+			var err error
+			state, err = g.convertCommandToLLB(node, cmd, state, step)
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 
-	if len(step.Outputs) > 0 {
+	if step.Outputs != nil {
 		result := llb.Scratch()
 
-		for _, output := range step.Outputs {
+		for _, output := range *step.Outputs {
 			result = result.File(llb.Copy(state, output, output, &llb.CopyInfo{
 				CreateDestPath:      true,
 				AllowWildcard:       true,
@@ -311,6 +315,18 @@ func (g *BuildGraph) convertCommandToLLB(node *Node, cmd plan.Command, state llb
 			opts = append(opts, llb.WithCustomName(cmd.CustomName))
 		}
 
+		if node.Step.UseSecrets == nil || *node.Step.UseSecrets { // default to using secrets
+			for _, secret := range g.Plan.Secrets {
+				opts = append(opts, llb.AddSecret(secret, llb.SecretID(secret), llb.SecretAsEnv(true), llb.SecretAsEnvName(secret)))
+			}
+
+			// If there is a secrets hash, add a mount to invalidate the cache if the secrets hash changes
+			if g.SecretsHash != "" {
+				opts = append(opts, llb.AddMount("/cache-invalidate",
+					llb.Scratch().File(llb.Mkfile("secrets-hash", 0644, []byte(g.SecretsHash)))))
+			}
+		}
+
 		if len(cmd.Caches) > 0 {
 			cacheOpts, err := g.getCacheMountOptions(cmd.Caches)
 			if err != nil {
@@ -526,6 +542,8 @@ func (g *BuildGraph) getCacheMountOptions(cacheKeys []string) ([]llb.RunOption,
 			opts = append(opts,
 				llb.AddMount(planCache.Directory, *cache.cacheState, llb.AsPersistentCacheDir(cache.cacheKey, cacheType)),
 			)
+
+			return opts, nil
 		} else {
 			return nil, fmt.Errorf("cache with key %q not found", cacheKey)
 		}

cli/build.go

@@ -2,11 +2,15 @@ package cli
 
 import (
 	"context"
+	"crypto/sha256"
 	"encoding/json"
+	"fmt"
 
 	"github.com/charmbracelet/log"
 	"github.com/railwayapp/railpack/buildkit"
 	"github.com/railwayapp/railpack/core"
+	"github.com/railwayapp/railpack/core/app"
+	"github.com/railwayapp/railpack/core/plan"
 	"github.com/urfave/cli/v3"
 )
 
@@ -53,7 +57,7 @@ var BuildCommand = &cli.Command{
 		},
 	},
 	Action: func(ctx context.Context, cmd *cli.Command) error {
-		buildResult, app, err := GenerateBuildResultForCommand(cmd)
+		buildResult, app, env, err := GenerateBuildResultForCommand(cmd)
 		if err != nil {
 			return cli.Exit(err, 1)
 		}
@@ -69,11 +73,20 @@ var BuildCommand = &cli.Command{
 			log.Debug(string(serializedPlan))
 		}
 
+		err = validateSecrets(buildResult.Plan, env)
+		if err != nil {
+			return cli.Exit(err, 1)
+		}
+
+		secretsHash := getSecretsHash(env)
+
 		err = buildkit.BuildWithBuildkitClient(app.Source, buildResult.Plan, buildkit.BuildWithBuildkitClientOptions{
 			ImageName:    cmd.String("name"),
 			DumpLLB:      cmd.Bool("llb"),
 			OutputDir:    cmd.String("output"),
 			ProgressMode: cmd.String("progress"),
+			SecretsHash:  secretsHash,
+			Secrets:      env.Variables,
 		})
 		if err != nil {
 			return cli.Exit(err, 1)
@@ -82,3 +95,22 @@ var BuildCommand = &cli.Command{
 		return nil
 	},
 }
+
+func validateSecrets(plan *plan.BuildPlan, env *app.Environment) error {
+	for _, secret := range plan.Secrets {
+		if _, ok := env.Variables[secret]; !ok {
+			return fmt.Errorf("missing environment variable: %s. Please set the envvar with --env %s=%s", secret, secret, "...")
+		}
+	}
+	return nil
+}
+
+func getSecretsHash(env *app.Environment) string {
+	secretsValue := ""
+	for _, v := range env.Variables {
+		secretsValue += v
+	}
+	hasher := sha256.New()
+	hasher.Write([]byte(secretsValue))
+	return fmt.Sprintf("%x", hasher.Sum(nil))
+}

cli/common.go

@@ -9,16 +9,16 @@ import (
 	"github.com/urfave/cli/v3"
 )
 
-func GenerateBuildResultForCommand(cmd *cli.Command) (*core.BuildResult, *a.App, error) {
+func GenerateBuildResultForCommand(cmd *cli.Command) (*core.BuildResult, *a.App, *a.Environment, error) {
 	directory := cmd.Args().First()
 
 	if directory == "" {
-		return nil, nil, cli.Exit("directory argument is required", 1)
+		return nil, nil, nil, cli.Exit("directory argument is required", 1)
 	}
 
 	app, err := a.NewApp(directory)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error creating app: %w", err)
+		return nil, nil, nil, fmt.Errorf("error creating app: %w", err)
 	}
 
 	log.Debugf("Building %s", app.Source)
@@ -27,7 +27,7 @@ func GenerateBuildResultForCommand(cmd *cli.Command) (*core.BuildResult, *a.App,
 
 	env, err := a.FromEnvs(envsArgs)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error creating env: %w", err)
+		return nil, nil, nil, fmt.Errorf("error creating env: %w", err)
 	}
 
 	generateOptions := &core.GenerateBuildPlanOptions{
@@ -37,8 +37,8 @@ func GenerateBuildResultForCommand(cmd *cli.Command) (*core.BuildResult, *a.App,
 
 	buildResult, err := core.GenerateBuildPlan(app, env, generateOptions)
 	if err != nil {
-		return nil, nil, fmt.Errorf("error generating build plan: %w", err)
+		return nil, nil, nil, fmt.Errorf("error generating build plan: %w", err)
 	}
 
-	return buildResult, app, nil
+	return buildResult, app, env, nil
 }

cli/info.go

@@ -37,7 +37,7 @@ var InfoCommand = &cli.Command{
 		},
 	},
 	Action: func(ctx context.Context, cmd *cli.Command) error {
-		buildResult, _, err := GenerateBuildResultForCommand(cmd)
+		buildResult, _, _, err := GenerateBuildResultForCommand(cmd)
 		if err != nil {
 			return cli.Exit(err, 1)
 		}

cli/plan.go

@@ -34,7 +34,7 @@ var PlanCommand = &cli.Command{
 		},
 	},
 	Action: func(ctx context.Context, cmd *cli.Command) error {
-		buildResult, _, err := GenerateBuildResultForCommand(cmd)
+		buildResult, _, _, err := GenerateBuildResultForCommand(cmd)
 		if err != nil {
 			return cli.Exit(err, 1)
 		}