Merge branch 'main' into jr/remove-mise-binary-final-image

buildkit/convert.go

@@ -136,7 +136,8 @@ func getBaseState(plan *p.BuildPlan, platform specs.Platform) llb.State {
 		llb.Platform(platform),
 	)
 
-	state = state.Run(llb.Shlex("sh -c 'apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*'"), llb.WithCustomName("install base packages")).Root()
+	state = state.AddEnv("DEBIAN_FRONTEND", "noninteractive")
+	state = state.Run(llb.Shlex("sh -c 'apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/*'")).Root()
 	state = state.Dir(WorkingDir)
 
 	return state

cli/build.go

@@ -19,28 +19,11 @@ var BuildCommand = &cli.Command{
 	Usage:                 "build an image with BuildKit",
 	ArgsUsage:             "DIRECTORY",
 	EnableShellCompletion: true,
-	Flags: []cli.Flag{
+	Flags: append([]cli.Flag{
 		&cli.StringFlag{
 			Name:  "name",
 			Usage: "name of the image to build",
 		},
-		&cli.StringSliceFlag{
-			Name:  "env",
-			Usage: "environment variables to set",
-		},
-		&cli.StringFlag{
-			Name:  "build-cmd",
-			Usage: "build command to use",
-		},
-		&cli.StringFlag{
-			Name:  "start-cmd",
-			Usage: "start command to use",
-		},
-		&cli.BoolFlag{
-			Name:  "llb",
-			Usage: "output the LLB plan to stdout instead of building the image",
-			Value: false,
-		},
 		&cli.StringFlag{
 			Name:  "output",
 			Usage: "output the final filesystem to a local directory",
@@ -55,11 +38,7 @@ var BuildCommand = &cli.Command{
 			Usage: "Show the build plan before building. This is useful for development and debugging.",
 			Value: false,
 		},
-		&cli.StringSliceFlag{
-			Name:  "previous-versions",
-			Usage: "versions of packages used for previous builds. These versions will be used instead of the defaults. format: NAME@VERSION",
-		},
-	},
+	}, commonPlanFlags()...),
 	Action: func(ctx context.Context, cmd *cli.Command) error {
 		buildResult, app, env, err := GenerateBuildResultForCommand(cmd)
 		if err != nil {

cli/common.go

@@ -10,6 +10,27 @@ import (
 	"github.com/urfave/cli/v3"
 )
 
+func commonPlanFlags() []cli.Flag {
+	return []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:  "env",
+			Usage: "environment variables to set",
+		},
+		&cli.StringSliceFlag{
+			Name:  "previous-versions",
+			Usage: "versions of packages used for previous builds",
+		},
+		&cli.StringFlag{
+			Name:  "build-cmd",
+			Usage: "build command to use",
+		},
+		&cli.StringFlag{
+			Name:  "start-cmd",
+			Usage: "start command to use",
+		},
+	}
+}
+
 func GenerateBuildResultForCommand(cmd *cli.Command) (*core.BuildResult, *a.App, *a.Environment, error) {
 	directory := cmd.Args().First()
 

cli/info.go

@@ -17,29 +17,17 @@ var InfoCommand = &cli.Command{
 	Usage:                 "get as much information as possible about an app",
 	ArgsUsage:             "DIRECTORY",
 	EnableShellCompletion: true,
-	Flags: []cli.Flag{
-		&cli.StringSliceFlag{
-			Name:  "env",
-			Usage: "environment variables to set. format: KEY=VALUE",
-		},
-		&cli.StringFlag{
-			Name:  "build-cmd",
-			Usage: "build command to use",
-		},
-		&cli.StringFlag{
-			Name:  "start-cmd",
-			Usage: "start command to use",
-		},
+	Flags: append([]cli.Flag{
 		&cli.StringFlag{
 			Name:  "format",
 			Usage: "output format. one of: pretty, json",
 			Value: "pretty",
 		},
-		&cli.StringSliceFlag{
-			Name:  "previous",
-			Usage: "versions of packages used for previous builds. These versions will be used instead of the defaults. format: NAME@VERSION",
+		&cli.StringFlag{
+			Name:  "out",
+			Usage: "output file name",
 		},
-	},
+	}, commonPlanFlags()...),
 	Action: func(ctx context.Context, cmd *cli.Command) error {
 		buildResult, _, _, err := GenerateBuildResultForCommand(cmd)
 		if err != nil {

cli/plan.go

@@ -16,21 +16,13 @@ var PlanCommand = &cli.Command{
 	Usage:                 "generate a build plan for a directory",
 	ArgsUsage:             "DIRECTORY",
 	EnableShellCompletion: true,
-	Flags: []cli.Flag{
-		&cli.StringSliceFlag{
-			Name:  "env",
-			Usage: "environment variables to set. format: KEY=VALUE",
-		},
+	Flags: append([]cli.Flag{
 		&cli.StringFlag{
 			Name:    "out",
 			Aliases: []string{"o"},
 			Usage:   "output file name",
 		},
-		&cli.StringSliceFlag{
-			Name:  "previous",
-			Usage: "versions of packages used for previous builds. These versions will be used instead of the defaults. format: NAME@VERSION",
-		},
-	},
+	}, commonPlanFlags()...),
 	Action: func(ctx context.Context, cmd *cli.Command) error {
 		buildResult, _, _, err := GenerateBuildResultForCommand(cmd)
 		if err != nil {

cmd/cli/main.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"os"
 
+	"github.com/charmbracelet/lipgloss"
 	"github.com/charmbracelet/log"
+	"github.com/muesli/termenv"
 	"github.com/railwayapp/railpack/cli"
 	urfave "github.com/urfave/cli/v3"
 )
@@ -19,6 +21,10 @@ func main() {
 	}).Writer()
 	urfave.ErrWriter = urfaveLogWriter
 
+	if os.Getenv("FORCE_COLOR") != "" {
+		lipgloss.SetColorProfile(termenv.TrueColor)
+	}
+
 	cmd := &urfave.Command{
 		Name:                  "railpack",
 		Usage:                 "Automatically analyze and generate build plans for applications",

core/prettyPrint.go

@@ -112,7 +112,7 @@ func FormatBuildResult(br *BuildResult, options ...PrintOptions) string {
 	// Steps section
 	stepsToPrint := getStepsToPrint(br)
 	if len(stepsToPrint) > 0 {
-		output.WriteString(sectionHeaderStyle.Render("Steps"))
+		output.WriteString(sectionHeaderStyle.MarginTop(1).Render("Steps"))
 		output.WriteString("\n")
 
 		stepCount := 0

core/providers/golang/golang.go

@@ -123,7 +123,7 @@ func (p *GoProvider) Install(ctx *generate.GenerateContext, packages *generate.M
 	// If CGO is enabled, we need to install the gcc packages
 	if p.hasCGOEnabled(ctx) {
 		aptStep := ctx.NewAptStepBuilder("cgo")
-		aptStep.Packages = []string{"gcc", "g++", "libc6-dev", "libgcc-9-dev", "libstdc++-9-dev"}
+		aptStep.Packages = []string{"gcc", "g++", "libc6-dev"}
 		install.DependsOn = append(install.DependsOn, aptStep.DisplayName)
 	} else {
 		install.AddCommand(plan.NewVariableCommand("CGO_ENABLED", "0"))

docs/astro.config.mjs

@@ -17,10 +17,8 @@ export default defineConfig({
         github: "https://github.com/railwayapp/railpack",
       },
       customCss: [
-        // Path to your Tailwind base styles:
         "./src/tailwind.css",
 
-        // Fontsource files for to regular and semi-bold font weights.
         "@fontsource/inter/400.css",
         "@fontsource/inter/600.css",
       ],
@@ -85,14 +83,13 @@ export default defineConfig({
               label: "Package Resolution",
               link: "/architecture/package-resolution",
             },
-            { label: "Plan Generation", link: "/architecture/plan-generation" },
-            { label: "Secrets and Environment", link: "/architecture/secrets" },
             {
-              label: "Previous Versions",
-              link: "/architecture/previous-versions",
+              label: "Secrets and Variables",
+              link: "/architecture/secrets",
             },
             { label: "BuildKit Generation", link: "/architecture/buildkit" },
             { label: "Caching", link: "/architecture/caching" },
+            { label: "User Config", link: "/architecture/user-config" },
           ],
         },
         {

docs/src/content/docs/architecture/buildkit.md

@@ -3,4 +3,24 @@ title: BuildKit LLB Generation
 description: Understanding how Railpack generates BuildKit LLB definitions
 ---
 
-_todo_
+Railpack takes the build plan and generates a BuildKit LLB definition using the
+[LLB Go API](https://github.com/moby/buildkit#exploring-llb).
+
+The LLB is then either [sent to the BuildKit daemon](/guides/building-with-cli)
+or [used by a custom frontend](/guides/custom-frontend).
+
+Generating LLB directly instead of transpiling the plan into a Dockerfile has
+several advantages:
+
+1. **Custom Frontend Integration**: Direct LLB generation enables integration
+   with BuildKit's frontend gateway. This allows the platform to either use
+   BuildKit through Docker or by interacting with the BuildKit daemon directly.
+
+1. **Caching and Optimization**: Direct LLB generation enables fine-grained
+   control over the build cache, allowing more complex caching than what's
+   possible with Dockerfile generation.
+
+1. **Secret Management**: LLB provides more secure and flexible secret mounting.
+
+1. **Type Safety and Compile-Time Validation**: The build defintion is checked
+   at compile-time using the first party Go library.

docs/src/content/docs/architecture/caching.md

@@ -3,4 +3,53 @@ title: Caching
 description: Understanding Railpack's caching mechanisms
 ---
 
-_todo_
+Railpack takes advantage of BuildKit layer and mount caches to speed up
+successive builds.
+
+## Layer Cache
+
+Railpack takes advantage of BuildKit's layer cache and avoids busting the cache
+when possible. Cache busting events are defined in a granular way as part of the
+[steps commands list](/architecture/overview/#build-step). These include
+
+- Copying files from the local context to the build context
+- Changing environment variables
+- Adding new generated files to the build context
+- Executing shell commands in the build context
+
+## Mount Cache
+
+The [BuildKit mount
+cache](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache)
+is used to save the contents of a directory from the build context between
+builds. This is useful for speeding up commands that download or compile assets
+(e.g. npm install). The directory **does not** appear in the final image.
+
+Caches are defined on the build plan and can be referenced via execution commands.
+
+```json
+{
+  "caches": {
+    "npm-install": {
+      "directory": "/root/.npm",
+      "type": "shared"
+    }
+  },
+
+  "steps": {
+    "install": {
+      "commands": [
+        {
+          "cmd": "npm install",
+          "caches": ["npm-install"]
+        }
+        // ...
+      ]
+      // ...
+    }
+  }
+}
+```
+
+Caches are shared across all steps. This is useful for common caches such as the
+apt-cache or apt-lists.

docs/src/content/docs/architecture/overview.md

@@ -3,4 +3,81 @@ title: High Level Overview
 description: Understanding Railpack's architecture and components
 ---
 
-_todo_
+Railpack is split up into three main components:
+
+- Core
+  - The main logic that analyzes the app and generates the build plan
+- Buildkit
+  - Takes the build plan and generates [BuildKit
+    LLB](https://github.com/moby/buildkit?tab=readme-ov-file#exploring-llb)
+  - Starts a custom frontend or creates a BuildKit client to execute the build
+    plan and generate an image
+- CLI
+  - The main entry point for Railpack
+
+The core can be thought of as a _compiler_. The build plan that is generated is
+independent from Docker, BuildKit, or any other tool that can be used to
+generate an image. At the moment, BuildKit is the only _backend_, but more could
+be added in the future.
+
+## Build Plan
+
+The build plan is a JSON object that contains all the information necessary to
+generate an image. Things that it includes are:
+
+- Base image
+  - File system to start from
+- Steps
+  - Group of commands to run
+- Start information
+  - Command, variables, path, etc. Things needed when starting a container from
+    the image
+- Caches
+  - Caches that are referenced by the commands in the steps
+- Secrets
+  - Build secrets that are referenced by the commands in the steps (just the
+    names, not the values)
+
+### Build Step
+
+A step is a group of commands that is executed sequentially in the build. Steps
+**can depend** on other steps, which means that they run with the filesystem of
+all the outputs of the dependent steps. The build graph is constructed in such a
+way that BuildKit will execute non-dependent steps in parallel.
+
+Steps contain:
+
+- Depends on
+  - List of other steps that this step must run after
+- Commands
+  - List of commands to run in the build
+    - Exec command: run a shell command
+    - Copy command: copy files from the local context (user app) or another
+      image into the current FS
+    - Variable command: Set an environment variable
+    - Path command: Prefix the global PATH with another directory
+    - File command: Create a new file in the current FS referencing the step
+      assets
+- Assets
+  - Mapping of name to file contents that is referenced in a file command
+- Use secrets
+  - Whether or not this step uses build secrets. [Docs](/architecture/secrets)
+- Starting image
+  - Instead of starting from the output of a previous step, it can start from a
+    completely different image. This is typically used for root steps
+- Outputs
+  - List of file system paths that is the "result" of running this step. Parts
+    of the FS that are not included will not appear in the final image. If not
+    explicitly defined, the entire FS is assumed to be the output.
+
+## Providers
+
+Language suppport is provided through... providers. Providers are typically
+associated a single language (e.g. node, python, php, etc.). A provider will
+
+- Detect
+  - Analyze the app and determine if it matches. (e.g. the node provider will
+    check for the precense of a `package.json` file).
+- Build
+  - Modifies the build context with all the steps, commands, and everything that
+    is needed to build for that language/framework.