From b8f0190edb190363e04e21268cb2df3105102df3 Mon Sep 17 00:00:00 2001 From: Wouter Verlaek Date: Fri, 13 Feb 2026 15:29:54 +0000 Subject: [PATCH] Add S3 Docker build cache support Add support for S3-backed Docker build layer caching using BuildKit's type=s3 cache backend. This enables faster Docker builds by caching intermediate layers in S3. Configuration via environment variables: - LEEWAY_DOCKER_S3_CACHE_BUCKET: S3 bucket name (required) - LEEWAY_DOCKER_S3_CACHE_REGION: AWS region (required) - LEEWAY_DOCKER_S3_CACHE_PREFIX: Optional prefix for cache keys - LEEWAY_DOCKER_S3_CACHE_MODE: Cache mode (min/max, default: max) - LEEWAY_DOCKER_S3_CACHE_ENDPOINT: Custom S3 endpoint Also adds corresponding CLI flags for all options. When S3 cache is enabled, leeway automatically uses docker buildx build with --cache-from and --cache-to flags. Co-authored-by: Ona --- README.md | 33 +++++ cmd/build.go | 31 +++++ pkg/leeway/build.go | 126 ++++++++++++++++- pkg/leeway/build_internal_test.go | 220 ++++++++++++++++++++++++++++++ 4 files changed, 409 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ff3c39c9..d155d267 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,39 @@ E.g. `component/nested:docker` becomes `COMPONENT_NESTED__DOCKER`. See `leeway build --help` for more details. +#### S3 Docker Build Cache + +Leeway supports S3-backed Docker build layer caching using BuildKit's `type=s3` cache backend. This can significantly speed up Docker builds by caching intermediate layers in S3. + +**Configuration via environment variables:** +```bash +export LEEWAY_DOCKER_S3_CACHE_BUCKET=my-cache-bucket +export LEEWAY_DOCKER_S3_CACHE_REGION=us-east-1 +export LEEWAY_DOCKER_S3_CACHE_PREFIX=docker-cache/ # optional +export LEEWAY_DOCKER_S3_CACHE_MODE=max # optional: 'min' or 'max' (default: max) +export LEEWAY_DOCKER_S3_CACHE_ENDPOINT=https://... # optional: for S3-compatible storage +``` + +**Configuration via CLI flags:** +```bash +leeway build \ + --docker-s3-cache-bucket=my-cache-bucket \ + --docker-s3-cache-region=us-east-1 \ + --docker-s3-cache-prefix=docker-cache/ \ + :my-docker-package +``` + +**Requirements:** +- Docker Buildx (automatically used when S3 cache is configured) +- AWS credentials configured (via environment variables, IAM role, or AWS config file) +- S3 bucket with appropriate permissions + +**Cache modes:** +- `max`: Cache all layers including intermediate layers (better cache hit rate, more storage) +- `min`: Cache only the final layer (less storage, fewer cache hits) + +When S3 cache is enabled, leeway automatically switches to `docker buildx build` with `--cache-from` and `--cache-to` flags pointing to the configured S3 bucket. + ### Generic packages ```YAML config: diff --git a/cmd/build.go b/cmd/build.go index 415827cb..ae6f4448 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -241,6 +241,13 @@ func addBuildFlags(cmd *cobra.Command) { cmd.Flags().Bool("report-github", os.Getenv("GITHUB_OUTPUT") != "", "Report package build success/failure to GitHub Actions using the GITHUB_OUTPUT environment variable") cmd.Flags().Bool("fixed-build-dir", true, "Use a fixed build directory for each package, instead of based on the package version, to better utilize caches based on absolute paths (defaults to true)") cmd.Flags().Bool("docker-export-to-cache", false, "Export Docker images to cache instead of pushing directly (enables SLSA L3 compliance)") + + // Docker S3 build cache flags + cmd.Flags().String("docker-s3-cache-bucket", os.Getenv(leeway.EnvvarDockerS3CacheBucket), "S3 bucket for Docker build layer caching (defaults to $LEEWAY_DOCKER_S3_CACHE_BUCKET)") + cmd.Flags().String("docker-s3-cache-region", os.Getenv(leeway.EnvvarDockerS3CacheRegion), "AWS region for Docker S3 cache bucket (defaults to $LEEWAY_DOCKER_S3_CACHE_REGION)") + cmd.Flags().String("docker-s3-cache-prefix", os.Getenv(leeway.EnvvarDockerS3CachePrefix), "Prefix for Docker S3 cache keys (defaults to $LEEWAY_DOCKER_S3_CACHE_PREFIX)") + cmd.Flags().String("docker-s3-cache-mode", os.Getenv(leeway.EnvvarDockerS3CacheMode), "Docker S3 cache mode: 'min' or 'max' (defaults to 'max')") + cmd.Flags().String("docker-s3-cache-endpoint", os.Getenv(leeway.EnvvarDockerS3CacheEndpoint), "Custom S3 endpoint for Docker cache (for S3-compatible storage)") } func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) { @@ -412,6 +419,29 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) { dockerExportSet = true } + // Get Docker S3 cache configuration from CLI flags (which default to env vars) + dockerS3CacheBucket, _ := cmd.Flags().GetString("docker-s3-cache-bucket") + dockerS3CacheRegion, _ := cmd.Flags().GetString("docker-s3-cache-region") + dockerS3CachePrefix, _ := cmd.Flags().GetString("docker-s3-cache-prefix") + dockerS3CacheMode, _ := cmd.Flags().GetString("docker-s3-cache-mode") + dockerS3CacheEndpoint, _ := cmd.Flags().GetString("docker-s3-cache-endpoint") + + var dockerS3Cache *leeway.DockerS3CacheConfig + if dockerS3CacheBucket != "" && dockerS3CacheRegion != "" { + dockerS3Cache = &leeway.DockerS3CacheConfig{ + Bucket: dockerS3CacheBucket, + Region: dockerS3CacheRegion, + Prefix: dockerS3CachePrefix, + Mode: dockerS3CacheMode, + Endpoint: dockerS3CacheEndpoint, + } + log.WithFields(log.Fields{ + "bucket": dockerS3Cache.Bucket, + "region": dockerS3Cache.Region, + "prefix": dockerS3Cache.Prefix, + }).Info("Docker S3 build cache enabled") + } + return []leeway.BuildOption{ leeway.WithLocalCache(localCache), leeway.WithRemoteCache(remoteCache), @@ -430,6 +460,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) { leeway.WithInFlightChecksums(inFlightChecksums), leeway.WithDockerExportToCache(dockerExportToCache, dockerExportSet), leeway.WithDockerExportEnv(dockerExportEnvValue, dockerExportEnvSet), + leeway.WithDockerS3Cache(dockerS3Cache), }, localCache } diff --git a/pkg/leeway/build.go b/pkg/leeway/build.go index 4153352b..0233cbdf 100644 --- a/pkg/leeway/build.go +++ b/pkg/leeway/build.go @@ -102,6 +102,21 @@ const ( // EnvvarWorkspaceRoot names the environment variable for workspace root path EnvvarWorkspaceRoot = "LEEWAY_WORKSPACE_ROOT" + // EnvvarDockerS3CacheBucket names the S3 bucket for Docker build layer caching + EnvvarDockerS3CacheBucket = "LEEWAY_DOCKER_S3_CACHE_BUCKET" + + // EnvvarDockerS3CacheRegion names the AWS region for the S3 cache bucket + EnvvarDockerS3CacheRegion = "LEEWAY_DOCKER_S3_CACHE_REGION" + + // EnvvarDockerS3CachePrefix names the prefix for S3 cache keys (optional) + EnvvarDockerS3CachePrefix = "LEEWAY_DOCKER_S3_CACHE_PREFIX" + + // EnvvarDockerS3CacheMode controls the cache mode (min or max, defaults to max) + EnvvarDockerS3CacheMode = "LEEWAY_DOCKER_S3_CACHE_MODE" + + // EnvvarDockerS3CacheEndpoint allows specifying a custom S3 endpoint (for S3-compatible storage) + EnvvarDockerS3CacheEndpoint = "LEEWAY_DOCKER_S3_CACHE_ENDPOINT" + // dockerImageNamesFiles is the name of the file store in poushed Docker build artifacts // which contains the names of the Docker images we just pushed dockerImageNamesFiles = "imgnames.txt" @@ -492,12 +507,70 @@ type buildOptions struct { DockerExportEnvValue bool // Value from explicit user env var DockerExportEnvSet bool // Whether user explicitly set env var (before workspace) + // Docker S3 build cache configuration + DockerS3Cache *DockerS3CacheConfig + context *buildContext } // DockerBuildOptions are options passed to "docker build" type DockerBuildOptions map[string]string +// DockerS3CacheConfig configures S3-backed Docker build layer caching. +// When configured, leeway adds --cache-from and --cache-to flags to docker buildx commands. +type DockerS3CacheConfig struct { + // Bucket is the S3 bucket name (required) + Bucket string + // Region is the AWS region for the bucket (required) + Region string + // Prefix is an optional prefix for cache keys (e.g., "cache/myproject/") + Prefix string + // Mode controls cache export mode: "min" (default layers only) or "max" (all layers) + // Defaults to "max" for better cache hit rates + Mode string + // Endpoint is an optional custom S3 endpoint for S3-compatible storage + Endpoint string +} + +// IsEnabled returns true if the S3 cache is properly configured +func (c *DockerS3CacheConfig) IsEnabled() bool { + return c != nil && c.Bucket != "" && c.Region != "" +} + +// CacheFromArg returns the --cache-from argument value for docker buildx +func (c *DockerS3CacheConfig) CacheFromArg() string { + if !c.IsEnabled() { + return "" + } + arg := fmt.Sprintf("type=s3,region=%s,bucket=%s", c.Region, c.Bucket) + if c.Prefix != "" { + arg += fmt.Sprintf(",blobs_prefix=%s,manifests_prefix=%s", c.Prefix, c.Prefix) + } + if c.Endpoint != "" { + arg += fmt.Sprintf(",endpoint_url=%s", c.Endpoint) + } + return arg +} + +// CacheToArg returns the --cache-to argument value for docker buildx +func (c *DockerS3CacheConfig) CacheToArg() string { + if !c.IsEnabled() { + return "" + } + mode := c.Mode + if mode == "" { + mode = "max" + } + arg := fmt.Sprintf("type=s3,region=%s,bucket=%s,mode=%s", c.Region, c.Bucket, mode) + if c.Prefix != "" { + arg += fmt.Sprintf(",blobs_prefix=%s,manifests_prefix=%s", c.Prefix, c.Prefix) + } + if c.Endpoint != "" { + arg += fmt.Sprintf(",endpoint_url=%s", c.Endpoint) + } + return arg +} + // BuildOption configures the build behaviour type BuildOption func(*buildOptions) error @@ -640,6 +713,33 @@ func WithDockerExportEnv(value, isSet bool) BuildOption { } } +// WithDockerS3Cache configures S3-backed Docker build layer caching +func WithDockerS3Cache(cfg *DockerS3CacheConfig) BuildOption { + return func(opts *buildOptions) error { + opts.DockerS3Cache = cfg + return nil + } +} + +// DockerS3CacheFromEnv creates a DockerS3CacheConfig from environment variables. +// Returns nil if the required environment variables are not set. +func DockerS3CacheFromEnv() *DockerS3CacheConfig { + bucket := os.Getenv(EnvvarDockerS3CacheBucket) + region := os.Getenv(EnvvarDockerS3CacheRegion) + + if bucket == "" || region == "" { + return nil + } + + return &DockerS3CacheConfig{ + Bucket: bucket, + Region: region, + Prefix: os.Getenv(EnvvarDockerS3CachePrefix), + Mode: os.Getenv(EnvvarDockerS3CacheMode), + Endpoint: os.Getenv(EnvvarDockerS3CacheEndpoint), + } +} + func withBuildContext(ctx *buildContext) BuildOption { return func(opts *buildOptions) error { opts.context = ctx @@ -2380,7 +2480,11 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p return nil, err } - // Use buildx for OCI layout export when exporting to cache + // Determine if we need buildx (required for OCI export or S3 cache) + useS3Cache := buildctx.DockerS3Cache.IsEnabled() + useBuildx := *cfg.ExportToCache || useS3Cache + + // Use buildx for OCI layout export when exporting to cache, or when S3 cache is enabled var buildcmd []string if *cfg.ExportToCache { // Build with OCI layout export for deterministic caching @@ -2388,11 +2492,31 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p buildcmd = []string{"docker", "buildx", "build", "--pull"} buildcmd = append(buildcmd, "--output", fmt.Sprintf("type=oci,dest=%s", imageTarPath)) buildcmd = append(buildcmd, "--tag", version) + } else if useBuildx { + // Use buildx for S3 cache support, but load to daemon for pushing + buildcmd = []string{"docker", "buildx", "build", "--pull", "--load", "-t", version} } else { // Normal build (load to daemon for pushing) buildcmd = []string{"docker", "build", "--pull", "-t", version} } + // Add S3 cache options if configured (only works with buildx) + if useS3Cache { + cacheFrom := buildctx.DockerS3Cache.CacheFromArg() + cacheTo := buildctx.DockerS3Cache.CacheToArg() + if cacheFrom != "" { + buildcmd = append(buildcmd, "--cache-from", cacheFrom) + } + if cacheTo != "" { + buildcmd = append(buildcmd, "--cache-to", cacheTo) + } + log.WithFields(log.Fields{ + "bucket": buildctx.DockerS3Cache.Bucket, + "region": buildctx.DockerS3Cache.Region, + "prefix": buildctx.DockerS3Cache.Prefix, + }).Debug("Docker S3 build cache enabled") + } + for arg, val := range cfg.BuildArgs { buildcmd = append(buildcmd, "--build-arg", fmt.Sprintf("%s=%s", arg, val)) } diff --git a/pkg/leeway/build_internal_test.go b/pkg/leeway/build_internal_test.go index 07e78f50..f6c323c3 100644 --- a/pkg/leeway/build_internal_test.go +++ b/pkg/leeway/build_internal_test.go @@ -333,3 +333,223 @@ func TestYarnAppExtraction_ScopedPackage(t *testing.T) { }) } } + +func TestDockerS3CacheConfig(t *testing.T) { + tests := []struct { + name string + config *DockerS3CacheConfig + wantEnabled bool + wantCacheFrom string + wantCacheTo string + }{ + { + name: "nil config", + config: nil, + wantEnabled: false, + }, + { + name: "empty config", + config: &DockerS3CacheConfig{}, + wantEnabled: false, + }, + { + name: "missing region", + config: &DockerS3CacheConfig{ + Bucket: "my-bucket", + }, + wantEnabled: false, + }, + { + name: "missing bucket", + config: &DockerS3CacheConfig{ + Region: "us-east-1", + }, + wantEnabled: false, + }, + { + name: "minimal config", + config: &DockerS3CacheConfig{ + Bucket: "my-bucket", + Region: "us-east-1", + }, + wantEnabled: true, + wantCacheFrom: "type=s3,region=us-east-1,bucket=my-bucket", + wantCacheTo: "type=s3,region=us-east-1,bucket=my-bucket,mode=max", + }, + { + name: "with prefix", + config: &DockerS3CacheConfig{ + Bucket: "my-bucket", + Region: "eu-west-1", + Prefix: "cache/myproject/", + }, + wantEnabled: true, + wantCacheFrom: "type=s3,region=eu-west-1,bucket=my-bucket,blobs_prefix=cache/myproject/,manifests_prefix=cache/myproject/", + wantCacheTo: "type=s3,region=eu-west-1,bucket=my-bucket,mode=max,blobs_prefix=cache/myproject/,manifests_prefix=cache/myproject/", + }, + { + name: "with mode min", + config: &DockerS3CacheConfig{ + Bucket: "my-bucket", + Region: "us-west-2", + Mode: "min", + }, + wantEnabled: true, + wantCacheFrom: "type=s3,region=us-west-2,bucket=my-bucket", + wantCacheTo: "type=s3,region=us-west-2,bucket=my-bucket,mode=min", + }, + { + name: "with custom endpoint", + config: &DockerS3CacheConfig{ + Bucket: "my-bucket", + Region: "us-east-1", + Endpoint: "https://minio.example.com", + }, + wantEnabled: true, + wantCacheFrom: "type=s3,region=us-east-1,bucket=my-bucket,endpoint_url=https://minio.example.com", + wantCacheTo: "type=s3,region=us-east-1,bucket=my-bucket,mode=max,endpoint_url=https://minio.example.com", + }, + { + name: "full config", + config: &DockerS3CacheConfig{ + Bucket: "my-bucket", + Region: "ap-southeast-1", + Prefix: "docker/", + Mode: "max", + Endpoint: "https://s3.custom.com", + }, + wantEnabled: true, + wantCacheFrom: "type=s3,region=ap-southeast-1,bucket=my-bucket,blobs_prefix=docker/,manifests_prefix=docker/,endpoint_url=https://s3.custom.com", + wantCacheTo: "type=s3,region=ap-southeast-1,bucket=my-bucket,mode=max,blobs_prefix=docker/,manifests_prefix=docker/,endpoint_url=https://s3.custom.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotEnabled := tt.config.IsEnabled() + if gotEnabled != tt.wantEnabled { + t.Errorf("IsEnabled() = %v, want %v", gotEnabled, tt.wantEnabled) + } + + if tt.wantEnabled { + gotCacheFrom := tt.config.CacheFromArg() + if gotCacheFrom != tt.wantCacheFrom { + t.Errorf("CacheFromArg() = %q, want %q", gotCacheFrom, tt.wantCacheFrom) + } + + gotCacheTo := tt.config.CacheToArg() + if gotCacheTo != tt.wantCacheTo { + t.Errorf("CacheToArg() = %q, want %q", gotCacheTo, tt.wantCacheTo) + } + } + }) + } +} + +func TestDockerS3CacheFromEnv(t *testing.T) { + // Save original env vars + origBucket := os.Getenv(EnvvarDockerS3CacheBucket) + origRegion := os.Getenv(EnvvarDockerS3CacheRegion) + origPrefix := os.Getenv(EnvvarDockerS3CachePrefix) + origMode := os.Getenv(EnvvarDockerS3CacheMode) + origEndpoint := os.Getenv(EnvvarDockerS3CacheEndpoint) + + // Restore env vars after test + defer func() { + os.Setenv(EnvvarDockerS3CacheBucket, origBucket) + os.Setenv(EnvvarDockerS3CacheRegion, origRegion) + os.Setenv(EnvvarDockerS3CachePrefix, origPrefix) + os.Setenv(EnvvarDockerS3CacheMode, origMode) + os.Setenv(EnvvarDockerS3CacheEndpoint, origEndpoint) + }() + + tests := []struct { + name string + envVars map[string]string + wantNil bool + wantCfg *DockerS3CacheConfig + }{ + { + name: "no env vars set", + envVars: map[string]string{}, + wantNil: true, + }, + { + name: "only bucket set", + envVars: map[string]string{ + EnvvarDockerS3CacheBucket: "my-bucket", + }, + wantNil: true, + }, + { + name: "only region set", + envVars: map[string]string{ + EnvvarDockerS3CacheRegion: "us-east-1", + }, + wantNil: true, + }, + { + name: "bucket and region set", + envVars: map[string]string{ + EnvvarDockerS3CacheBucket: "my-bucket", + EnvvarDockerS3CacheRegion: "us-east-1", + }, + wantNil: false, + wantCfg: &DockerS3CacheConfig{ + Bucket: "my-bucket", + Region: "us-east-1", + }, + }, + { + name: "all env vars set", + envVars: map[string]string{ + EnvvarDockerS3CacheBucket: "my-bucket", + EnvvarDockerS3CacheRegion: "eu-west-1", + EnvvarDockerS3CachePrefix: "cache/", + EnvvarDockerS3CacheMode: "min", + EnvvarDockerS3CacheEndpoint: "https://minio.local", + }, + wantNil: false, + wantCfg: &DockerS3CacheConfig{ + Bucket: "my-bucket", + Region: "eu-west-1", + Prefix: "cache/", + Mode: "min", + Endpoint: "https://minio.local", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Clear all env vars first + os.Unsetenv(EnvvarDockerS3CacheBucket) + os.Unsetenv(EnvvarDockerS3CacheRegion) + os.Unsetenv(EnvvarDockerS3CachePrefix) + os.Unsetenv(EnvvarDockerS3CacheMode) + os.Unsetenv(EnvvarDockerS3CacheEndpoint) + + // Set test env vars + for k, v := range tt.envVars { + os.Setenv(k, v) + } + + got := DockerS3CacheFromEnv() + + if tt.wantNil { + if got != nil { + t.Errorf("DockerS3CacheFromEnv() = %+v, want nil", got) + } + return + } + + if got == nil { + t.Fatal("DockerS3CacheFromEnv() = nil, want non-nil") + } + + if diff := cmp.Diff(tt.wantCfg, got); diff != "" { + t.Errorf("DockerS3CacheFromEnv() mismatch (-want +got):\n%s", diff) + } + }) + } +}