From 6ba78f6aa5eb291d8406db9d331cf3adb09eb52f Mon Sep 17 00:00:00 2001 From: Wouter Verlaek Date: Tue, 27 Jan 2026 12:56:06 +0000 Subject: [PATCH] feat: add node_modules caching for yarn builds Add LEEWAY_NODE_MODULES_CACHE env var to cache node_modules between yarn package builds. Speeds up CI builds by avoiding repeated yarn install operations. Co-authored-by: Ona --- README.md | 49 +++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 3 +++ pkg/leeway/build.go | 34 +++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) diff --git a/README.md b/README.md index ff3c39c9..e540ee36 100644 --- a/README.md +++ b/README.md @@ -598,8 +598,57 @@ variables have an effect on leeway: - `LEEWAY_CACHE_DIR`: Location of the local build cache. The directory does not have to exist yet. - `LEEWAY_BUILD_DIR`: Working location of leeway (i.e. where the actual builds happen). This location will see heavy I/O which makes it advisable to place this on a fast SSD or in RAM. - `LEEWAY_YARN_MUTEX`: Configures the mutex flag leeway will pass to yarn. Defaults to "network". See https://yarnpkg.com/lang/en/docs/cli/#toc-concurrency-and-mutex for possible values. +- `LEEWAY_NODE_MODULES_CACHE`: Directory to cache node_modules between yarn builds. When set, leeway restores node_modules from cache before `yarn install` and saves it after. Cache is keyed by package name and yarn.lock hash. See [Caching node_modules in CI](#caching-node_modules-in-ci) for usage. - `LEEWAY_EXPERIMENTAL`: Enables experimental features +# Caching node_modules in CI + +Yarn package builds can be slow due to `yarn install` downloading and extracting dependencies. The `LEEWAY_NODE_MODULES_CACHE` environment variable enables caching of `node_modules` directories between builds. + +When enabled, leeway: +1. Before `yarn install`: Restores `node_modules` from cache if available +2. After `yarn install`: Saves `node_modules` to cache + +The cache is keyed by package name and the first 12 characters of the yarn.lock SHA256 hash, so it automatically invalidates when dependencies change. + +## GitHub Actions Example + +```yaml +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Cache node_modules + uses: actions/cache@v4 + with: + path: ~/.leeway-node-modules + key: node-modules-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + node-modules-${{ runner.os }}- + + - name: Build + env: + LEEWAY_NODE_MODULES_CACHE: ~/.leeway-node-modules + run: leeway build :my-yarn-package +``` + +## Performance Impact + +| Scenario | Typical yarn install time | +|----------|---------------------------| +| No cache (cold) | 60-180 seconds | +| With node_modules cache | <5 seconds | + +The cache directory structure is: +``` +$LEEWAY_NODE_MODULES_CACHE/ + component_package-name/ # package full name (slashes replaced with underscores) + a1b2c3d4e5f6/ # first 12 chars of yarn.lock hash + node_modules/ +``` + # OpenTelemetry Tracing Leeway supports distributed tracing using OpenTelemetry for build performance visibility. diff --git a/cmd/root.go b/cmd/root.go index 9d5e6088..b68ce993 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -129,6 +129,9 @@ variables have an effect on leeway: which makes it advisable to place this on a fast SSD or in RAM. LEEWAY_YARN_MUTEX Configures the mutex flag leeway will pass to yarn. Defaults to "network". See https://yarnpkg.com/lang/en/docs/cli/#toc-concurrency-and-mutex for possible values. + LEEWAY_NODE_MODULES_CACHE Directory to cache node_modules between yarn builds. When set, leeway restores + node_modules from cache before yarn install and saves it after. Cache is keyed by + package name and yarn.lock hash. Useful for CI to avoid repeated yarn installs. LEEWAY_DEFAULT_CACHE_LEVEL Sets the default cache level for builds. Defaults to "remote". LEEWAY_SLSA_CACHE_VERIFICATION Enables SLSA verification for cached artifacts (true/false). LEEWAY_SLSA_SOURCE_URI Expected source URI for SLSA verification (github.com/owner/repo). diff --git a/pkg/leeway/build.go b/pkg/leeway/build.go index 2c33e5b5..27962a1e 100644 --- a/pkg/leeway/build.go +++ b/pkg/leeway/build.go @@ -96,6 +96,11 @@ const ( // Defaults to "network". EnvvarYarnMutex = "LEEWAY_YARN_MUTEX" + // EnvvarNodeModulesCache specifies a directory to cache node_modules between builds. + // When set, leeway will restore node_modules from cache before yarn install and save + // it back after. Cache is keyed by package name and yarn.lock hash. + EnvvarNodeModulesCache = "LEEWAY_NODE_MODULES_CACHE" + // EnvvarDockerExportToCache controls whether Docker images are exported to cache instead of pushed directly EnvvarDockerExportToCache = "LEEWAY_DOCKER_EXPORT_TO_CACHE" @@ -1892,11 +1897,40 @@ func (p *Package) buildYarn(buildctx *buildContext, wd, result string) (bld *pac yarnMutex = "network" } yarnCache := filepath.Join(buildctx.BuildDir(), fmt.Sprintf("yarn-cache-%s", buildctx.buildID)) + + // node_modules caching: restore from cache before yarn install + nodeModulesCacheDir := os.Getenv(EnvvarNodeModulesCache) + var nodeModulesCachePath string + if nodeModulesCacheDir != "" { + // Compute cache key from yarn.lock hash + yarnLockPath := filepath.Join(wd, "yarn.lock") + yarnLockHash, err := computeSHA256(yarnLockPath) + if err != nil { + log.WithField("package", p.FullName()).WithError(err).Debug("cannot compute yarn.lock hash for node_modules cache") + } else { + // Use package name and yarn.lock hash as cache key + // Replace slashes in package name with underscores for filesystem safety + safePkgName := strings.ReplaceAll(p.FullName(), "/", "_") + nodeModulesCachePath = filepath.Join(nodeModulesCacheDir, safePkgName, yarnLockHash[:12]) + + // Restore node_modules from cache if it exists + restoreCmd := fmt.Sprintf("if [ -d \"%s/node_modules\" ]; then echo \"Restoring node_modules from cache...\"; cp -a \"%s/node_modules\" ./node_modules; fi", nodeModulesCachePath, nodeModulesCachePath) + commands[PackageBuildPhasePrep] = append(commands[PackageBuildPhasePrep], []string{"sh", "-c", restoreCmd}) + log.WithField("package", p.FullName()).WithField("cachePath", nodeModulesCachePath).Debug("node_modules cache enabled") + } + } + if len(cfg.Commands.Install) == 0 { commands[PackageBuildPhasePull] = append(commands[PackageBuildPhasePull], []string{"yarn", "install", "--frozen-lockfile", "--mutex", yarnMutex, "--cache-folder", yarnCache}) } else { commands[PackageBuildPhasePull] = append(commands[PackageBuildPhasePull], cfg.Commands.Install) } + + // node_modules caching: save to cache after yarn install + if nodeModulesCachePath != "" { + saveCmd := fmt.Sprintf("mkdir -p \"%s\" && rm -rf \"%s/node_modules\" && cp -a ./node_modules \"%s/node_modules\"", nodeModulesCachePath, nodeModulesCachePath, nodeModulesCachePath) + commands[PackageBuildPhasePull] = append(commands[PackageBuildPhasePull], []string{"sh", "-c", saveCmd}) + } if len(cfg.Commands.Build) == 0 { commands[PackageBuildPhaseBuild] = append(commands[PackageBuildPhaseBuild], []string{"yarn", "build"}) } else {