feat(deployments): install deployed image base packages from a pinned Debian snapshot

.changeset/deterministic-apt-layer.md

@@ -0,0 +1,5 @@
+---
+"trigger.dev": patch
+---
+
+Deployed images now install base system packages from a pinned Debian snapshot archive, so pushed images typically share one identical package layer across projects and builds instead of a near-duplicate per project, speeding up image pulls. Base package versions are frozen at the pinned snapshot date and move forward with CLI releases. Set TRIGGER_BUILD_SKIP_APT_SNAPSHOT=1 to fall back to the live package archive.

docs/config/extensions/aptGet.mdx

@@ -32,3 +32,12 @@ export default defineConfig({
   },
 });
 ```
+
+<Note>
+  Packages install from a [Debian snapshot archive](https://snapshot.debian.org) pinned per CLI
+  version, so builds are reproducible and version pins keep working even after the live archive
+  moves on. This also means a pinned version must exist in that snapshot: a version published after
+  the CLI release's snapshot date won't resolve until you update the CLI. Set the
+  `TRIGGER_BUILD_SKIP_APT_SNAPSHOT=1` environment variable when running the deploy command to
+  install from the live Debian archive instead.
+</Note>

packages/cli-v3/src/deploy/buildImage.test.ts

@@ -1,5 +1,5 @@
 import type { BuildRuntime } from "@trigger.dev/core/v3/schemas";
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import { generateContainerfile } from "./buildImage.js";
 
 const nodeImages: Array<[BuildRuntime, string]> = [
@@ -74,4 +74,116 @@ describe("generateContainerfile", () => {
       expect(rmNodeModules).toBeGreaterThan(codeStage);
     }
   );
+
+  describe("apt snapshot pinning", () => {
+    afterEach(() => {
+      vi.unstubAllEnvs();
+    });
+
+    it.each(["node", "bun"] as BuildRuntime[])(
+      "pins apt to the Debian snapshot archive and scrubs timestamped files for %s",
+      async (runtime) => {
+        const containerfile = await generateContainerfile({
+          runtime,
+          build: {},
+          image: undefined,
+          indexScript: "index.js",
+          entrypoint: "entrypoint.js",
+        });
+
+        expect(containerfile).toMatch(
+          /deb \[check-valid-until=no signed-by=\S+\] http:\/\/snapshot\.debian\.org\/archive\/debian\/\d{8}T\d{6}Z bookworm main/
+        );
+        // codename guard so a future non-bookworm base pin fails with an actionable error
+        expect(containerfile).toContain('[ "$VERSION_CODENAME" = "bookworm" ]');
+        expect(containerfile).toContain("/archive/debian-security/");
+        expect(containerfile).toContain("rm -f /etc/apt/sources.list.d/debian.sources");
+        expect(containerfile).toContain("/var/log/dpkg.log");
+      }
+    );
+
+    it("keeps the default package layer identical for customized projects", async () => {
+      const containerfile = await generateContainerfile({
+        runtime: "node-22",
+        build: {},
+        image: {
+          pkgs: ["jq", "curl"],
+          instructions: ["RUN echo custom > /etc/marker"],
+        },
+        indexScript: "index.js",
+        entrypoint: "entrypoint.js",
+      });
+
+      const defaultInstall = containerfile.indexOf(
+        "apt-get install -y --no-install-recommends busybox ca-certificates dumb-init git openssl"
+      );
+      const instructions = containerfile.indexOf("RUN echo custom > /etc/marker");
+      // sorted, deduplicated, and separate from the default install line
+      const userInstall = containerfile.indexOf(
+        "apt-get install -y --no-install-recommends --allow-downgrades curl jq"
+      );
+
+      expect(defaultInstall).toBeGreaterThan(-1);
+      expect(instructions).toBeGreaterThan(defaultInstall);
+      expect(userInstall).toBeGreaterThan(instructions);
+      // the user install reuses the sources written by the default install
+      expect(containerfile.slice(userInstall)).not.toContain("snapshot.debian.org");
+    });
+
+    it("drops the snapshot pin when TRIGGER_BUILD_SKIP_APT_SNAPSHOT is set", async () => {
+      vi.stubEnv("TRIGGER_BUILD_SKIP_APT_SNAPSHOT", "1");
+
+      const containerfile = await generateContainerfile({
+        runtime: "node-22",
+        build: {},
+        image: undefined,
+        indexScript: "index.js",
+        entrypoint: "entrypoint.js",
+      });
+
+      expect(containerfile).not.toContain("snapshot.debian.org");
+      // the base-stage default install must still be present, from the live archive
+      expect(containerfile).toContain(
+        "apt-get install -y --no-install-recommends busybox ca-certificates dumb-init git openssl"
+      );
+    });
+
+    it("repairs dpkg state after instructions when there are no user packages", async () => {
+      const containerfile = await generateContainerfile({
+        runtime: "node-22",
+        build: {},
+        image: { instructions: ["RUN echo custom > /etc/marker"] },
+        indexScript: "index.js",
+        entrypoint: "entrypoint.js",
+      });
+
+      const instructions = containerfile.indexOf("RUN echo custom > /etc/marker");
+      const repair = containerfile.indexOf("apt-get --fix-broken install -y");
+
+      expect(instructions).toBeGreaterThan(-1);
+      expect(repair).toBeGreaterThan(instructions);
+    });
+
+    it("pins a snapshot no older than 90 days", async () => {
+      const containerfile = await generateContainerfile({
+        runtime: "node-22",
+        build: {},
+        image: undefined,
+        indexScript: "index.js",
+        entrypoint: "entrypoint.js",
+      });
+
+      const match = containerfile.match(/archive\/debian\/(\d{4})(\d{2})(\d{2})T/);
+      expect(match).not.toBeNull();
+
+      const [, year, month, day] = match!;
+      const snapshotAgeDays =
+        (Date.now() - Date.UTC(Number(year), Number(month) - 1, Number(day))) / 86_400_000;
+
+      expect(
+        snapshotAgeDays,
+        "DEBIAN_SNAPSHOT is stale; deployed images are missing recent Debian security updates. Bump it in buildImage.ts."
+      ).toBeLessThan(90);
+    });
+  });
 });

packages/cli-v3/src/deploy/buildImage.ts

@@ -700,6 +700,72 @@ const BASE_IMAGE: Record<BuildRuntime, string> = {
 
 const DEFAULT_PACKAGES = ["busybox", "ca-certificates", "dumb-init", "git", "openssl"];
 
+// Freezes the Debian archive so the package layer is a pure function of this
+// timestamp and the base image, making it byte-identical across projects.
+// INVARIANT: must be at or after the archive state every BASE_IMAGE was built
+// from, or apt hits unsatisfiable exact-version downgrades; bump it whenever
+// a base pin is bumped (this is also how packages pick up security updates).
+const DEBIAN_SNAPSHOT = "20260810T000000Z";
+
+// Must match the Debian release of every BASE_IMAGE; the generated Containerfile
+// asserts this at build time
+const DEBIAN_SUITE = "bookworm";
+
+// Files apt/dpkg write with wall-clock timestamps, which would break layer determinism
+const APT_SCRUB =
+  "rm -rf /var/lib/apt/lists/* /var/log/dpkg.log /var/log/apt /var/log/alternatives.log /var/cache/ldconfig/aux-cache /var/cache/debconf/*-old";
+
+function aptSourcesSetup(): string {
+  if (process.env.TRIGGER_BUILD_SKIP_APT_SNAPSHOT === "1") {
+    return "";
+  }
+
+  // check-valid-until=no: pinned Release files outlive their Valid-Until window.
+  // http, not https: the slim base images have no ca-certificates yet, and apt
+  // integrity comes from GPG-signed Release files rather than TLS.
+  const sourceOptions =
+    "[check-valid-until=no signed-by=/usr/share/keyrings/debian-archive-keyring.gpg]";
+  const sources = [
+    `deb ${sourceOptions} http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE} main`,
+    `deb ${sourceOptions} http://snapshot.debian.org/archive/debian-security/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-security main`,
+    `deb ${sourceOptions} http://snapshot.debian.org/archive/debian/${DEBIAN_SNAPSHOT} ${DEBIAN_SUITE}-updates main`,
+  ];
+
+  return `. /etc/os-release && [ "$VERSION_CODENAME" = "${DEBIAN_SUITE}" ] || { echo "Base image is Debian $VERSION_CODENAME but the CLI pins ${DEBIAN_SUITE} apt sources. Set TRIGGER_BUILD_SKIP_APT_SNAPSHOT=1 to use the live archive."; exit 1; } && \\
+  printf '%s\\n' ${sources.map((line) => `'${line}'`).join(" ")} > /etc/apt/sources.list && \\
+  rm -f /etc/apt/sources.list.d/debian.sources && \\
+  echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries && \\
+  `;
+}
+
+function aptInstall(packages: string[], { setupSources }: { setupSources: boolean }): string {
+  // fix-broken repairs dpkg state left by instructions (e.g. dpkg -i of a local
+  // .deb); pointless in the default install, which runs on a pristine base.
+  // --allow-downgrades: a user pin of a default package (e.g. openssl=<older>) is
+  // a downgrade by the time the user install runs on top of the default layer.
+  const repair = setupSources
+    ? ""
+    : `apt-get --fix-broken install -y && \\
+  `;
+  const installFlags = setupSources
+    ? "-y --no-install-recommends"
+    : "-y --no-install-recommends --allow-downgrades";
+
+  return `RUN ${setupSources ? aptSourcesSetup() : ""}apt-get update && \\
+  ${repair}apt-get install ${installFlags} ${packages.join(" ")} && \\
+  apt-get clean && \\
+  ${APT_SCRUB}`;
+}
+
+// Instructions can leave dpkg in a broken state that the user-packages install
+// would normally repair; when there are no user packages, repair explicitly
+function aptRepair(): string {
+  return `RUN apt-get update && \\
+  apt-get --fix-broken install -y && \\
+  apt-get clean && \\
+  ${APT_SCRUB}`;
+}
+
 export async function generateContainerfile(options: GenerateContainerfileOptions) {
   switch (options.runtime) {
     case "node":
@@ -726,36 +792,54 @@ const parseGenerateOptions = (options: GenerateContainerfileOptions) => {
   const postInstallCommands = (options.build.commands || []).map((cmd) => `RUN ${cmd}`).join("\n");
 
   const baseInstructions = (options.image?.instructions || []).join("\n");
-  const packages = Array.from(new Set(DEFAULT_PACKAGES.concat(options.image?.pkgs || []))).join(
-    " "
-  );
+
+  // Default packages install alone so their layer stays identical across
+  // projects; user packages and instructions only add layers on top
+  const defaultPackages = [...DEFAULT_PACKAGES].sort();
+  const userPackages = Array.from(new Set(options.image?.pkgs || []))
+    .filter((pkg) => !DEFAULT_PACKAGES.includes(pkg))
+    .sort();
+
+  const defaultPackagesInstall = aptInstall(defaultPackages, { setupSources: true });
+  const userPackagesInstall =
+    userPackages.length > 0
+      ? aptInstall(userPackages, { setupSources: false })
+      : baseInstructions.length > 0
+        ? aptRepair()
+        : "";
 
   return {
     baseImage: BASE_IMAGE[options.runtime],
     baseInstructions,
     buildArgs,
     buildEnvVars,
-    packages,
+    defaultPackagesInstall,
+    userPackagesInstall,
     postInstallCommands,
   };
 };
 
 async function generateBunContainerfile(options: GenerateContainerfileOptions) {
-  const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } =
-    parseGenerateOptions(options);
+  const {
+    baseImage,
+    buildArgs,
+    buildEnvVars,
+    postInstallCommands,
+    baseInstructions,
+    defaultPackagesInstall,
+    userPackagesInstall,
+  } = parseGenerateOptions(options);
 
   return `# syntax=docker/dockerfile:1
 # check=skip=SecretsUsedInArgOrEnv
 FROM ${baseImage} AS base
 
+ENV DEBIAN_FRONTEND=noninteractive
+${defaultPackagesInstall}
+
 ${baseInstructions}
 
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-  apt-get --fix-broken install -y && \
-  apt-get install -y --no-install-recommends ${packages} && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/*
+${userPackagesInstall}
 
 FROM base AS build
 
@@ -853,20 +937,26 @@ CMD []
 }
 
 async function generateNodeContainerfile(options: GenerateContainerfileOptions) {
-  const { baseImage, buildArgs, buildEnvVars, postInstallCommands, baseInstructions, packages } =
-    parseGenerateOptions(options);
+  const {
+    baseImage,
+    buildArgs,
+    buildEnvVars,
+    postInstallCommands,
+    baseInstructions,
+    defaultPackagesInstall,
+    userPackagesInstall,
+  } = parseGenerateOptions(options);
 
   return `# syntax=docker/dockerfile:1
 # check=skip=SecretsUsedInArgOrEnv
 FROM ${baseImage} AS base
 
+ENV DEBIAN_FRONTEND=noninteractive
+${defaultPackagesInstall}
+
 ${baseInstructions}
 
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-  apt-get --fix-broken install -y && \
-  apt-get install -y --no-install-recommends ${packages} && \
-  apt-get clean && rm -rf /var/lib/apt/lists/*
+${userPackagesInstall}
 
 FROM base AS build