Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion actions/build/scripts/constants.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ export const CONTENT_TYPES = {
".wasm": "application/wasm",
}
export const MAX_ENTRIES = 2_000
export const MAX_FILE_BYTES = 25 * 1024 * 1024
export const MAX_FILE_BYTES = 100 * 1024 * 1024
export const MAX_PATH_DEPTH = 20
export const MAX_SCREENSHOT_BYTES = 25 * 1024 * 1024
export const MAX_SCREENSHOT_HEIGHT = 16_384
Expand Down
34 changes: 33 additions & 1 deletion actions/build/scripts/validate-build-output/index.test.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import assert from "node:assert/strict"
import { readdir, symlink, writeFile } from "node:fs/promises"
import { readdir, symlink, truncate, writeFile } from "node:fs/promises"
import path from "node:path"
import { describe, it } from "node:test"

Expand Down Expand Up @@ -53,6 +53,38 @@ describe("validateBuildOutput()", () => {
})
})

it("accepts a 100 MiB index and rejects one additional byte", async () => {
const root = await buildRoot()
const index = path.join(root, "dist", "index.html")
const maximumBytes = 100 * 1024 * 1024
await truncate(index, maximumBytes)

assert.deepEqual(await validateBuildOutput(root), {
bytes: maximumBytes,
files: [{ path: "index.html", bytes: maximumBytes }],
})

await truncate(index, maximumBytes + 1)

await assert.rejects(
validateBuildOutput(root),
/Build output file must be no larger than 104857600 bytes: index\.html has 104857601 bytes\./,
)
})

it("retains the 100 MiB aggregate limit for individually allowed files", async () => {
const root = await buildRoot()
await writeFile(path.join(root, "dist", "index.html"), "hi")
const asset = path.join(root, "dist", "data.bin")
await writeFile(asset, "")
await truncate(asset, 100 * 1024 * 1024 - 1)

await assert.rejects(
validateBuildOutput(root),
/Build output files must total no more than 104857600 bytes; found 104857601\./,
)
})

it("rejects symbolic links", async () => {
const root = await buildRoot()
await symlink("index.html", path.join(root, "dist", "linked.html"))
Expand Down
2 changes: 1 addition & 1 deletion hub/docs/artifact-storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ The API's `/api/ready` includes configured storage. `isReady()` has a two-second

Keys are case-sensitive opaque strings of 1–128 ASCII letters, digits, underscores, or hyphens, beginning with a letter or digit. They are not paths or URLs. The domain rejects traversal, separators, percent encoding, and other characters before calling a provider.

Writes accept a `Uint8Array` of at most 32 MiB, including empty bytes, and never overwrite an existing key. Reads return a bounded `Uint8Array`. A successful operation returns `{ status: "ok" }`; a successful read also includes `bytes`. Each operation has its own result type: reads may fail with `invalid-key`, `too-large`, `not-found`, or `unavailable`; writes with `invalid-key`, `too-large`, `already-exists`, or `unavailable`; deletes with `invalid-key`, `not-found`, or `unavailable`. Missing reads and deletes are explicit `not-found` results, not empty bytes or silent success. Provider messages and credentials are not included in results.
Writes accept a `Uint8Array` of at most 100 MiB (104,857,600 bytes), including empty bytes, and never overwrite an existing key. Reads return a `Uint8Array` bounded by the same limit. A successful operation returns `{ status: "ok" }`; a successful read also includes `bytes`. Each operation has its own result type: reads may fail with `invalid-key`, `too-large`, `not-found`, or `unavailable`; writes with `invalid-key`, `too-large`, `already-exists`, or `unavailable`; deletes with `invalid-key`, `not-found`, or `unavailable`. Missing reads and deletes are explicit `not-found` results, not empty bytes or silent success. Provider messages and credentials are not included in results.

Immutability prevents overwriting bytes while a key exists. Deleting a key permits later reuse. Callers own authorization, key allocation, retention, and coordination between deletion and other work. Build integrations use opaque deterministic keys and persist `BuildArtifact.kind`, package-relative `path`, `storageKey`, and `byteLength`; preview serving derives the response content type from the checked package path.

Expand Down
4 changes: 4 additions & 0 deletions hub/docs/build-previews.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ Package paths remain relative and may not contain empty, dot, dot-dot, control,

Content types come from the checked package path, with unknown extensions served as `application/octet-stream`. HTML is private and revalidated. Immutable assets use a private one-year cache. Desktop and narrow screenshots are served from their package paths through the same Build authorization.

Preview reads share a 100 MiB source-byte allowance per API process across Build origins, private previews, template previews, and public links. Hub reserves each file's recorded size before reading storage and keeps that reservation until the read settles and the HTTP response finishes or disconnects. A request returns 503 without reading the file when the allowance is full. Small and empty files reserve at least 64 KiB each. Temporary template-manifest reads release their reservation before reading the selected HTML file.

This allowance bounds concurrent buffered preview files, not total process memory. Storage downloads can hold chunks and their combined buffer at once, and HTML bridge injection needs additional strings and encoded bytes. Imports and other runtime work need separate headroom within the deployment's memory allocation.

HTML responses inject the generated hosted preview bridge. The bridge installs the shared Controls engine and bindings, which load that Build's `/controls.replica.json`, and reports review element or area selections to the authenticated parent page. Review records store the anchor and Build identity separately; the bridge does not write artifacts or receive Hub session cookies. The generated bridge is checked by `npm run controls:assets:check`.

## Browser boundary
Expand Down
16 changes: 15 additions & 1 deletion hub/docs/github-actions-builds.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,21 @@ Deletion events are still recorded when received, rather than confirmed synchron

`replica-screenshots-v1` contains `screenshots/**` and `screenshots.json`, `{ "version": 1, "distDigest": "sha256:…" }`. `distDigest` is the SHA-256 of the sorted `path\0sha256\n` lines for every dist file and `templates.json`.

Both packages use the same path, size, and entry-count limits as local Builds.
Both packages use the same path, size, and entry-count limits as local Builds:

| Content | Limit |
| ----------------------------- | ------------------------- |
| Each static file | 100 MiB (104,857,600 bytes) |
| All uncompressed `dist/` files | 100 MiB |
| Each screenshot | 25 MiB |
| All extracted package files | 150 MiB |
| Downloaded ZIP | 160 MiB |

A separate data asset must leave room for `index.html` and the rest of the app within the total `dist/` budget. Package validation also enforces path, entry-count, and archive protections.

Hub buffers the downloaded ZIP and extracted files in memory; storage writes also copy the current file to preserve its bytes during the write. Keep the default on-demand worker concurrency of one for the 1 GiB Azure runtime. Additional concurrent imports need memory headroom. The [preview layer](build-previews.md#file-behavior) bounds buffered responses across concurrent HTTP requests independently of worker concurrency.

To publish larger files, run Hub with this limit and update every selected workflow's Replica publish, screenshots, and optional build actions to the same reviewed revision containing it. Older action pins retain their earlier limits. Updating Replica alone neither deploys Hub nor changes downstream workflow references.

## Screenshots

Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
import { randomUUID } from "node:crypto"
import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"
import { mkdir, readFile, rm, symlink, truncate, writeFile } from "node:fs/promises"
import { resolve } from "node:path"
import { describe, expect, it, onTestFinished, vi } from "vitest"

import { createAzureBlobArtifactStorage } from "../../../integrations/azure-blob/create-azure-blob-artifact-storage"

import { artifactMaxBytes, createArtifactStorage } from "."
import { createArtifactStorage } from "."

vi.mock("../../../integrations/azure-blob/create-azure-blob-artifact-storage", () => ({
createAzureBlobArtifactStorage: vi.fn(),
Expand Down Expand Up @@ -117,15 +117,17 @@ describe("createArtifactStorage()", () => {
await mkdir(directory)
onTestFinished(() => rm(directory, { recursive: true, force: true }))
const storage = await createArtifactStorage({ provider: "filesystem", directory })
const maximumBytes = 100 * 1024 * 1024

expect(await storage.write("limit", new Uint8Array(artifactMaxBytes))).toEqual({ status: "ok" })
expect(await storage.write("limit", new Uint8Array(maximumBytes))).toEqual({ status: "ok" })
const result = await storage.read("limit")
expect(result.status).toBe("ok")
if (result.status === "ok") expect(result.bytes.length).toBe(artifactMaxBytes)
expect(await storage.write("large", new Uint8Array(artifactMaxBytes + 1))).toEqual({
if (result.status === "ok") expect(result.bytes.length).toBe(maximumBytes)
expect(await storage.write("large", new Uint8Array(maximumBytes + 1))).toEqual({
status: "too-large",
})
await writeFile(`${directory}/large`, new Uint8Array(artifactMaxBytes + 1))
await writeFile(`${directory}/large`, "")
await truncate(`${directory}/large`, maximumBytes + 1)
expect(await storage.read("large")).toEqual({ status: "too-large" })
})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import { log } from "../../../integrations/pino/log/index.ts"
import { parseArtifactStorageSettings } from "../parse-artifact-storage-settings/index.ts"
import type { ArtifactStorage, ArtifactStorageSettings } from "./types.ts"

export const artifactMaxBytes = 32 * 1024 * 1024
export const artifactMaxBytes = 100 * 1024 * 1024
Comment thread
weastler marked this conversation as resolved.
Resolved
const keyPattern = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/

export async function createArtifactStorage(
Expand Down
2 changes: 1 addition & 1 deletion hub/server/domains/build-artifacts/constants.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
export const maximumBuildPackageEntries = 2_090
export const maximumDistFiles = 2_000
export const maximumDistFileBytes = 25 * 1024 * 1024
export const maximumDistFileBytes = 100 * 1024 * 1024
export const maximumDistPathDepth = 20
export const maximumScreenshotBytes = 25 * 1024 * 1024
// Matches the build action: dist output is capped separately from the whole package.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,14 @@ describe("extractDistPackage", () => {
}
}

it("rejects a static file one byte larger than 100 MiB", () => {
const archive = createZipArchive([
{ name: "dist/index.html", bytes: Buffer.alloc(100 * 1024 * 1024 + 1) },
])

expect(() => extractDistPackage(archive)).toThrow("dist/index.html exceeds its size limit.")
})

it("accepts a legacy package with full dist output and screenshots", () => {
const archive = createZipArchive([
...dist(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,12 @@ import { readBuildPreviewFile } from "."

describe("readBuildPreviewFile()", () => {
it("serves directory-style routes through the application entry point", async () => {
const bytes = new TextEncoder().encode("<h1>Checkout</h1>")
const findArtifact = vi.fn().mockImplementation((_buildId: string, path: string) => {
if (path === "dist/index.html") {
return Promise.resolve({
storageKey: "index-key",
byteLength: 15,
byteLength: bytes.byteLength,
})
}
return Promise.resolve(null)
Expand All @@ -26,7 +27,7 @@ describe("readBuildPreviewFile()", () => {
isReady: vi.fn(),
read: vi.fn().mockResolvedValue({
status: "ok",
bytes: new TextEncoder().encode("<h1>Checkout</h1>"),
bytes,
}),
write: vi.fn(),
delete: vi.fn(),
Expand All @@ -46,13 +47,6 @@ describe("readBuildPreviewFile()", () => {
})

it("serves dotted routes declared by the template manifest", async () => {
const findArtifact = vi.fn().mockImplementation((_buildId: string, path: string) => {
const artifacts = {
"templates.json": { storageKey: "manifest-key", byteLength: 166 },
"dist/index.html": { storageKey: "index-key", byteLength: 15 },
}
return Promise.resolve(artifacts[path as keyof typeof artifacts] ?? null)
})
const manifest = {
version: 1,
templates: [
Expand All @@ -65,6 +59,15 @@ describe("readBuildPreviewFile()", () => {
},
],
}
const manifestBytes = new TextEncoder().encode(JSON.stringify(manifest))
const htmlBytes = new TextEncoder().encode("<h1>Release notes</h1>")
const findArtifact = vi.fn().mockImplementation((_buildId: string, path: string) => {
const artifacts = {
"templates.json": { storageKey: "manifest-key", byteLength: manifestBytes.byteLength },
"dist/index.html": { storageKey: "index-key", byteLength: htmlBytes.byteLength },
}
return Promise.resolve(artifacts[path as keyof typeof artifacts] ?? null)
})

const result = await readBuildPreviewFile(
{
Expand All @@ -81,11 +84,11 @@ describe("readBuildPreviewFile()", () => {
storageKey === "manifest-key"
? {
status: "ok",
bytes: new TextEncoder().encode(JSON.stringify(manifest)),
bytes: manifestBytes,
}
: {
status: "ok",
bytes: new TextEncoder().encode("<h1>Release notes</h1>"),
bytes: htmlBytes,
},
),
),
Expand Down Expand Up @@ -129,6 +132,44 @@ describe("readBuildPreviewFile()", () => {
)
})

it.each(["asset", "template manifest"] as const)(
"rejects %s bytes that exceed the saved artifact size",
async (artifactKind) => {
const read = vi.fn().mockResolvedValue({
status: "ok",
bytes: new TextEncoder().encode(
artifactKind === "asset"
? "asset bytes"
: JSON.stringify({ templates: [{ path: "/release/v2.0" }] }),
),
})
const findArtifact = vi
.fn()
.mockImplementation((_buildId: string, artifactPath: string) =>
Promise.resolve(
artifactKind === "asset" || artifactPath === "templates.json"
? { storageKey: "underreported-key", byteLength: 1 }
: null,
),
)

const result = await readBuildPreviewFile(
{
acceptsHtml: artifactKind === "template manifest",
buildId: "00000000-0000-4000-8000-000000000001",
requestPath: artifactKind === "asset" ? "/asset.bin" : "/release/v2.0",
},
{
findArtifact,
storage: { isReady: vi.fn(), read, write: vi.fn(), delete: vi.fn() },
},
)

expect(result).toEqual({ status: "unavailable" })
expect(read).toHaveBeenCalledOnce()
},
)

it("decodes one URL path layer before artifact lookup", async () => {
const findArtifact = vi.fn().mockResolvedValue({
storageKey: "banner-key",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,13 +65,22 @@ export async function readBuildPreviewFile(
if (!artifact && input.acceptsHtml && !useSpaFallback) {
const manifest = await dependencies.findArtifact(input.buildId, "templates.json")
if (manifest) {
const storedManifest = await dependencies.storage.read(manifest.storageKey)
if (storedManifest.status !== "ok") {
return {
status: storedManifest.status === "not-found" ? "not-found" : "unavailable",
const release = dependencies.reserveBytes?.(manifest.byteLength)
if (release === null) return { status: "unavailable" }
try {
const storedManifest = await dependencies.storage.read(manifest.storageKey)
if (storedManifest.status !== "ok") {
return {
status: storedManifest.status === "not-found" ? "not-found" : "unavailable",
}
}
if (storedManifest.bytes.byteLength > manifest.byteLength) {
return { status: "unavailable" }
}
useSpaFallback = readDeclaredRoutePathnames(storedManifest.bytes).includes(requestPath)
} finally {
release?.()
}
useSpaFallback = readDeclaredRoutePathnames(storedManifest.bytes).includes(requestPath)
}
}
if (!artifact && useSpaFallback) {
Expand All @@ -80,10 +89,14 @@ export async function readBuildPreviewFile(
}
if (!artifact) return { status: "not-found" }

if (dependencies.reserveBytes?.(artifact.byteLength) === null) {
return { status: "unavailable" }
}
const stored = await dependencies.storage.read(artifact.storageKey)
if (stored.status !== "ok") {
return { status: stored.status === "not-found" ? "not-found" : "unavailable" }
}
if (stored.bytes.byteLength > artifact.byteLength) return { status: "unavailable" }

const contentType = contentTypeForBuildPath(path)
const html = contentType.startsWith("text/html")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export interface ReadBuildPreviewFileInput {

export interface ReadBuildPreviewFileDependencies {
storage: ArtifactStorage
reserveBytes?(byteLength: number): (() => void) | null
findArtifact(
buildId: string,
path: string,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { describe, expect, it } from "vitest"

import { reserveBuildPreviewBytes } from "."

describe("reserveBuildPreviewBytes()", () => {
it("bounds empty and small reads without making released capacity available twice", () => {
const remaining = reserveBuildPreviewBytes(100 * 1024 * 1024 - 64 * 1024)
const empty = reserveBuildPreviewBytes(0)
let small: (() => void) | null = null

try {
expect(remaining).toBeTypeOf("function")
expect(empty).toBeTypeOf("function")
expect(reserveBuildPreviewBytes(1)).toBeNull()

empty?.()
empty?.()
small = reserveBuildPreviewBytes(1)

expect(small).toBeTypeOf("function")
expect(reserveBuildPreviewBytes(1)).toBeNull()
} finally {
remaining?.()
empty?.()
small?.()
}
})
})
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
const maximumReservedBytes = 100 * 1024 * 1024
const minimumReservationBytes = 64 * 1024
let reservedBytes = 0

/** Shares the source-byte allowance across preview requests in this process. */
export function reserveBuildPreviewBytes(byteLength: number): (() => void) | null {
if (!Number.isSafeInteger(byteLength) || byteLength < 0) return null
const bytes = Math.max(byteLength, minimumReservationBytes)
if (bytes > maximumReservedBytes - reservedBytes) return null
reservedBytes += bytes
let released = false
return () => {
if (released) return
released = true
reservedBytes -= bytes
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ export async function storeGitHubBuildFiles(
const write = await storage.write(storageKey, file.bytes)
if (write.status === "already-exists") {
const existing = await storage.read(storageKey)
if (existing.status !== "ok" || !Buffer.from(existing.bytes).equals(file.bytes)) {
if (existing.status !== "ok" || !file.bytes.equals(existing.bytes)) {
throw new Error(`Artifact storage contains different bytes for ${file.path}.`)
}
} else if (write.status !== "ok") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export async function createFilesystemArtifactStorage(
length += bytesRead
}
if (length > stat.size) return { status: "unavailable" } as const
return { status: "ok", bytes: bytes.slice(0, length) } as const
return { status: "ok", bytes: bytes.subarray(0, length) } as const
} finally {
await handle.close()
}
Expand Down
Loading
Loading