Skip to content

Commit b8ab8f4

Browse files
committed
chore(ci): qualify bounded compiler policy and current source guards
1 parent eac2190 commit b8ab8f4

7 files changed

Lines changed: 169 additions & 39 deletions

File tree

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
from pathlib import Path
2+
import subprocess,os,json
3+
p=Path('scripts/prune-turbopack-cache.ts');original=p.read_text();out=Path(os.environ['RUNNER_TEMP']);results=[]
4+
def case(name,old,new,pattern):
5+
assert original.count(old)==1,(name,old)
6+
try:
7+
p.write_text(original.replace(old,new))
8+
report=out/f'mutant-{name}.json'
9+
run=subprocess.run(['bun','run','test:scripts','scripts/prune-turbopack-cache.test.ts','-t',pattern,'--reporter=json','--outputFile.json='+str(report)])
10+
data=json.loads(report.read_text());assert run.returncode!=0 and data['numFailedTests']>0,(name,'mutation unexpectedly passed')
11+
results.append({'name':name,'failed':data['numFailedTests']})
12+
finally:p.write_text(original)
13+
case('mode',"const BUILD_CACHE = process.argv.includes('--build')","const BUILD_CACHE = false",'clears only build contents|does not substitute dev')
14+
case('retention','if (!force && gb <= maxGb)','if (false)','retains useful build state')
15+
case('root',"if (root.isSymbolicLink() || !root.isDirectory())","if (false)",'refuses a symlink')
16+
case('failure','if (BUILD_CACHE) process.exitCode = 1','if (false) process.exitCode = 1','refuses a symlink|rejects a non-directory ancestor')
17+
case('external',"if (entry.isDirectory()) {","if (entry.isDirectory() || entry.isSymbolicLink()) {",'retains useful build state')
18+
case('mount',"for await (const entry of await opendir(CACHE_DIR))\n await rm(path.join(CACHE_DIR, entry.name), { recursive: true, force: true })","await rm(CACHE_DIR, { recursive: true, force: true })",'clears only build contents')
19+
(out/'mutant-summary.json').write_text(json.dumps(results,indent=2));print(json.dumps(results))

‎.github/workflows/ci-cache-policy-proof.yml‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,17 @@ jobs:
88
lifecycle:
99
runs-on: blacksmith-2vcpu-ubuntu-2404
1010
env:
11-
LIFECYCLE_PHASE: cancel
11+
LIFECYCLE_PHASE: verify-rejection
1212
steps:
1313
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
1414
- uses: ./.github/actions/cache
1515
with:
1616
provider: blacksmith
1717
key: ${{ github.repository }}-compiler-requalification-44040218009
1818
path: ./apps/sim/.next/cache
19+
- uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6
20+
with:
21+
bun-version: 1.4.4
1922
- name: Qualify lifecycle across completed workflow runs
2023
run: |
2124
python3 - <<'PYCODE'
@@ -53,6 +56,15 @@ jobs:
5356
report['entriesAfter']=len(list(p.iterdir()))
5457
Path(os.environ['RUNNER_TEMP'],'cross-run-lifecycle.json').write_text(json.dumps(report,indent=2))
5558
PYCODE
59+
- name: Bound actual mounted working state without removing its root
60+
working-directory: apps/sim
61+
env:
62+
SIM_TURBOPACK_CACHE_MAX_GB: '0.000001'
63+
run: |
64+
before=$(stat -c '%i' .next/cache)
65+
bun run ../../scripts/prune-turbopack-cache.ts --build
66+
test "$before" = "$(stat -c '%i' .next/cache)"
67+
test -z "$(find .next/cache -mindepth 1 -print -quit)"
5668
- name: Hold partial producer for controlled cancellation
5769
if: env.LIFECYCLE_PHASE == 'cancel'
5870
run: |
@@ -71,3 +83,38 @@ jobs:
7183
with:
7284
name: cross-run-lifecycle
7385
path: ${{ runner.temp }}/cross-run-lifecycle.json
86+
qualify:
87+
runs-on: ubuntu-24.04
88+
timeout-minutes: 20
89+
env:
90+
NEXT_PUBLIC_APP_URL: http://localhost:3000
91+
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/sim_test
92+
NEXT_BUILD_SKIP_TYPE_CHECK: 'true'
93+
NEXT_BUILD_SKIP_SOURCE_MAPS: 'true'
94+
steps:
95+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
96+
- uses: ./.github/actions/setup
97+
with:
98+
provider: github
99+
- name: Format and lint the repository
100+
run: bun run lint
101+
- name: Prove real config and flag hash boundaries
102+
run: python3 .github/scripts/compiler-config-proof.py
103+
- name: Prove real filesystem CLI contracts
104+
run: bun run test:scripts scripts/prune-turbopack-cache.test.ts --reporter=default --reporter=json --outputFile.json="$RUNNER_TEMP/cache-policy-tests.json"
105+
- name: Revert individual ownership guards and require red
106+
run: python3 .github/scripts/compiler-policy-mutants.py
107+
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9
108+
if: always()
109+
with:
110+
name: compiler-policy-qualified
111+
path: |
112+
${{ runner.temp }}/compiler-config-report.json
113+
${{ runner.temp }}/cache-policy-tests.json
114+
${{ runner.temp }}/mutant-*.json
115+
${{ runner.temp }}/mutant-summary.json
116+
scripts/prune-turbopack-cache.ts
117+
scripts/prune-turbopack-cache.test.ts
118+
apps/sim/lib/core/config/env.ts
119+
apps/sim/next.config.ts
120+
apps/sim/turbo.json

‎apps/sim/lib/core/config/env.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -347,6 +347,7 @@ export const env = createEnv({
347347
// Infrastructure & Deployment
348348
NEXT_RUNTIME: z.string().optional(), // Next.js runtime environment
349349
DOCKER_BUILD: z.boolean().optional(), // Flag indicating Docker build environment
350+
NEXT_BUILD_PERSISTENT_CACHE: z.boolean().optional(), // Enables incremental compiler state for qualified CI builds
350351
NEXT_BUILD_SKIP_SOURCE_MAPS: z.boolean().optional(), // Omits unused browser source maps in acceptance builds
351352
NEXT_BUILD_SKIP_TYPE_CHECK: z.boolean().optional(), // Skips `next build`'s TypeScript pass where another CI job type-checks
352353

‎apps/sim/next.config.ts‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -203,18 +203,11 @@ const nextConfig: NextConfig = {
203203
*/
204204
turbopackFileSystemCacheForDev: true,
205205
/**
206-
* Turbopack's persistent build cache (beta) stays off — it is a net loss at
207-
* this app's size. A controlled A/B on a byte-identical module graph (PR
208-
* #6078) measured compile at 113s with it off, 162s cold with it on, and 360s
209-
* warm: the cache made the same build 3.2x slower. It also grew 5.1 GB ->
210-
* 12 GB across two runs of an unchanged tree, so a cache degrades the longer
211-
* it lives. Restoring across commits is separately undocumented-as-supported
212-
* (vercel/next.js#87283 reports stale HTML from a cache built elsewhere).
213-
*
214-
* The explicit pin is load-bearing: 16.3.0 flipped this default to true for
215-
* stable (vercel/next.js#94616), so dropping it re-enables the slower cache.
206+
* Only qualified CI builds opt into persistent compiler state. Other builds
207+
* remain cache-off until their own memory, invalidation and restore costs are
208+
* qualified. CI keeps this state separate from complete build-output caches.
216209
*/
217-
turbopackFileSystemCacheForBuild: false,
210+
turbopackFileSystemCacheForBuild: isTruthy(env.NEXT_BUILD_PERSISTENT_CACHE),
218211
/**
219212
* TypeScript 7 ships no JavaScript compiler API until 7.1, so Next's default
220213
* checker cannot load it — this shells out to the project-local `tsc` instead.

‎apps/sim/turbo.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@
207207
]
208208
},
209209
"build": {
210-
"env": ["NEXT_BUILD_SKIP_SOURCE_MAPS", "NEXT_BUILD_SKIP_TYPE_CHECK"],
210+
"env": ["NEXT_BUILD_PERSISTENT_CACHE", "NEXT_BUILD_SKIP_SOURCE_MAPS", "NEXT_BUILD_SKIP_TYPE_CHECK"],
211211
"inputs": [
212212
"$TURBO_DEFAULT$",
213213
".env*",

‎scripts/prune-turbopack-cache.test.ts‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import { spawnSync } from 'node:child_process'
22
import {
33
existsSync,
4+
lstatSync,
45
mkdirSync,
56
mkdtempSync,
67
readdirSync,
8+
readFileSync,
79
rmSync,
810
statSync,
911
symlinkSync,
@@ -56,13 +58,37 @@ describe('compiler cache maintenance through the real CLI', () => {
5658
expect(existsSync(join(outside, 'state'))).toBe(true)
5759
})
5860

61+
it('retains useful build state without counting external symlink targets', () => {
62+
const { root, build, dev, outside } = fixture()
63+
const inode = statSync(build).ino
64+
const buildBytes = readFileSync(join(build, 'state'))
65+
const devBytes = readFileSync(join(dev, 'state'))
66+
const outsideBytes = readFileSync(join(outside, 'state'))
67+
expect(run(root, ['--build'], '0.000006').status).toBe(0)
68+
expect(statSync(build).ino).toBe(inode)
69+
expect(readFileSync(join(build, 'state'))).toEqual(buildBytes)
70+
expect(readFileSync(join(dev, 'state'))).toEqual(devBytes)
71+
expect(readFileSync(join(outside, 'state'))).toEqual(outsideBytes)
72+
})
73+
5974
it('does not substitute dev state when the build cache is absent', () => {
6075
const { root, build, dev } = fixture()
6176
rmSync(build, { recursive: true })
6277
expect(run(root, ['--build']).status).toBe(0)
6378
expect(existsSync(join(dev, 'state'))).toBe(true)
6479
})
6580

81+
it('rejects a non-directory ancestor instead of treating it as an absent build cache', () => {
82+
const { root, outside } = fixture()
83+
const parent = join(root, '.next')
84+
rmSync(parent, { recursive: true })
85+
writeFileSync(parent, 'unrelated parent file')
86+
const result = run(root, ['--build'])
87+
expect(result.status, result.stderr).not.toBe(0)
88+
expect(readFileSync(parent, 'utf8')).toBe('unrelated parent file')
89+
expect(existsSync(join(outside, 'state'))).toBe(true)
90+
})
91+
6692
it('keeps the existing dev command scoped to dev state', () => {
6793
const { root, build, dev, outside } = fixture()
6894
expect(run(root, []).status).toBe(0)
@@ -78,13 +104,15 @@ describe('compiler cache maintenance through the real CLI', () => {
78104
expect(existsSync(join(dev, 'state'))).toBe(true)
79105
})
80106

81-
it('refuses a symlink build root without deleting its target', () => {
107+
it.each([true, false])('refuses a symlink build root when target existence is %s', (targetExists) => {
82108
const { root, build, dev, outside } = fixture()
83109
rmSync(build, { recursive: true })
84110
symlinkSync(outside, build, process.platform === 'win32' ? 'junction' : 'dir')
111+
if (!targetExists) rmSync(outside, { recursive: true })
85112
const result = run(root, ['--build', '--force'])
86113
expect(result.status, result.stderr).not.toBe(0)
87-
expect(existsSync(join(outside, 'state'))).toBe(true)
114+
expect(lstatSync(build).isSymbolicLink()).toBe(true)
115+
expect(existsSync(join(outside, 'state'))).toBe(targetExists)
88116
expect(existsSync(join(dev, 'state'))).toBe(true)
89117
})
90118
})

‎scripts/prune-turbopack-cache.ts‎

Lines changed: 66 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,16 @@
11
#!/usr/bin/env bun
22
/**
3-
* Caps the size of Turbopack's dev filesystem cache.
3+
* Caps Turbopack's dev and CI build filesystem caches.
44
*
55
* `experimental.turbopackFileSystemCacheForDev` is ON (see `apps/sim/next.config.ts`)
66
* because it makes dev-server restarts 5.4x faster and cuts RSS ~1.9x. The tradeoff
7-
* is that the cache is an append-heavy LSM store with no upstream GC: it grows with
7+
* is that the cache is an append-heavy LSM store without automatic GC: it grows with
88
* every route compiled and every code change, and it is never pruned. An abandoned
99
* cache in this repo reached 78 GB across 1,848 SST files.
1010
*
1111
* A stale cache is also slower to read back, so an unbounded one eventually erodes
1212
* the win it exists to provide. This script drops the whole cache once it crosses a
13-
* threshold; Turbopack rebuilds it on the next run (one cold compile, ~32s).
13+
* threshold; Turbopack rebuilds it on the next run (one cold dev compile, ~32s).
1414
*
1515
* Also the remedy when Turbopack aborts with `Cache corruption detected:
1616
* checksum mismatch` — it does not self-heal from a corrupted cache, so prune
@@ -21,6 +21,11 @@
2121
* bun run scripts/prune-turbopack-cache.ts # prune if over the cap
2222
* bun run scripts/prune-turbopack-cache.ts --force # always prune
2323
* bun run scripts/prune-turbopack-cache.ts --dry-run # report only
24+
* bun run scripts/prune-turbopack-cache.ts --build # production cache, only between builds
25+
*
26+
* Build mode retains the mounted cache root and fails maintenance errors; dev
27+
* maintenance stays nonblocking. Size checks stream entries and stop once the
28+
* build cache exceeds the cap.
2429
*
2530
* Override the cap with `SIM_TURBOPACK_CACHE_MAX_GB` (default 20).
2631
*
@@ -39,32 +44,49 @@
3944
* `kill -9`. Turbopack discards a partially-written cache and rebuilds silently,
4045
* so a hard kill makes a real cache win read as no win at all.
4146
*/
42-
import { existsSync, statSync } from 'node:fs'
43-
import { readdir, rm, stat } from 'node:fs/promises'
47+
import { existsSync, lstatSync, statSync } from 'node:fs'
48+
import { opendir, rm, stat } from 'node:fs/promises'
4449
import path from 'node:path'
4550

4651
const DEFAULT_MAX_GB = 20
4752
const BYTES_PER_GB = 1024 ** 3
53+
const BUILD_CACHE = process.argv.includes('--build')
4854

4955
/**
50-
* Resolved from the working directory, not hardcoded to one app: every Next app
51-
* in the monorepo has its own dev cache (`apps/docs` runs `next dev` too and, with
52-
* no override, uses the Next default where the cache is on). Each app caps its
53-
* own, so one app's dev start never prunes a cache another app is holding open.
56+
* Each app owns its cache under its working directory. Build maintenance selects
57+
* a separate root, so it cannot prune a live dev server's state.
5458
*/
55-
const CACHE_DIR = path.resolve(process.cwd(), '.next', 'dev', 'cache', 'turbopack')
59+
const CACHE_DIR = path.resolve(
60+
process.cwd(),
61+
BUILD_CACHE ? '.next/cache' : '.next/dev/cache/turbopack'
62+
)
63+
64+
function isMissingPath(error: unknown): boolean {
65+
return error instanceof Error && 'code' in error && error.code === 'ENOENT'
66+
}
5667

57-
/** Sums the size of every regular file under `dir`, skipping symlinks. */
58-
async function directorySize(dir: string): Promise<number> {
68+
/** Counts regular files without following symlinks or retaining the tree in memory. */
69+
async function directorySize(dir: string, limit: number): Promise<number> {
5970
let total = 0
60-
const entries = await readdir(dir, { withFileTypes: true, recursive: true })
61-
for (const entry of entries) {
62-
if (!entry.isFile()) continue
63-
try {
64-
total += (await stat(path.join(entry.parentPath, entry.name))).size
65-
} catch {
66-
// Turbopack compacts while we walk; a file vanishing mid-scan is expected.
71+
let entries: Awaited<ReturnType<typeof opendir>>
72+
try {
73+
entries = await opendir(dir)
74+
} catch (error) {
75+
if (BUILD_CACHE && isMissingPath(error)) return 0
76+
throw error
77+
}
78+
for await (const entry of entries) {
79+
const entryPath = path.join(dir, entry.name)
80+
if (entry.isDirectory()) {
81+
total += await directorySize(entryPath, limit - total)
82+
} else if (entry.isFile()) {
83+
try {
84+
total += (await stat(entryPath)).size
85+
} catch (error) {
86+
if (BUILD_CACHE && !isMissingPath(error)) throw error
87+
}
6788
}
89+
if (total > limit) break
6890
}
6991
return total
7092
}
@@ -86,12 +108,27 @@ async function main() {
86108
const force = process.argv.includes('--force')
87109
const dryRun = process.argv.includes('--dry-run')
88110

89-
if (!existsSync(CACHE_DIR) || !statSync(CACHE_DIR).isDirectory()) return
111+
if (BUILD_CACHE) {
112+
let root: ReturnType<typeof lstatSync>
113+
try {
114+
root = lstatSync(CACHE_DIR)
115+
} catch (error) {
116+
if (isMissingPath(error)) return
117+
throw error
118+
}
119+
if (root.isSymbolicLink() || !root.isDirectory())
120+
throw new Error('Build cache root must be a directory without symlinks')
121+
} else if (!existsSync(CACHE_DIR) || !statSync(CACHE_DIR).isDirectory()) {
122+
return
123+
}
90124

91125
const maxGb = parseMaxGb()
92-
const bytes = await directorySize(CACHE_DIR)
126+
const limit = BUILD_CACHE ? maxGb * BYTES_PER_GB : Number.POSITIVE_INFINITY
127+
const bytes = force ? 0 : await directorySize(CACHE_DIR, limit)
93128
const gb = bytes / BYTES_PER_GB
94-
const sizeLabel = `${gb.toFixed(1)} GB`
129+
const sizeLabel = force
130+
? 'cache'
131+
: `${BUILD_CACHE && bytes > limit ? 'at least ' : ''}${gb.toFixed(1)} GB`
95132

96133
if (!force && gb <= maxGb) {
97134
if (dryRun) console.log(`[turbopack-cache] ${sizeLabel} (cap ${maxGb} GB) — keeping`)
@@ -105,10 +142,15 @@ async function main() {
105142
}
106143

107144
console.log(`[turbopack-cache] ${sizeLabel} is ${reason} — pruning; next start recompiles cold`)
108-
await rm(CACHE_DIR, { recursive: true, force: true })
145+
if (BUILD_CACHE) {
146+
for await (const entry of await opendir(CACHE_DIR))
147+
await rm(path.join(CACHE_DIR, entry.name), { recursive: true, force: true })
148+
} else {
149+
await rm(CACHE_DIR, { recursive: true, force: true })
150+
}
109151
}
110152

111153
main().catch((error) => {
112-
// Never block `next dev` on a cache-maintenance failure.
113154
console.warn('[turbopack-cache] prune skipped:', error)
155+
if (BUILD_CACHE) process.exitCode = 1
114156
})

0 commit comments

Comments
 (0)