diff --git a/.github/workflows/lint-build-test.yml b/.github/workflows/lint-build-test.yml index 3603948425..1a71853082 100644 --- a/.github/workflows/lint-build-test.yml +++ b/.github/workflows/lint-build-test.yml @@ -64,11 +64,14 @@ jobs: validate-changelog: name: Validate changelog runs-on: ubuntu-latest - needs: prepare + if: needs.get-changed-packages.outputs.package-names != '[]' + needs: + - prepare + - get-changed-packages strategy: matrix: node-version: [24.x] - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.get-changed-packages.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -87,6 +90,52 @@ jobs: exit 1 fi + get-changed-packages: + name: Get changed packages + runs-on: ubuntu-latest + needs: prepare + outputs: + merge-base: ${{ steps.fetch-merge-base.outputs.merge-base }} + package-names: ${{ steps.packages.outputs.package-names }} + steps: + - name: Checkout and setup environment + uses: MetaMask/action-checkout-and-setup@v3 + with: + is-high-risk-environment: false + persist-credentials: false + node-version: 24.x + - name: Fetch merge base + id: fetch-merge-base + if: github.base_ref != '' || github.event.merge_group.base_ref != '' + run: | + set -euo pipefail + + PREFIXED_REF_REGEX='refs/heads/(.+)' + if [[ "$BASE_REF" =~ $PREFIXED_REF_REGEX ]]; then + BASE_REF="${BASH_REMATCH[1]}" + fi + + MERGE_BASE=$(gh api "repos/$GITHUB_REPOSITORY/compare/$BASE_REF...$HEAD_SHA" --jq '.merge_base_commit.sha') + git fetch --unshallow --filter=blob:none --no-tags origin HEAD + + echo "merge-base=$MERGE_BASE" >> "$GITHUB_OUTPUT" + env: + BASE_REF: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + GH_TOKEN: ${{ github.token }} + - name: Get changed package names + id: packages + run: | + if [[ -n "$MERGE_BASE" ]]; then + PACKAGES=$(yarn tsx scripts/get-changed-workspaces.mts "$MERGE_BASE" "$HEAD_SHA") + else + PACKAGES=$(yarn workspaces list --no-private --json | jq --slurp --raw-output 'map(.name) | @json') + fi + echo "package-names=$PACKAGES" >> "$GITHUB_OUTPUT" + env: + MERGE_BASE: ${{ steps.fetch-merge-base.outputs.merge-base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + validate-changelog-diffs: name: Validate changelog diffs if: github.event_name == 'pull_request' || github.event_name == 'merge_group' @@ -102,7 +151,9 @@ jobs: build: name: Build runs-on: ubuntu-latest - needs: prepare + needs: + - prepare + - get-changed-packages strategy: matrix: node-version: [24.x] @@ -113,7 +164,28 @@ jobs: is-high-risk-environment: false persist-credentials: false node-version: ${{ matrix.node-version }} - - run: yarn build + - name: Unshallow checkout + if: needs.get-changed-packages.outputs.merge-base != '' + run: | + # Unshallow so git can walk history back to the merge base for + # `git diff --name-only`. Using `--filter=blob:none` avoids + # downloading file content — only commit and tree objects are needed. + git fetch --unshallow --filter=blob:none --no-tags origin HEAD + - name: Build + run: | + if [[ -n "$MERGE_BASE" ]]; then + TSCONFIG=$(mktemp --tmpdir="$GITHUB_WORKSPACE" --suffix=.json) + yarn tsx scripts/generate-partial-build-tsconfig.mts "$MERGE_BASE" "$HEAD_SHA" > "$TSCONFIG" + if [[ -s "$TSCONFIG" ]]; then + yarn ts-bridge --project "$TSCONFIG" --verbose + fi + rm -f "$TSCONFIG" + else + yarn build + fi + env: + MERGE_BASE: ${{ needs.get-changed-packages.outputs.merge-base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} - name: Require clean working directory shell: bash run: | @@ -151,10 +223,13 @@ jobs: test-18: name: Test (18.x) runs-on: ubuntu-latest - needs: prepare + if: needs.get-changed-packages.outputs.package-names != '[]' + needs: + - prepare + - get-changed-packages strategy: matrix: - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.get-changed-packages.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -176,10 +251,13 @@ jobs: test-20: name: Test (20.x) runs-on: ubuntu-latest - needs: prepare + if: needs.get-changed-packages.outputs.package-names != '[]' + needs: + - prepare + - get-changed-packages strategy: matrix: - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.get-changed-packages.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -201,10 +279,13 @@ jobs: test-22: name: Test (22.x) runs-on: ubuntu-latest - needs: prepare + if: needs.get-changed-packages.outputs.package-names != '[]' + needs: + - prepare + - get-changed-packages strategy: matrix: - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.get-changed-packages.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -246,7 +327,10 @@ jobs: test-wallet-cli-e2e: name: Test wallet-cli daemon e2e (${{ matrix.node-version }}) runs-on: ubuntu-latest - needs: prepare + if: needs.get-changed-packages.outputs.package-names == '' || contains(fromJson(needs.get-changed-packages.outputs.package-names), '@metamask/wallet-cli') + needs: + - prepare + - get-changed-packages strategy: matrix: node-version: [20.x, 22.x, 24.x] diff --git a/scripts/generate-partial-build-tsconfig.mts b/scripts/generate-partial-build-tsconfig.mts new file mode 100644 index 0000000000..978afed55d --- /dev/null +++ b/scripts/generate-partial-build-tsconfig.mts @@ -0,0 +1,50 @@ +import { + getTypeScriptWorkspaces, + computeChangedWorkspaces, +} from './lib/workspaces.mjs'; + +/** + * Generate a filtered tsconfig.build.json for partial CI builds. + * + * Given a merge base SHA, outputs a tsconfig that references only the + * TypeScript packages that changed since that commit plus their transitive + * dependants and dependencies. Pipe the output to a temp file and pass it + * to `ts-bridge --project`. + * + * Dependencies are always included because TypeScript project references + * require every referenced project's dist output to already exist on disk. + * + * Usage: `tsx scripts/generate-partial-build-tsconfig.mts []` + */ +async function main() { + const mergeBase = process.argv[2]; + if (!mergeBase) { + console.error( + 'Usage: generate-partial-build-tsconfig.mts []', + ); + process.exitCode = 1; + return; + } + + const headRef = process.argv[3] ?? 'HEAD'; + + const workspaces = await getTypeScriptWorkspaces(); + const packagesToBuild = await computeChangedWorkspaces( + workspaces, + mergeBase, + headRef, + true, + ); + + const references = workspaces + .filter(({ name }) => packagesToBuild.has(name)) + .map(({ location }) => ({ path: `./${location}/tsconfig.build.json` })); + + if (references.length === 0) { + return; + } + + console.log(JSON.stringify({ files: [], include: [], references }, null, 2)); +} + +await main(); diff --git a/scripts/get-changed-workspaces.mts b/scripts/get-changed-workspaces.mts new file mode 100644 index 0000000000..48962750da --- /dev/null +++ b/scripts/get-changed-workspaces.mts @@ -0,0 +1,43 @@ +import { + getAllWorkspaces, + computeChangedWorkspaces, +} from './lib/workspaces.mjs'; + +/** + * List workspace package names that need to be checked given a merge base. + * + * Outputs a JSON array of package names to stdout. Always includes packages + * that changed since the merge base plus their transitive dependants. Pass + * `--include-dependencies` to also include transitive dependencies (needed + * for TypeScript project reference builds where dist outputs must exist). + * + * Usage: `tsx scripts/get-changed-workspaces.mts [] [--include-dependencies]` + */ +const args = process.argv.slice(2); +const includeDependencies = args.includes('--include-dependencies'); +const positional = args.filter((arg) => !arg.startsWith('--')); + +const mergeBase = positional[0]; +if (!mergeBase) { + console.error( + 'Usage: get-changed-workspaces.mts [] [--include-dependencies]', + ); + process.exitCode = 1; + process.exit(); +} + +const headRef = positional[1] ?? 'HEAD'; + +const workspaces = await getAllWorkspaces(); +const changed = await computeChangedWorkspaces( + workspaces, + mergeBase, + headRef, + includeDependencies, +); + +const names = workspaces + .filter(({ name }) => changed.has(name)) + .map(({ name }) => name); + +console.log(JSON.stringify(names)); diff --git a/scripts/lib/workspaces.mts b/scripts/lib/workspaces.mts new file mode 100644 index 0000000000..c73f7b1e4a --- /dev/null +++ b/scripts/lib/workspaces.mts @@ -0,0 +1,194 @@ +import { fileExists } from '@metamask/utils/node'; +import execa from 'execa'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +export const ROOT_WORKSPACE = new URL('../..', import.meta.url).pathname; + +// Files that can change without requiring a full rebuild/test run. +const IGNORED_ROOT_FILES = new Set([ + '.gitignore', + 'AGENTS.md', + 'CLAUDE.md', + 'README.md', + 'eslint-suppressions.json', + 'teams.json', + 'yarn.lock', +]); + +export type Workspace = { + location: string; + name: string; +}; + +export type DependencyGraph = { + dependants: Record>; + dependencies: Record>; +}; + +/** + * Get all non-root workspaces in the monorepo. + * + * @returns All workspaces. + */ +export async function getAllWorkspaces(): Promise { + const { stdout } = await execa( + 'yarn', + ['workspaces', 'list', '--no-private', '--json'], + { + cwd: ROOT_WORKSPACE, + encoding: 'utf8', + }, + ); + + return stdout + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .filter(({ location }: Workspace) => location !== '.'); +} + +/** + * Get all TypeScript workspaces in the monorepo. This filters to packages + * containing a "tsconfig.build.json" file. + * + * @returns All TypeScript workspaces. + */ +export async function getTypeScriptWorkspaces(): Promise { + const workspaces = await getAllWorkspaces(); + + return ( + await Promise.all( + workspaces.map(async (workspace) => { + const hasTsConfig = await fileExists( + join(ROOT_WORKSPACE, workspace.location, 'tsconfig.build.json'), + ); + return hasTsConfig ? workspace : null; + }), + ) + ).filter((workspace): workspace is Workspace => workspace !== null); +} + +/** + * Get dependency and dependant maps for all workspaces. + * + * @param workspaces - The workspaces to build the graph for. + * @returns Maps of package name to dependants and package name to dependencies. + */ +export async function getWorkspaceDependencies( + workspaces: Workspace[], +): Promise { + const dependants: Record> = Object.fromEntries( + workspaces.map(({ name }) => [name, new Set()]), + ); + const dependencies: Record> = Object.fromEntries( + workspaces.map(({ name }) => [name, new Set()]), + ); + + for (const { name, location } of workspaces) { + const pkg = JSON.parse( + await readFile(join(ROOT_WORKSPACE, location, 'package.json'), { + encoding: 'utf-8', + }), + ); + + for (const dependency of Object.keys({ + ...pkg.dependencies, + ...pkg.devDependencies, + })) { + if (dependants[dependency] !== undefined) { + dependants[dependency].add(name); + dependencies[name].add(dependency); + } + } + } + + return { dependants, dependencies }; +} + +/** + * Get the list of files changed between the merge base and the PR head. + * + * @param mergeBase - The merge base SHA. + * @param headRef - The PR branch tip SHA (or "HEAD" as fallback). + * @returns A list of changed file paths. + */ +export async function getChangedFiles( + mergeBase: string, + headRef: string, +): Promise { + const { stdout } = await execa( + 'git', + ['diff', '--name-only', `${mergeBase}...${headRef}`], + { + cwd: ROOT_WORKSPACE, + encoding: 'utf8', + }, + ); + + return stdout.trim().split('\n').filter(Boolean); +} + +/** + * Compute the set of workspace names that need to be checked given a merge + * base, by finding changed packages and expanding to transitive dependants. + * + * When `includeDependencies` is true, also expands to transitive dependencies. + * This is needed for TypeScript project reference builds, where every + * referenced project's dist output must already exist on disk. + * + * @param workspaces - The workspace set to compute against. + * @param mergeBase - The merge base SHA. + * @param headRef - The PR branch tip SHA (or "HEAD" as fallback). + * @param includeDependencies - Whether to also expand to transitive dependencies. + * @returns The set of workspace names to check. + */ +export async function computeChangedWorkspaces( + workspaces: Workspace[], + mergeBase: string, + headRef: string, + includeDependencies: boolean, +): Promise> { + const changedFiles = await getChangedFiles(mergeBase, headRef); + const { dependants, dependencies } = + await getWorkspaceDependencies(workspaces); + + // If any changed file lives outside all package directories (e.g. root + // configs, workflow files, scripts), rebuild and test everything. + const hasRootChange = changedFiles.some( + (file) => + !IGNORED_ROOT_FILES.has(file) && + !workspaces.some(({ location }) => file.startsWith(`${location}/`)), + ); + + if (hasRootChange) { + return new Set(workspaces.map(({ name }) => name)); + } + + const result = new Set( + changedFiles.flatMap((file) => { + const workspace = workspaces.find(({ location }) => + file.startsWith(`${location}/`), + ); + return workspace ? [workspace.name] : []; + }), + ); + + // Expand to transitive dependants (packages that depend on what changed). + for (const pkg of result) { + for (const dependant of dependants[pkg] ?? []) { + result.add(dependant); + } + } + + if (includeDependencies) { + // Expand to transitive dependencies (dist files must exist to build dependants). + for (const pkg of result) { + for (const dependency of dependencies[pkg] ?? []) { + result.add(dependency); + } + } + } + + return result; +} diff --git a/tsconfig.scripts.json b/tsconfig.scripts.json index b4adc2c51e..691fa67fa7 100644 --- a/tsconfig.scripts.json +++ b/tsconfig.scripts.json @@ -18,6 +18,6 @@ "noErrorTruncation": true, "noUncheckedIndexedAccess": true }, - "include": ["./scripts/**/*.ts"], + "include": ["./scripts/**/*.ts", "./scripts/**/*.mts"], "exclude": ["**/node_modules"] }