Optimize bundle compilation workflow

.github/workflows/codeql-compilation-caches.yml

@@ -43,6 +43,7 @@ jobs:
       cache_release: ${{ steps.version.outputs.cache_release }}
       catalog_branch: ${{ steps.version.outputs.catalog_branch }}
       matrix: ${{ steps.plan.outputs.matrix || '[{"language":"skip","target":"skip"}]' }}
+      cpp: ${{ steps.plan.outputs.cpp || '{"language":"skip","target":"skip"}' }}
       release: ${{ steps.version.outputs.release }}
       skip: ${{ steps.existing.outputs.skip }}
     steps:
@@ -129,7 +130,8 @@ jobs:
             --cache-release "${{ steps.version.outputs.cache_release }}" \
             --output release-plan.json
           echo "cache_release=$(jq -r '.cache_release' release-plan.json)" >> "$GITHUB_OUTPUT"
-          echo "matrix=$(jq -c '.targets' release-plan.json)" >> "$GITHUB_OUTPUT"
+          echo "matrix=$(jq -c '[.targets[] | select(.language != "cpp")]' release-plan.json)" >> "$GITHUB_OUTPUT"
+          echo "cpp=$(jq -c '.targets[] | select(.language == "cpp")' release-plan.json)" >> "$GITHUB_OUTPUT"
 
       - if: steps.existing.outputs.skip != 'true'
         uses: actions/upload-artifact@v4
@@ -141,7 +143,9 @@ jobs:
   build:
     needs: plan
     if: needs.plan.outputs.skip != 'true'
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.cache.language == 'java' && vars.CODEQL_CACHE_JAVA_RUNNER || 'ubuntu-latest' }}
+    env:
+      CODEQL_CACHE_RAM: ${{ matrix.cache.language == 'java' && vars.CODEQL_CACHE_JAVA_RAM || vars.CODEQL_CACHE_RAM }}
     strategy:
       fail-fast: false
       matrix:
@@ -165,11 +169,18 @@ jobs:
           key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}
 
       - name: Build and verify cache
+        shell: bash
         run: |
-          codeql-bundle-cache build \
-            --plan release-plan.json \
-            --target "${{ matrix.cache.target }}" \
+          args=(
+            build
+            --plan release-plan.json
+            --target "${{ matrix.cache.target }}"
             --output-dir dist
+          )
+          if [[ -n "$CODEQL_CACHE_RAM" ]]; then
+            args+=(--ram "$CODEQL_CACHE_RAM")
+          fi
+          codeql-bundle-cache "${args[@]}"
 
       - uses: actions/upload-artifact@v4
         with:
@@ -181,10 +192,155 @@ jobs:
           if-no-files-found: error
           compression-level: 0
 
+  build-cpp:
+    needs: plan
+    if: needs.plan.outputs.skip != 'true'
+    runs-on: ubuntu-latest
+    env:
+      CODEQL_CACHE_RAM: ${{ vars.CODEQL_CACHE_CPP_RAM || vars.CODEQL_CACHE_RAM }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - run: python -m pip install .
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: release-plan
+
+      - uses: actions/cache@v4
+        with:
+          path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }}
+          key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}
+
+      - name: Build and verify C++ cache
+        shell: bash
+        run: |
+          args=(
+            build
+            --plan release-plan.json
+            --target "${{ fromJSON(needs.plan.outputs.cpp).target }}"
+            --output-dir dist
+          )
+          if [[ -n "$CODEQL_CACHE_RAM" ]]; then
+            args+=(--ram "$CODEQL_CACHE_RAM")
+          fi
+          codeql-bundle-cache "${args[@]}"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cache-${{ fromJSON(needs.plan.outputs.cpp).language }}
+          path: |
+            dist/*.tar.gz
+            dist/*.sha256
+            dist/*.metadata.json
+          if-no-files-found: error
+          compression-level: 0
+
+  consumer-test:
+    needs:
+      - plan
+      - build-cpp
+    if: needs.plan.outputs.skip != 'true'
+    runs-on: ${{ vars.CODEQL_CACHE_CONSUMER_RUNNER || 'ubuntu-latest' }}
+    env:
+      CODEQL_CACHE_RAM: ${{ vars.CODEQL_CACHE_CONSUMER_RAM || vars.CODEQL_CACHE_RAM }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - run: python -m pip install .
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: release-plan
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: cache-cpp
+          path: dist
+
+      - uses: actions/cache@v4
+        with:
+          path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }}
+          key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }}
+
+      - name: Test automatic cache use with a customization
+        shell: bash
+        run: |
+          jq '.targets = [.targets[] | select(.language == "cpp")]' \
+            release-plan.json > cpp-release-plan.json
+          codeql-bundle-cache catalog-entry \
+            --plan cpp-release-plan.json \
+            --assets-dir dist \
+            --validated-platform linux64 \
+            --output candidate-entry.json
+
+          python - <<'PY' > cache-server.log 2>&1 &
+          from functools import partial
+          from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
+          from pathlib import Path
+
+          handler = partial(SimpleHTTPRequestHandler, directory="dist")
+          server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
+          Path("cache-server-port").write_text(str(server.server_port))
+          server.serve_forever()
+          PY
+          server_pid=$!
+          trap 'kill "$server_pid" 2>/dev/null || true' EXIT
+          for _ in {1..50}; do
+            [[ -s cache-server-port ]] && break
+            sleep 0.1
+          done
+          port="$(cat cache-server-port)"
+          asset_name="$(jq -r '.compilation_caches["codeql/cpp-all"].asset.name' candidate-entry.json)"
+          jq \
+            --arg url "http://127.0.0.1:${port}/${asset_name}" \
+            '{
+              schema_version: 1,
+              bundles: [
+                (. | .compilation_caches["codeql/cpp-all"].asset.url = $url)
+              ]
+            }' \
+            candidate-entry.json > candidate-manifest.json
+
+          mkdir -p "${{ runner.temp }}/custom-bundles"
+          bundle_args=(
+            --bundle "${{ needs.plan.outputs.release }}"
+            --cache-manifest candidate-manifest.json
+            --cache-dir "${{ runner.temp }}/consumer-cache"
+            --output "${{ runner.temp }}/custom-bundles"
+            --workspace tests/workspace
+            --platform linux64
+          )
+          if [[ -n "$CODEQL_CACHE_RAM" ]]; then
+            bundle_args+=(--ram "$CODEQL_CACHE_RAM")
+          fi
+          codeql-bundle "${bundle_args[@]}" foo/cpp-customizations
+          mkdir -p "${{ runner.temp }}/custom-codeql"
+          tar -xzf \
+            "${{ runner.temp }}/custom-bundles/codeql-bundle-linux64.tar.gz" \
+            -C "${{ runner.temp }}/custom-codeql"
+          cp -R tests/workspace "${{ runner.temp }}/test-workspace"
+          rm -rf "${{ runner.temp }}/test-workspace/cpp/foo-customizations"
+          find "${{ runner.temp }}/test-workspace" \
+            -name codeql-pack.lock.yml \
+            -delete
+          "${{ runner.temp }}/custom-codeql/codeql/codeql" test run \
+            --additional-packs="${{ runner.temp }}/custom-codeql/codeql:${{ runner.temp }}/test-workspace" \
+            "${{ runner.temp }}/test-workspace/cpp/foo-bundle-customizations-tests"
+
   verify:
     needs:
       - plan
       - build
+      - build-cpp
     if: needs.plan.outputs.skip != 'true'
     strategy:
       fail-fast: false
@@ -225,6 +381,8 @@ jobs:
     needs:
       - plan
       - build
+      - build-cpp
+      - consumer-test
       - verify
     if: needs.plan.outputs.skip != 'true'
     runs-on: ubuntu-latest
@@ -314,32 +472,6 @@ jobs:
             --catalog codeql_bundle/supported-codeql-bundles.json \
             --entry catalog-entry.json
 
-      - name: Test automatic cache use with a customization
-        if: steps.recheck.outputs.skip != 'true'
-        shell: bash
-        run: |
-          mkdir -p "${{ runner.temp }}/custom-bundles"
-          codeql-bundle \
-            --bundle "${{ needs.plan.outputs.release }}" \
-            --cache-manifest codeql_bundle/supported-codeql-bundles.json \
-            --cache-dir "${{ runner.temp }}/consumer-cache" \
-            --output "${{ runner.temp }}/custom-bundles" \
-            --workspace tests/workspace \
-            --platform linux64 \
-            foo/cpp-customizations
-          mkdir -p "${{ runner.temp }}/custom-codeql"
-          tar -xzf \
-            "${{ runner.temp }}/custom-bundles/codeql-bundle-linux64.tar.gz" \
-            -C "${{ runner.temp }}/custom-codeql"
-          cp -R tests/workspace "${{ runner.temp }}/test-workspace"
-          rm -rf "${{ runner.temp }}/test-workspace/cpp/foo-customizations"
-          find "${{ runner.temp }}/test-workspace" \
-            -name codeql-pack.lock.yml \
-            -delete
-          "${{ runner.temp }}/custom-codeql/codeql/codeql" test run \
-            --additional-packs="${{ runner.temp }}/custom-codeql/codeql:${{ runner.temp }}/test-workspace" \
-            "${{ runner.temp }}/test-workspace/cpp/foo-bundle-customizations-tests"
-
       - name: Open catalog update pull request
         if: steps.recheck.outputs.skip != 'true'
         env:

README.md

@@ -77,6 +77,21 @@ missing, unvalidated, or corrupt cache is an error. Use
 `--no-precompile` also skips cache resolution because no query compilation is
 performed.
 
+### Resource tuning
+
+Version 0.5.1 adds controls for constrained or high-capacity build agents:
+
+- `--threads` sets CodeQL compiler parallelism; the default uses all available
+  cores.
+- `--ram` caps the memory in MB available to CodeQL pack compilation.
+- `--compression-level` selects the gzip level for generated bundle archives.
+  The default is 6, which materially reduces archive time with little size
+  increase compared with level 9.
+
+Set RAM below the machine's physical limit so the operating system and archive
+creation retain headroom. Avoid running multiple `pack create` processes with
+`--threads=0` on the same agent because each process will try to use every core.
+
 By default, downloads are stored in the platform cache directory. Override it
 with `--cache-dir` or `CODEQL_BUNDLE_CACHE_DIR`. Use `--cache-manifest` with a
 local path or URL to test a candidate catalog or to operate against a pinned
@@ -131,6 +146,15 @@ The latest stable release is independent of backfill work; releases from
 v2.18.0 onward can be dispatched separately in any order. `force` rebuilds a
 release that already has a catalog entry or open update pull request.
 
+Repository variables can opt slow cache stages into larger runners without
+changing the workflow:
+
+- `CODEQL_CACHE_JAVA_RUNNER` selects the Java cache-build runner.
+- `CODEQL_CACHE_CONSUMER_RUNNER` selects the customized C++ acceptance runner.
+- `CODEQL_CACHE_RAM` optionally sets a common RAM cap.
+- `CODEQL_CACHE_JAVA_RAM`, `CODEQL_CACHE_CPP_RAM`, and
+  `CODEQL_CACHE_CONSUMER_RAM` override that cap for their respective jobs.
+
 ## CodeQL customization packs
 
 The CodeQL bundle CLI application provides a development experience for customization packs that mimics the development experience for official CodeQL packs.

codeql_bundle/cli.py

@@ -10,7 +10,12 @@
 import click
 from pathlib import Path
 from codeql_bundle.helpers.codeql import CodeQLException
-from codeql_bundle.helpers.bundle import CustomBundle, BundleException, BundlePlatform
+from codeql_bundle.helpers.bundle import (
+    DEFAULT_COMPRESSION_LEVEL,
+    BundleException,
+    BundlePlatform,
+    CustomBundle,
+)
 from codeql_bundle.cache import (
     BundleCatalog,
     BundleSourceResolver,
@@ -86,6 +91,19 @@
     type=click.INT,
     help="Use this many threads to compile queries.",
 )
+@click.option(
+    "-M",
+    "--ram",
+    type=click.IntRange(min=1),
+    help="Set total amount of RAM in MB that the compiler may use.",
+)
+@click.option(
+    "--compression-level",
+    type=click.IntRange(min=0, max=9),
+    default=DEFAULT_COMPRESSION_LEVEL,
+    show_default=True,
+    help="Gzip compression level for generated bundle archives.",
+)
 @click.option(
     "--cache-dir",
     type=click.Path(path_type=Path),
@@ -115,6 +133,8 @@ def main(
     code_scanning_config: Optional[Path],
     additional_data_config: Optional[Path],
     threads: Optional[int],
+    ram: Optional[int],
+    compression_level: int,
     cache_dir: Path,
     cache_manifest: Optional[str],
     no_compilation_cache: bool,
@@ -169,6 +189,8 @@ def main(
         # options for custom bundle
         bundle.disable_precompilation = no_precompile
         bundle.threads = threads
+        bundle.ram = ram
+        bundle.compression_level = compression_level
 
         unsupported_platforms = list(
             filter(