diff --git a/.github/workflows/codeql-compilation-caches.yml b/.github/workflows/codeql-compilation-caches.yml new file mode 100644 index 0000000..86e0da9 --- /dev/null +++ b/.github/workflows/codeql-compilation-caches.yml @@ -0,0 +1,376 @@ +name: Build CodeQL compilation caches + +on: + push: + branches: + - main + paths: + - ".github/workflows/codeql-compilation-caches.yml" + - "codeql_bundle/cache.py" + - "codeql_bundle/cache_cli.py" + - "codeql_bundle/helpers/**" + - "codeql_bundle/supported-codeql-bundles*" + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + inputs: + bundle_version: + description: CodeQL bundle tag or version to build + required: false + type: string + force: + description: Rebuild even if a catalog entry or update pull request exists + required: false + default: false + type: boolean + +permissions: + contents: read + pull-requests: read + +concurrency: + group: codeql-compilation-cache-${{ inputs.bundle_version || 'latest' }} + cancel-in-progress: false + +env: + CODEQL_BUNDLE_CACHE_DIR: ${{ runner.temp }}/codeql-bundle-cache + PIP_DISABLE_PIP_VERSION_CHECK: "1" + +jobs: + plan: + runs-on: ubuntu-latest + outputs: + cache_release: ${{ steps.version.outputs.cache_release }} + catalog_branch: ${{ steps.version.outputs.catalog_branch }} + matrix: ${{ steps.plan.outputs.matrix || '[{"language":"skip","target":"skip"}]' }} + release: ${{ steps.version.outputs.release }} + skip: ${{ steps.existing.outputs.skip }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: python -m pip install . + + - id: version + name: Select upstream release + env: + REQUESTED_BUNDLE_VERSION: ${{ inputs.bundle_version }} + FORCE_REBUILD: ${{ inputs.force || false }} + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + release="$REQUESTED_BUNDLE_VERSION" + if [[ -z "$release" ]]; then + release="$(codeql-bundle-cache latest-release)" + elif [[ "$release" != codeql-bundle-* ]]; then + release="codeql-bundle-v${release#v}" + fi + if [[ ! "$release" =~ ^codeql-bundle-v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Invalid CodeQL bundle release: $release" >&2 + exit 1 + fi + base_cache_release="codeql-compilation-cache-${release#codeql-bundle-}" + cache_release="$base_cache_release" + catalog_branch="automation/$base_cache_release" + if [[ "$FORCE_REBUILD" == "true" ]] || + gh release view "$base_cache_release" >/dev/null 2>&1; then + cache_release="${base_cache_release}-r${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + fi + if [[ "$FORCE_REBUILD" == "true" ]]; then + catalog_branch="automation/$cache_release" + fi + echo "cache_release=$cache_release" >> "$GITHUB_OUTPUT" + echo "catalog_branch=$catalog_branch" >> "$GITHUB_OUTPUT" + echo "release=$release" >> "$GITHUB_OUTPUT" + + - id: existing + name: Check authoritative catalog + env: + GH_TOKEN: ${{ github.token }} + FORCE_REBUILD: ${{ inputs.force || false }} + shell: bash + run: | + present="$(codeql-bundle-cache catalog-has \ + --catalog codeql_bundle/supported-codeql-bundles.json \ + --release "${{ steps.version.outputs.release }}")" + open_prs="$(gh pr list \ + --state open \ + --head "${{ steps.version.outputs.catalog_branch }}" \ + --json number \ + --jq length)" + if [[ "$FORCE_REBUILD" != "true" && "$present" == "true" ]]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "The catalog already contains ${{ steps.version.outputs.release }}." + elif [[ "$FORCE_REBUILD" != "true" && "$open_prs" != "0" ]]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "A catalog update pull request is already open for ${{ steps.version.outputs.release }}." + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - if: steps.existing.outputs.skip != 'true' + uses: actions/cache@v4 + with: + path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ steps.version.outputs.release }} + key: codeql-source-${{ steps.version.outputs.release }}-${{ runner.os }} + + - if: steps.existing.outputs.skip != 'true' + id: plan + name: Create release plan + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + codeql-bundle-cache plan-release \ + --release "${{ steps.version.outputs.release }}" \ + --cache-release "${{ steps.version.outputs.cache_release }}" \ + --output release-plan.json + echo "cache_release=$(jq -r '.cache_release' release-plan.json)" >> "$GITHUB_OUTPUT" + echo "matrix=$(jq -c '.targets' release-plan.json)" >> "$GITHUB_OUTPUT" + + - if: steps.existing.outputs.skip != 'true' + uses: actions/upload-artifact@v4 + with: + name: release-plan + path: release-plan.json + if-no-files-found: error + + build: + needs: plan + if: needs.plan.outputs.skip != 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + cache: ${{ fromJSON(needs.plan.outputs.matrix) }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: python -m pip install . + + - uses: actions/download-artifact@v4 + with: + name: release-plan + + - uses: actions/cache@v4 + with: + path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + + - name: Build and verify cache + run: | + codeql-bundle-cache build \ + --plan release-plan.json \ + --target "${{ matrix.cache.target }}" \ + --output-dir dist + + - uses: actions/upload-artifact@v4 + with: + name: cache-${{ matrix.cache.language }} + path: | + dist/*.tar.gz + dist/*.sha256 + dist/*.metadata.json + if-no-files-found: error + compression-level: 0 + + verify: + needs: + - plan + - build + if: needs.plan.outputs.skip != 'true' + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: python -m pip install . + + - uses: actions/download-artifact@v4 + with: + name: release-plan + + - uses: actions/download-artifact@v4 + with: + pattern: cache-* + path: dist + merge-multiple: true + + - uses: actions/cache@v4 + with: + path: ${{ env.CODEQL_BUNDLE_CACHE_DIR }}/sources/${{ needs.plan.outputs.release }} + key: codeql-source-${{ needs.plan.outputs.release }}-${{ runner.os }} + + - name: Verify all caches + run: codeql-bundle-cache verify-all --plan release-plan.json --assets-dir dist + + publish: + needs: + - plan + - build + - verify + if: needs.plan.outputs.skip != 'true' + runs-on: ubuntu-latest + permissions: + actions: write + contents: write + pull-requests: write + concurrency: + group: publish-codeql-compilation-cache-${{ needs.plan.outputs.release }} + cancel-in-progress: false + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: python -m pip install . + + - uses: actions/download-artifact@v4 + with: + name: release-plan + + - uses: actions/download-artifact@v4 + with: + pattern: cache-* + path: dist + merge-multiple: true + + - id: recheck + name: Recheck catalog update pull request + env: + GH_TOKEN: ${{ github.token }} + FORCE_REBUILD: ${{ inputs.force || false }} + shell: bash + run: | + branch="${{ needs.plan.outputs.catalog_branch }}" + git fetch --quiet origin main + git show \ + origin/main:codeql_bundle/supported-codeql-bundles.json \ + > "${{ runner.temp }}/current-catalog.json" + present="$(codeql-bundle-cache catalog-has \ + --catalog "${{ runner.temp }}/current-catalog.json" \ + --release "${{ needs.plan.outputs.release }}")" + open_prs="$(gh pr list \ + --state open \ + --head "$branch" \ + --json number \ + --jq length)" + if [[ "$FORCE_REBUILD" != "true" && "$present" == "true" ]]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "The catalog was updated by another run." + elif [[ "$FORCE_REBUILD" != "true" && "$open_prs" != "0" ]]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "A catalog update pull request was opened by another run." + else + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Publish cache release + if: steps.recheck.outputs.skip != 'true' + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + mapfile -t assets < <(find dist -maxdepth 1 -type f | sort) + gh release create "${{ needs.plan.outputs.cache_release }}" \ + "${assets[@]}" \ + --latest=false \ + --title "Compilation caches for ${{ needs.plan.outputs.release }}" \ + --notes "Validated CodeQL compilation caches generated from ${{ needs.plan.outputs.release }}." + + - name: Create candidate catalog entry + if: steps.recheck.outputs.skip != 'true' + run: | + codeql-bundle-cache catalog-entry \ + --plan release-plan.json \ + --assets-dir dist \ + --validated-platform linux64 \ + --validated-platform osx64 \ + --validated-platform win64 \ + --output catalog-entry.json + codeql-bundle-cache verify-entry \ + --entry catalog-entry.json \ + --cache-dir "${{ runner.temp }}/published-cache" + codeql-bundle-cache update-catalog \ + --catalog codeql_bundle/supported-codeql-bundles.json \ + --entry catalog-entry.json + + - name: Test automatic cache use with a customization + if: steps.recheck.outputs.skip != 'true' + shell: bash + run: | + mkdir -p "${{ runner.temp }}/custom-bundles" + codeql-bundle \ + --bundle "${{ needs.plan.outputs.release }}" \ + --cache-manifest codeql_bundle/supported-codeql-bundles.json \ + --cache-dir "${{ runner.temp }}/consumer-cache" \ + --output "${{ runner.temp }}/custom-bundles" \ + --workspace tests/workspace \ + --platform linux64 \ + foo/cpp-customizations + mkdir -p "${{ runner.temp }}/custom-codeql" + tar -xzf \ + "${{ runner.temp }}/custom-bundles/codeql-bundle-linux64.tar.gz" \ + -C "${{ runner.temp }}/custom-codeql" + cp -R tests/workspace "${{ runner.temp }}/test-workspace" + rm -rf "${{ runner.temp }}/test-workspace/cpp/foo-customizations" + find "${{ runner.temp }}/test-workspace" \ + -name codeql-pack.lock.yml \ + -delete + "${{ runner.temp }}/custom-codeql/codeql/codeql" test run \ + --additional-packs="${{ runner.temp }}/custom-codeql/codeql:${{ runner.temp }}/test-workspace" \ + "${{ runner.temp }}/test-workspace/cpp/foo-bundle-customizations-tests" + + - name: Open catalog update pull request + if: steps.recheck.outputs.skip != 'true' + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + branch="${{ needs.plan.outputs.catalog_branch }}" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git switch -c "$branch" + git add codeql_bundle/supported-codeql-bundles.json + git commit -m "Add compilation caches for ${{ needs.plan.outputs.release }}" + git push --force origin "HEAD:refs/heads/$branch" + open_prs="$(gh pr list \ + --state open \ + --head "$branch" \ + --json number \ + --jq length)" + if [[ "$open_prs" == "0" ]]; then + gh pr create \ + --base main \ + --head "$branch" \ + --title "Add compilation caches for ${{ needs.plan.outputs.release }}" \ + --body "Publishes the validated per-language compilation caches for \`${{ needs.plan.outputs.release }}\`." + fi + + - name: Trigger catalog pull request tests + if: steps.recheck.outputs.skip != 'true' + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + gh workflow run test.yml \ + --ref "${{ needs.plan.outputs.catalog_branch }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4a81071 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,30 @@ +name: Test + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +jobs: + unit: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: + - "3.11" + - "3.12" + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - run: python -m pip install . + + - run: python -m unittest discover -s tests -p "test_*.py" -v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca1a1b2..25e7267 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,6 +18,17 @@ Please note that this project is released with a [Contributor Code of Conduct][c 3. Push to your fork and [submit a pull request][pr]. 4. Ensure the required checks pass. +Run the local unit tests with: + +```bash +python -m unittest discover -s tests -p "test_*.py" -v +``` + +Cache release development does not require GitHub Actions. The +`codeql-bundle-cache` commands documented in the README perform release +inspection, cache construction, cache-hit verification, and catalog generation +locally. The workflow should remain a thin wrapper around those commands. + Here are a few things you can do that will increase the likelihood of your pull request being accepted: - Write tests. diff --git a/README.md b/README.md index ec0ede4..cdf2c8e 100644 --- a/README.md +++ b/README.md @@ -18,20 +18,23 @@ For more details on CodeQL customization packs see the section [CodeQL customiza The CodeQL bundle application can be installed using `pip` with the command: ```bash -python3.11 -m pip install https://github.com/rvermeulen/codeql-bundle/releases/download/v0.1.8/codeql_bundle-0.1.8-py3-none-any.whl +python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/releases/download/v0.5.0/codeql_bundle-0.5.0-py3-none-any.whl ``` ## Usage -Before you can use the CodeQL bundle application you must download a bundle you want to customize from the CodeQL Action [releases](https://github.com/github/codeql-action/releases) page. +The source bundle can be an existing local archive or directory, a +`github/codeql-action` release tag, or an HTTP(S) URL. Release tags and URLs are +downloaded into a persistent local cache. The CodeQL bundle application requires a [CodeQL workspace](https://codeql.github.com/docs/codeql-cli/about-codeql-workspaces/) to locate the packs you want to include in a custom bundle. You can see the packs available in your workspace by running `codeql pack ls -- ` where `` is the root directory of your CodeQL workspace. -With both a CodeQL bundle and a CodeQL workspace you can create a bundle with the command: +With a CodeQL bundle release and a CodeQL workspace you can create a bundle +with the command: ```bash -codeql-bundle --bundle --output codeql-custom-bundle.tar.gz --workspace --log INFO +codeql-bundle --bundle codeql-bundle-v2.26.1 --output codeql-custom-bundle.tar.gz --workspace --log INFO ``` If the source bundle is the platform agnostic bundle then you can create platform specific bundles to reduce the size of the used bundle(s). @@ -41,6 +44,93 @@ The following example creates platform specific bundles for all the currently su codeql-bundle --bundle --output --workspace --log INFO -p linux64 -p osx64 -p win64 ``` +### Compilation caches + +The repository maintains +[`supported-codeql-bundles.json`](codeql_bundle/supported-codeql-bundles.json) +as the authoritative catalog of validated source bundles and compilation-cache +assets. Each entry pins: + +- the upstream release, CLI version, source asset URLs, sizes, and SHA-256 + digests; +- a content fingerprint of the CodeQL packs; +- the consumer platforms on which the cache was validated; and +- one cache asset for each customizable standard library. + +When a customization requires standard query packs to be recreated, +`codeql-bundle` downloads only the relevant cache assets and passes them to +`codeql pack create`. Local archives downloaded by tools such as QLT are +recognized by their exact SHA-256 digest. Extracted directories are recognized +by their CLI version and pack-content fingerprint. + +Compilation caches are content-addressed by the query, its transitive +dependencies, and the compiler. Customizing a standard library therefore +invalidates the entries that depend on the changed library while preserving +entries for unaffected compiler stages. Cache-aware builds reduce compilation +time, but they do not eliminate compilation; the speedup depends on the +customization and query packs involved. + +An input that is not in the catalog remains supported, but is built without +this optimization and produces a warning. For a catalog-listed bundle, a +missing, unvalidated, or corrupt cache is an error. Use +`--no-compilation-cache` to explicitly continue without the optimization. +`--no-precompile` also skips cache resolution because no query compilation is +performed. + +By default, downloads are stored in the platform cache directory. Override it +with `--cache-dir` or `CODEQL_BUNDLE_CACHE_DIR`. Use `--cache-manifest` with a +local path or URL to test a candidate catalog or to operate against a pinned +catalog. Remove source bundles and caches that have not been used in 30 days with +`codeql-bundle-cache prune`; use `--max-age-days` to choose a different +retention period or `--dry-run` to inspect what would be removed. + +### Building cache releases locally + +The `codeql-bundle-cache` command contains the implementation used by GitHub +Actions. A release can be prepared locally with: + +```bash +codeql-bundle-cache plan-release \ + --release codeql-bundle-v2.26.1 \ + --output release-plan.json + +jq -r '.targets[].target' release-plan.json | while read -r target; do + codeql-bundle-cache build \ + --plan release-plan.json \ + --target "$target" \ + --output-dir dist +done + +codeql-bundle-cache verify-all \ + --plan release-plan.json \ + --assets-dir dist +``` + +`plan-release` validates upstream release metadata and records every source +asset. `build` compiles the real standard query packs into a per-language cache +bounded to 1536 MiB and requires a second real compilation to report a cache +hit. `verify-all` repeats that check using the current platform's upstream +bundle. + +After release assets have been published, `catalog-entry`, `verify-entry`, and +`update-catalog` create, download-test, and insert the candidate catalog entry. +Run `codeql-bundle-cache --help` for the complete command surface. + +### Repository automation + +The +[`Build CodeQL compilation caches`](.github/workflows/codeql-compilation-caches.yml) +workflow polls for the latest stable upstream release and runs immediately when +the cache implementation lands on `main`. It uses the local commands above, +parallelizes cache construction, validates every cache on Linux, macOS, and +Windows, publishes a dedicated release, performs a consumer-path customization +test, and opens a pull request for manual review of the catalog update. + +Use `workflow_dispatch` with `bundle_version` to backfill a specific release. +The latest stable release is independent of backfill work; releases from +v2.18.0 onward can be dispatched separately in any order. `force` rebuilds a +release that already has a catalog entry or open update pull request. + ## CodeQL customization packs The CodeQL bundle CLI application provides a development experience for customization packs that mimics the development experience for official CodeQL packs. diff --git a/codeql_bundle/cache.py b/codeql_bundle/cache.py new file mode 100644 index 0000000..2aeeee7 --- /dev/null +++ b/codeql_bundle/cache.py @@ -0,0 +1,1141 @@ +from __future__ import annotations + +from dataclasses import dataclass +from hashlib import sha256 +from importlib.resources import files +from pathlib import Path +from typing import Any, Iterable, Optional +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlparse +from urllib.request import Request, urlopen +from contextlib import contextmanager +import json +import logging +import os +import platform +import re +import shutil +import tarfile +import time +import uuid + +from jsonschema import Draft202012Validator, FormatChecker + +from .helpers.codeql import CodeQLPack + + +logger = logging.getLogger(__name__) + +CATALOG_RESOURCE = "supported-codeql-bundles.json" +CATALOG_SCHEMA_RESOURCE = "supported-codeql-bundles.schema.json" +DEFAULT_CATALOG_URL = ( + "https://raw.githubusercontent.com/advanced-security/codeql-bundle/" + f"main/codeql_bundle/{CATALOG_RESOURCE}" +) +CODEQL_ACTION_REPOSITORY = "github/codeql-action" +CACHE_FORMAT_VERSION = 1 +DOWNLOAD_CHUNK_SIZE = 1024 * 1024 +CATALOG_REFRESH_SECONDS = 60 * 60 +RELEASE_PATTERN = re.compile(r"^codeql-bundle-v\d+\.\d+\.\d+$") +CACHE_RELEASE_PATTERN = re.compile( + r"^codeql-compilation-cache-v\d+\.\d+\.\d+(?:-[A-Za-z0-9._-]+)?$" +) + + +class CacheException(Exception): + pass + + +class CatalogException(CacheException): + pass + + +class DownloadException(CacheException): + pass + + +@dataclass(frozen=True) +class ReleaseAsset: + name: str + url: str + sha256: str + size: int + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "ReleaseAsset": + return cls( + name=value["name"], + url=value["url"], + sha256=value["sha256"], + size=value["size"], + ) + + def to_dict(self) -> dict[str, Any]: + return { + "name": self.name, + "url": self.url, + "sha256": self.sha256, + "size": self.size, + } + + +@dataclass(frozen=True) +class SourceAsset(ReleaseAsset): + platform: str + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "SourceAsset": + return cls( + name=value["name"], + url=value["url"], + sha256=value["sha256"], + size=value["size"], + platform=value["platform"], + ) + + def to_dict(self) -> dict[str, Any]: + value = ReleaseAsset.to_dict(self) + value["platform"] = self.platform + return value + + +@dataclass(frozen=True) +class CompilationCache: + language: str + query_packs: tuple[str, ...] + asset: ReleaseAsset + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "CompilationCache": + return cls( + language=value["language"], + query_packs=tuple(value["query_packs"]), + asset=ReleaseAsset.from_dict(value["asset"]), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "language": self.language, + "query_packs": list(self.query_packs), + "asset": self.asset.to_dict(), + } + + +@dataclass(frozen=True) +class SupportedBundle: + release: str + cli_version: str + source_repository: str + pack_fingerprint: str + source_assets: tuple[SourceAsset, ...] + cache_release: str + cache_format: int + compilation_caches: dict[str, CompilationCache] + validated_platforms: tuple[str, ...] + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "SupportedBundle": + return cls( + release=value["release"], + cli_version=value["cli_version"], + source_repository=value["source_repository"], + pack_fingerprint=value["pack_fingerprint"], + source_assets=tuple( + SourceAsset.from_dict(asset) for asset in value["source_assets"] + ), + cache_release=value["cache_release"], + cache_format=value["cache_format"], + compilation_caches={ + target: CompilationCache.from_dict(cache) + for target, cache in value["compilation_caches"].items() + }, + validated_platforms=tuple(value["validated_platforms"]), + ) + + def to_dict(self) -> dict[str, Any]: + return { + "release": self.release, + "cli_version": self.cli_version, + "source_repository": self.source_repository, + "pack_fingerprint": self.pack_fingerprint, + "source_assets": [ + asset.to_dict() + for asset in sorted(self.source_assets, key=lambda item: item.platform) + ], + "cache_release": self.cache_release, + "cache_format": self.cache_format, + "compilation_caches": { + target: cache.to_dict() + for target, cache in sorted(self.compilation_caches.items()) + }, + "validated_platforms": sorted(self.validated_platforms), + } + + def source_asset_for_platform(self, platform_name: str) -> Optional[SourceAsset]: + return next( + ( + asset + for asset in self.source_assets + if asset.platform == platform_name + ), + None, + ) + + +class BundleCatalog: + def __init__(self, bundles: Iterable[SupportedBundle]): + self.bundles = tuple(bundles) + _validate_catalog( + { + "schema_version": 1, + "bundles": [bundle.to_dict() for bundle in self.bundles], + } + ) + self._by_release = {bundle.release: bundle for bundle in self.bundles} + + if len(self._by_release) != len(self.bundles): + raise CatalogException("The supported bundle catalog has duplicate releases.") + for bundle in self.bundles: + platforms = [asset.platform for asset in bundle.source_assets] + if len(set(platforms)) != len(platforms): + raise CatalogException( + f"Bundle {bundle.release} has duplicate source platforms." + ) + + source_digests = [ + asset.sha256 + for bundle in self.bundles + for asset in bundle.source_assets + ] + if len(set(source_digests)) != len(source_digests): + raise CatalogException( + "The supported bundle catalog has duplicate source digests." + ) + + @classmethod + def from_dict(cls, value: dict[str, Any]) -> "BundleCatalog": + _validate_catalog(value) + return cls(SupportedBundle.from_dict(bundle) for bundle in value["bundles"]) + + def to_dict(self) -> dict[str, Any]: + return { + "$schema": f"./{CATALOG_SCHEMA_RESOURCE}", + "schema_version": 1, + "bundles": [ + bundle.to_dict() + for bundle in sorted(self.bundles, key=lambda item: item.release) + ], + } + + def find_release(self, release: str) -> Optional[SupportedBundle]: + return self._by_release.get(release) + + def find_source_url( + self, url: str + ) -> Optional[tuple[SupportedBundle, SourceAsset]]: + for bundle in self.bundles: + for asset in bundle.source_assets: + if asset.url == url: + return bundle, asset + return None + + def find_source_digest( + self, digest: str + ) -> Optional[tuple[SupportedBundle, SourceAsset]]: + matches = [ + (bundle, asset) + for bundle in self.bundles + for asset in bundle.source_assets + if asset.sha256 == digest + ] + if len(matches) > 1: + raise CatalogException( + f"Source digest {digest} identifies multiple catalog assets." + ) + return matches[0] if matches else None + + def find_fingerprint( + self, cli_version: str, pack_fingerprint: str + ) -> Optional[SupportedBundle]: + matches = [ + bundle + for bundle in self.bundles + if bundle.cli_version == cli_version + and bundle.pack_fingerprint == pack_fingerprint + ] + if len(matches) > 1: + raise CatalogException( + "The bundle CLI version and pack fingerprint identify multiple releases." + ) + return matches[0] if matches else None + + def updated(self, bundle: SupportedBundle) -> "BundleCatalog": + return BundleCatalog( + candidate + for candidate in self.bundles + if candidate.release != bundle.release + ).with_added_bundle(bundle) + + def with_added_bundle(self, bundle: SupportedBundle) -> "BundleCatalog": + return BundleCatalog((*self.bundles, bundle)) + + def write(self, path: Path) -> None: + value = self.to_dict() + _validate_catalog(value) + write_json(path, value) + + @classmethod + def empty(cls) -> "BundleCatalog": + return cls(()) + + +@dataclass(frozen=True) +class ResolvedBundleSource: + path: Path + supported_bundle: Optional[SupportedBundle] + digest: Optional[str] + + +def default_cache_dir() -> Path: + configured = os.environ.get("CODEQL_BUNDLE_CACHE_DIR") + if configured: + return Path(configured).expanduser() + + system = platform.system() + if system == "Windows": + root = os.environ.get("LOCALAPPDATA") + return ( + Path(root) / "codeql-bundle" + if root + else Path.home() / "AppData" / "Local" / "codeql-bundle" + ) + if system == "Darwin": + return Path.home() / "Library" / "Caches" / "codeql-bundle" + + root = os.environ.get("XDG_CACHE_HOME") + return ( + Path(root) / "codeql-bundle" + if root + else Path.home() / ".cache" / "codeql-bundle" + ) + + +def current_bundle_platform() -> str: + system = platform.system() + if system == "Linux": + return "linux64" + if system == "Darwin": + return "osx64" + if system == "Windows": + return "win64" + raise CacheException(f"Unsupported system: {system}") + + +def sha256_file(path: Path) -> str: + digest = sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(DOWNLOAD_CHUNK_SIZE), b""): + digest.update(chunk) + return digest.hexdigest() + + +def compute_pack_fingerprint(cli_version: str, packs: Iterable[CodeQLPack]) -> str: + inventory = [] + for pack in packs: + sources = [] + if pack.path.parent.is_dir(): + for source_path in sorted(pack.path.parent.rglob("*")): + if ( + source_path.is_file() + and source_path.suffix.lower() != ".qlx" + and ".codeql" not in source_path.parts + and ".cache" not in source_path.parts + ): + sources.append( + { + "path": source_path.relative_to( + pack.path.parent + ).as_posix(), + "sha256": sha256_file(source_path), + } + ) + sources.sort(key=lambda source: source["path"]) + inventory.append( + { + "dependencies": { + name: str(version) + for name, version in sorted(pack.config.dependencies.items()) + }, + "extractor": pack.config.extractor, + "library": pack.config.library, + "name": pack.config.name, + "sources": sources, + "version": str(pack.config.version), + } + ) + + encoded = json.dumps( + { + "cli_version": cli_version, + "packs": sorted(inventory, key=lambda item: (item["name"], item["version"])), + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return sha256(encoded).hexdigest() + + +def safe_extract_tar(archive_path: Path, destination: Path) -> None: + destination.mkdir(parents=True, exist_ok=True) + try: + with tarfile.open(archive_path) as archive: + if hasattr(tarfile, "data_filter"): + archive.extractall(destination, filter="data") + return + + root = destination.resolve() + for member in archive.getmembers(): + member_path = (destination / member.name).resolve() + if not member_path.is_relative_to(root): + raise CacheException( + f"Archive {archive_path} contains an unsafe path: {member.name}" + ) + if member.ischr() or member.isblk() or member.isfifo(): + raise CacheException( + f"Archive {archive_path} contains a special file: {member.name}" + ) + if member.issym(): + link_path = (member_path.parent / member.linkname).resolve() + if not link_path.is_relative_to(root): + raise CacheException( + f"Archive {archive_path} contains an unsafe link: {member.name}" + ) + if member.islnk(): + link_path = (destination / member.linkname).resolve() + if not link_path.is_relative_to(root): + raise CacheException( + f"Archive {archive_path} contains an unsafe link: {member.name}" + ) + archive.extractall(destination) + except (OSError, tarfile.TarError) as error: + raise CacheException(f"Failed to extract {archive_path}: {error}") from error + + +def download_file( + url: str, + destination: Path, + *, + expected_sha256: Optional[str] = None, + expected_size: Optional[int] = None, +) -> str: + destination.parent.mkdir(parents=True, exist_ok=True) + + if destination.exists(): + if expected_size is not None and destination.stat().st_size != expected_size: + destination.unlink() + else: + digest = sha256_file(destination) + if expected_sha256 is None or digest == expected_sha256: + _touch(destination) + logger.info(f"Using cached download {destination}.") + return digest + destination.unlink() + + temporary_path = destination.with_name( + f".{destination.name}.{uuid.uuid4().hex}.part" + ) + digest = sha256() + size = 0 + try: + request = Request( + url, headers={"User-Agent": "advanced-security/codeql-bundle"} + ) + with urlopen(request, timeout=60) as response, temporary_path.open("wb") as file: + while chunk := response.read(DOWNLOAD_CHUNK_SIZE): + file.write(chunk) + digest.update(chunk) + size += len(chunk) + except (HTTPError, URLError, OSError, ValueError) as error: + temporary_path.unlink(missing_ok=True) + raise DownloadException(f"Failed to download {url}: {error}") from error + + actual_sha256 = digest.hexdigest() + if expected_size is not None and size != expected_size: + temporary_path.unlink(missing_ok=True) + raise DownloadException( + f"Downloaded {url} with size {size}, expected {expected_size}." + ) + if expected_sha256 is not None and actual_sha256 != expected_sha256: + temporary_path.unlink(missing_ok=True) + raise DownloadException( + f"Downloaded {url} with SHA-256 {actual_sha256}, " + f"expected {expected_sha256}." + ) + + try: + os.replace(temporary_path, destination) + except OSError as error: + temporary_path.unlink(missing_ok=True) + raise DownloadException( + f"Failed to install download at {destination}: {error}" + ) from error + return actual_sha256 + + +class CatalogLoader: + def __init__(self, cache_dir: Optional[Path] = None): + self.cache_dir = cache_dir if cache_dir is not None else default_cache_dir() + + def load(self, source: Optional[str] = None) -> BundleCatalog: + if source: + return BundleCatalog.from_dict(self._read_source(source)) + + cached_catalog_path = self.cache_dir / "catalog" / CATALOG_RESOURCE + if ( + cached_catalog_path.is_file() + and time.time() - cached_catalog_path.stat().st_mtime + < CATALOG_REFRESH_SECONDS + ): + try: + return BundleCatalog.from_dict(_read_json(cached_catalog_path)) + except CacheException as error: + logger.warning( + f"Unable to use the cached bundle catalog: {error}" + ) + + try: + value = self._read_url(DEFAULT_CATALOG_URL) + catalog = BundleCatalog.from_dict(value) + try: + write_json(cached_catalog_path, catalog.to_dict()) + except CacheException as error: + logger.warning( + f"Unable to cache the supported bundle catalog: {error}" + ) + return catalog + except CacheException as error: + if cached_catalog_path.exists(): + try: + catalog = BundleCatalog.from_dict( + _read_json(cached_catalog_path) + ) + logger.warning( + f"Unable to refresh the supported bundle catalog: {error}. " + "Using the last downloaded catalog." + ) + return catalog + except CacheException as cached_error: + logger.warning( + f"Unable to use the downloaded bundle catalog: {cached_error}" + ) + + logger.warning( + f"Unable to download the supported bundle catalog: {error}. " + "Using the catalog bundled with this installation." + ) + return self._load_bundled_catalog() + + @staticmethod + def _load_bundled_catalog() -> BundleCatalog: + try: + value = json.loads( + files("codeql_bundle").joinpath(CATALOG_RESOURCE).read_text() + ) + return BundleCatalog.from_dict(value) + except (CacheException, FileNotFoundError, OSError, TypeError) as error: + logger.warning( + "The packaged supported bundle catalog is unavailable. " + f"Continuing without compilation caches: {error}" + ) + return BundleCatalog.empty() + except json.JSONDecodeError as error: + logger.warning( + "The packaged supported bundle catalog is invalid. " + f"Continuing without compilation caches: {error}" + ) + return BundleCatalog.empty() + + def _read_source(self, source: str) -> dict[str, Any]: + path = Path(source).expanduser() + if path.exists(): + return _read_json(path) + if _is_url(source): + return self._read_url(source) + raise CatalogException(f"Catalog {source} does not exist.") + + def _read_url(self, url: str) -> dict[str, Any]: + try: + request = Request( + url, headers={"User-Agent": "advanced-security/codeql-bundle"} + ) + with urlopen(request, timeout=30) as response: + return json.load(response) + except ( + HTTPError, + URLError, + OSError, + ValueError, + json.JSONDecodeError, + ) as error: + raise CatalogException(f"Failed to read catalog {url}: {error}") from error + + +class GitHubReleaseClient: + def __init__(self, repository: str = CODEQL_ACTION_REPOSITORY): + self.repository = repository + + def release(self, tag: str) -> dict[str, Any]: + url = ( + f"https://api.github.com/repos/{self.repository}/releases/tags/" + f"{quote(tag, safe='')}" + ) + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "advanced-security/codeql-bundle", + "X-GitHub-Api-Version": "2022-11-28", + } + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + try: + with urlopen(Request(url, headers=headers), timeout=30) as response: + return json.load(response) + except ( + HTTPError, + URLError, + OSError, + ValueError, + json.JSONDecodeError, + ) as error: + raise DownloadException( + f"Failed to read release {self.repository}@{tag}: {error}" + ) from error + + def releases(self, limit: int = 100) -> list[dict[str, Any]]: + url = ( + f"https://api.github.com/repos/{self.repository}/releases" + f"?per_page={min(limit, 100)}" + ) + headers = { + "Accept": "application/vnd.github+json", + "User-Agent": "advanced-security/codeql-bundle", + "X-GitHub-Api-Version": "2022-11-28", + } + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + try: + with urlopen(Request(url, headers=headers), timeout=30) as response: + return json.load(response) + except ( + HTTPError, + URLError, + OSError, + ValueError, + json.JSONDecodeError, + ) as error: + raise DownloadException( + f"Failed to list releases for {self.repository}: {error}" + ) from error + + def asset(self, tag: str, name: str) -> dict[str, Any]: + release = self.release(tag) + return self.find_asset(release, name) + + @staticmethod + def find_asset(release: dict[str, Any], name: str) -> dict[str, Any]: + for asset in release.get("assets", []): + if asset["name"] == name: + return asset + raise DownloadException( + f"Release {release.get('tag_name')} does not contain {name}." + ) + + +class BundleSourceResolver: + def __init__( + self, + catalog: BundleCatalog, + cache_dir: Optional[Path] = None, + release_client: Optional[GitHubReleaseClient] = None, + ): + self.catalog = catalog + self.cache_dir = cache_dir if cache_dir is not None else default_cache_dir() + self.release_client = release_client or GitHubReleaseClient() + + def resolve( + self, source: str, requested_platforms: Iterable[str] = () + ) -> ResolvedBundleSource: + path = Path(source).expanduser() + if path.exists(): + return self._resolve_local(path) + if _is_url(source): + return self._resolve_url(source) + if source.startswith("codeql-bundle-"): + validate_release(source) + return self._resolve_release(source, tuple(requested_platforms)) + raise CacheException( + f"Bundle {source} is not a local path, URL, or CodeQL bundle release tag." + ) + + def _resolve_local(self, path: Path) -> ResolvedBundleSource: + path = path.resolve() + if path.is_dir() or not self.catalog.bundles: + return ResolvedBundleSource(path, None, None) + digest = sha256_file(path) + match = self.catalog.find_source_digest(digest) + return ResolvedBundleSource(path, match[0] if match else None, digest) + + def _resolve_url(self, url: str) -> ResolvedBundleSource: + match = self.catalog.find_source_url(url) + if match: + bundle, asset = match + destination = cache_path( + self.cache_dir, "sources", bundle.release, asset.name + ) + digest = download_file( + url, + destination, + expected_sha256=asset.sha256, + expected_size=asset.size, + ) + return ResolvedBundleSource(destination, bundle, digest) + + destination = cache_path( + self.cache_dir, + "sources", + f"url-{sha256(url.encode()).hexdigest()[:16]}", + "codeql-bundle.tar.gz", + ) + digest = download_file(url, destination) + digest_match = self.catalog.find_source_digest(digest) + return ResolvedBundleSource( + destination, digest_match[0] if digest_match else None, digest + ) + + def _resolve_release( + self, release: str, requested_platforms: tuple[str, ...] + ) -> ResolvedBundleSource: + platform_name = source_platform_for_request(requested_platforms) + validate_release(release) + bundle = self.catalog.find_release(release) + if bundle: + asset = bundle.source_asset_for_platform(platform_name) + if asset is None and platform_name != "all": + asset = bundle.source_asset_for_platform("all") + if asset is None: + raise CatalogException( + f"Bundle {release} has no source asset for {platform_name}." + ) + destination = cache_path( + self.cache_dir, "sources", release, asset.name + ) + digest = download_file( + asset.url, + destination, + expected_sha256=asset.sha256, + expected_size=asset.size, + ) + return ResolvedBundleSource(destination, bundle, digest) + + asset_name = source_asset_name(platform_name) + release_value = self.release_client.release(release) + asset = self.release_client.find_asset(release_value, asset_name) + destination = cache_path( + self.cache_dir, "sources", release, asset_name + ) + expected_digest = github_asset_digest(asset) + if expected_digest is None: + checksum_asset = self.release_client.find_asset( + release_value, f"{asset_name}.checksum.txt" + ) + expected_digest = download_checksum( + checksum_asset["browser_download_url"], asset_name + ) + digest = download_file( + asset["browser_download_url"], + destination, + expected_sha256=expected_digest, + expected_size=asset.get("size"), + ) + digest_match = self.catalog.find_source_digest(digest) + return ResolvedBundleSource( + destination, digest_match[0] if digest_match else None, digest + ) + + +class CompilationCacheManager: + def __init__( + self, + supported_bundle: Optional[SupportedBundle], + cache_dir: Optional[Path] = None, + *, + enabled: bool = True, + platform_name: Optional[str] = None, + ): + self.supported_bundle = supported_bundle + self.cache_dir = cache_dir if cache_dir is not None else default_cache_dir() + self.enabled = enabled + self.platform_name = platform_name or current_bundle_platform() + self._warned_unknown_bundle = False + + def cache_for(self, target: str) -> Optional[Path]: + if not self.enabled: + return None + if self.supported_bundle is None: + if not self._warned_unknown_bundle: + logger.warning( + "The source bundle is not in the supported bundle catalog. " + "Continuing without a compilation cache." + ) + self._warned_unknown_bundle = True + return None + if self.platform_name not in self.supported_bundle.validated_platforms: + raise CacheException( + f"Compilation caches for {self.supported_bundle.release} have not " + f"been validated on {self.platform_name}. Use " + "--no-compilation-cache to continue without one." + ) + + cache = self.supported_bundle.compilation_caches.get(target) + if cache is None: + raise CacheException( + f"No compilation cache is published for {target} in " + f"{self.supported_bundle.release}. Use --no-compilation-cache " + "to continue without one." + ) + return self._install(target, cache) + + def _install(self, target: str, cache: CompilationCache) -> Path: + safe_target = _safe_name(target) + install_path = cache_path( + self.cache_dir, + "compilation", + self.supported_bundle.release, + safe_target, + cache.asset.sha256, + ) + marker_path = install_path / ".installed.json" + installed_cache_path = install_path / "cache" + if self._is_installed( + marker_path, installed_cache_path, target, cache.asset.sha256 + ): + _touch(install_path) + return installed_cache_path + + archive_path = cache_path( + self.cache_dir, + "cache-archives", + self.supported_bundle.release, + cache.asset.name, + ) + download_file( + cache.asset.url, + archive_path, + expected_sha256=cache.asset.sha256, + expected_size=cache.asset.size, + ) + + temporary_path = install_path.with_name( + f".{install_path.name}.{uuid.uuid4().hex}.tmp" + ) + shutil.rmtree(temporary_path, ignore_errors=True) + try: + safe_extract_tar(archive_path, temporary_path) + extracted_cache_path = temporary_path / "cache" + if not extracted_cache_path.is_dir(): + raise CacheException( + f"Compilation cache archive {cache.asset.name} has no cache directory." + ) + metadata_path = temporary_path / "metadata.json" + if not metadata_path.is_file(): + raise CacheException( + f"Compilation cache {cache.asset.name} has no metadata." + ) + metadata = _read_json(metadata_path) + if metadata.get("release") != self.supported_bundle.release: + raise CacheException( + f"Compilation cache {cache.asset.name} is for " + f"{metadata.get('release')}, not {self.supported_bundle.release}." + ) + if metadata.get("target") != target: + raise CacheException( + f"Compilation cache {cache.asset.name} is for " + f"{metadata.get('target')}, not {target}." + ) + if metadata.get("cache_format") != CACHE_FORMAT_VERSION: + raise CacheException( + f"Compilation cache {cache.asset.name} uses unsupported " + f"format {metadata.get('cache_format')}." + ) + if metadata.get("cli_version") != self.supported_bundle.cli_version: + raise CacheException( + f"Compilation cache {cache.asset.name} is for CodeQL " + f"{metadata.get('cli_version')}, not " + f"{self.supported_bundle.cli_version}." + ) + if metadata.get("language") != cache.language: + raise CacheException( + f"Compilation cache {cache.asset.name} is for language " + f"{metadata.get('language')}, not {cache.language}." + ) + if tuple(metadata.get("query_packs", ())) != cache.query_packs: + raise CacheException( + f"Compilation cache {cache.asset.name} has an unexpected " + "standard query pack inventory." + ) + + write_json( + temporary_path / ".installed.json", + { + "cache_format": CACHE_FORMAT_VERSION, + "sha256": cache.asset.sha256, + "target": target, + }, + ) + install_path.parent.mkdir(parents=True, exist_ok=True) + lock_path = install_path.parent / f".{install_path.name}.lock" + with _exclusive_lock(lock_path): + if self._is_installed( + marker_path, + installed_cache_path, + target, + cache.asset.sha256, + ): + shutil.rmtree(temporary_path) + _touch(install_path) + return installed_cache_path + quarantine_path = None + if install_path.exists(): + quarantine_path = install_path.with_name( + f".{install_path.name}.{uuid.uuid4().hex}.invalid" + ) + os.rename(install_path, quarantine_path) + try: + os.rename(temporary_path, install_path) + except OSError: + if ( + quarantine_path is not None + and not install_path.exists() + ): + os.rename(quarantine_path, install_path) + raise + if quarantine_path is not None: + shutil.rmtree(quarantine_path, ignore_errors=True) + except (CacheException, OSError): + shutil.rmtree(temporary_path, ignore_errors=True) + raise + return installed_cache_path + + @staticmethod + def _is_installed( + marker_path: Path, + cache_path: Path, + target: str, + digest: str, + ) -> bool: + if not marker_path.is_file() or not cache_path.is_dir(): + return False + try: + marker = _read_json(marker_path) + except CatalogException: + return False + return ( + marker.get("sha256") == digest + and marker.get("target") == target + and marker.get("cache_format") == CACHE_FORMAT_VERSION + ) + + +def _validate_catalog(value: dict[str, Any]) -> None: + try: + schema = json.loads( + files("codeql_bundle").joinpath(CATALOG_SCHEMA_RESOURCE).read_text() + ) + except (FileNotFoundError, OSError, TypeError) as error: + if ( + isinstance(value, dict) + and value.get("schema_version") == 1 + and value.get("bundles") == [] + ): + return + raise CatalogException( + f"The packaged catalog schema is unavailable: {error}" + ) from error + except json.JSONDecodeError as error: + raise CatalogException( + f"The packaged catalog schema is invalid: {error}" + ) from error + errors = sorted( + Draft202012Validator( + schema, format_checker=FormatChecker() + ).iter_errors(value), + key=lambda error: list(error.absolute_path), + ) + if errors: + error = errors[0] + location = ".".join(map(str, error.absolute_path)) or "" + raise CatalogException( + f"Invalid supported bundle catalog at {location}: {error.message}" + ) + for bundle in value["bundles"]: + for asset in bundle["source_assets"]: + validate_remote_url(asset["url"]) + for cache in bundle["compilation_caches"].values(): + validate_remote_url(cache["asset"]["url"]) + + +def _read_json(path: Path) -> dict[str, Any]: + try: + with path.open() as file: + return json.load(file) + except (OSError, json.JSONDecodeError) as error: + raise CatalogException(f"Failed to read {path}: {error}") from error + + +def write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + try: + with temporary_path.open("w") as file: + json.dump(value, file, indent=2) + file.write("\n") + os.replace(temporary_path, path) + except OSError as error: + temporary_path.unlink(missing_ok=True) + raise CacheException(f"Failed to write {path}: {error}") from error + + +def source_asset_name(platform_name: str) -> str: + if platform_name == "all": + return "codeql-bundle.tar.gz" + if platform_name not in {"linux64", "osx64", "win64"}: + raise CacheException(f"Unsupported bundle platform: {platform_name}") + return f"codeql-bundle-{platform_name}.tar.gz" + + +def source_platform_for_request(requested_platforms: Iterable[str]) -> str: + requested = tuple(requested_platforms) + current = current_bundle_platform() + if not requested: + return "all" + if requested == (current,): + return current + return "all" + + +def github_asset_digest(asset: dict[str, Any]) -> Optional[str]: + digest = asset.get("digest") + if not digest: + return None + algorithm, separator, value = digest.partition(":") + if separator and algorithm == "sha256" and re.fullmatch(r"[0-9a-f]{64}", value): + return value + raise DownloadException( + f"Release asset {asset.get('name')} has unsupported digest {digest}." + ) + + +def download_checksum(url: str, expected_name: str) -> str: + try: + request = Request( + url, headers={"User-Agent": "advanced-security/codeql-bundle"} + ) + with urlopen(request, timeout=30) as response: + value = response.read().decode() + except ( + HTTPError, + URLError, + OSError, + UnicodeDecodeError, + ValueError, + ) as error: + raise DownloadException(f"Failed to read checksum {url}: {error}") from error + for line in value.splitlines(): + match = re.fullmatch( + r"([0-9a-fA-F]{64})\s+\*?(.+)", + line.strip(), + ) + if match is not None and match.group(2) == expected_name: + return match.group(1).lower() + raise DownloadException( + f"Checksum asset for {expected_name} has no matching SHA-256 digest." + ) + + +def _is_url(value: str) -> bool: + try: + return urlparse(value).scheme in {"http", "https", "file"} + except ValueError: + return False + + +def _safe_name(value: str) -> str: + return re.sub(r"[^A-Za-z0-9._-]+", "-", value).strip("-") + + +def validate_release(value: str) -> None: + if not RELEASE_PATTERN.fullmatch(value): + raise CacheException(f"Invalid CodeQL bundle release tag: {value}") + + +def validate_cache_release(value: str) -> None: + if not CACHE_RELEASE_PATTERN.fullmatch(value): + raise CacheException(f"Invalid compilation cache release tag: {value}") + + +def validate_remote_url(value: str) -> None: + try: + value.encode("ascii") + parsed = urlparse(value) + hostname = parsed.hostname + except (UnicodeEncodeError, ValueError) as error: + raise CatalogException(f"Invalid catalog URL: {value}") from error + if ( + parsed.scheme not in {"http", "https"} + or not hostname + or parsed.username is not None + or parsed.password is not None + or parsed.fragment + ): + raise CatalogException(f"Invalid catalog URL: {value}") + + +def cache_path(root: Path, *parts: str) -> Path: + resolved_root = root.expanduser().resolve() + candidate = resolved_root.joinpath(*parts).resolve() + if not candidate.is_relative_to(resolved_root): + raise CacheException(f"Cache path escapes {resolved_root}: {candidate}") + return candidate + + +def _touch(path: Path) -> None: + try: + path.touch() + except OSError as error: + logger.debug(f"Unable to update cache access time for {path}: {error}") + + +@contextmanager +def _exclusive_lock(path: Path, timeout: float = 60.0): + deadline = time.monotonic() + timeout + path.parent.mkdir(parents=True, exist_ok=True) + while True: + try: + descriptor = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.close(descriptor) + break + except FileExistsError: + try: + if time.time() - path.stat().st_mtime > timeout: + path.unlink() + continue + except FileNotFoundError: + continue + if time.monotonic() >= deadline: + raise CacheException(f"Timed out waiting for cache lock {path}.") + time.sleep(0.1) + try: + yield + finally: + path.unlink(missing_ok=True) diff --git a/codeql_bundle/cache_cli.py b/codeql_bundle/cache_cli.py new file mode 100644 index 0000000..5b92983 --- /dev/null +++ b/codeql_bundle/cache_cli.py @@ -0,0 +1,864 @@ +from __future__ import annotations + +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Optional +import json +import logging +import re +import shutil +import tarfile +import time + +import click +from semantic_version import Version + +from codeql_bundle.cache import ( + CACHE_FORMAT_VERSION, + CODEQL_ACTION_REPOSITORY, + BundleCatalog, + BundleSourceResolver, + CacheException, + CatalogLoader, + CompilationCache, + CompilationCacheManager, + GitHubReleaseClient, + ReleaseAsset, + SourceAsset, + SupportedBundle, + cache_path, + compute_pack_fingerprint, + current_bundle_platform, + default_cache_dir, + download_checksum, + download_file, + github_asset_digest, + safe_extract_tar, + sha256_file, + source_asset_name, + validate_cache_release, + validate_release, + validate_remote_url, + write_json, +) +from codeql_bundle.helpers.bundle import ( + Bundle, + ResolvedCodeQLPack, + get_compilation_cache_targets, +) + + +logger = logging.getLogger(__name__) + +SOURCE_PLATFORMS = ("all", "linux64", "osx64", "win64") +MAX_RELEASE_ASSET_SIZE = 2 * 1024 * 1024 * 1024 +DEFAULT_COMPILATION_CACHE_SIZE_MB = 1536 + + +@click.group() +def main() -> None: + """Build and manage published CodeQL compilation caches.""" + logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) + + +@main.command("latest-release") +def latest_release() -> None: + """Print the latest stable upstream CodeQL bundle release tag.""" + releases = GitHubReleaseClient().releases() + candidates = [] + for release in releases: + match = re.fullmatch(r"codeql-bundle-v(\d+\.\d+\.\d+)", release["tag_name"]) + if ( + match + and not release.get("draft") + and not release.get("prerelease") + ): + candidates.append((Version(match.group(1)), release["tag_name"])) + if not candidates: + raise click.ClickException("No stable CodeQL bundle release was found.") + click.echo(max(candidates)[1]) + + +@main.command("catalog-has") +@click.option( + "--catalog", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option("--release", required=True) +def catalog_has(catalog: Path, release: str) -> None: + """Print whether a catalog already contains a release.""" + click.echo( + "true" + if CatalogLoader().load(str(catalog)).find_release(release) + else "false" + ) + + +@main.command("prune") +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, +) +@click.option( + "--max-age-days", + type=click.FloatRange(min=0), + default=30.0, + show_default=True, +) +@click.option("--dry-run", is_flag=True) +def prune_cache(cache_dir: Path, max_age_days: float, dry_run: bool) -> None: + """Remove source bundles and compilation caches older than the given age.""" + cutoff = time.time() - max_age_days * 24 * 60 * 60 + removed_bytes = 0 + removed_entries = 0 + for area in ("sources", "cache-archives", "compilation"): + root = cache_path(cache_dir, area) + if not root.is_dir(): + continue + for entry in sorted(root.iterdir()): + modified = _path_modified_time(entry) + if modified > cutoff: + continue + size = _path_size(entry) + action = "Would remove" if dry_run else "Removing" + click.echo(f"{action} {entry} ({size} bytes).") + if not dry_run: + if entry.is_dir() and not entry.is_symlink(): + shutil.rmtree(entry) + else: + entry.unlink(missing_ok=True) + removed_bytes += size + removed_entries += 1 + verb = "Would remove" if dry_run else "Removed" + click.echo(f"{verb} {removed_entries} entries ({removed_bytes} bytes).") + + +@main.command("inspect") +@click.option("--bundle", "bundle_source", required=True) +@click.option( + "--platform", + "platforms", + multiple=True, + type=click.Choice(["linux64", "osx64", "win64"]), +) +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, +) +@click.option("--catalog") +@click.option( + "--output", + type=click.Path(dir_okay=False, path_type=Path), +) +def inspect_bundle( + bundle_source: str, + platforms: tuple[str, ...], + cache_dir: Path, + catalog: Optional[str], + output: Optional[Path], +) -> None: + """Inspect a local bundle, release tag, or bundle URL.""" + loaded_catalog = CatalogLoader(cache_dir).load(catalog) + source = BundleSourceResolver(loaded_catalog, cache_dir).resolve( + bundle_source, platforms + ) + bundle = Bundle(source.path) + supported_bundle = source.supported_bundle or loaded_catalog.find_fingerprint( + str(bundle.codeql.version()), bundle.pack_fingerprint() + ) + value = { + "cli_version": str(bundle.codeql.version()), + "digest": source.digest, + "pack_fingerprint": bundle.pack_fingerprint(), + "platforms": sorted(map(str, bundle.platforms)), + "source_path": str(source.path), + "supported_release": ( + supported_bundle.release if supported_bundle is not None else None + ), + "targets": _target_values(bundle), + } + _emit_json(value, output) + + +@main.command("plan-release") +@click.option("--release", required=True) +@click.option("--cache-release") +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, +) +@click.option( + "--output", + type=click.Path(dir_okay=False, path_type=Path), + required=True, +) +def plan_release( + release: str, + cache_release: Optional[str], + cache_dir: Path, + output: Path, +) -> None: + """Inspect an upstream release and write a deterministic cache build plan.""" + validate_release(release) + cache_release = cache_release or _cache_release_tag(release) + validate_cache_release(cache_release) + client = GitHubReleaseClient() + release_value = client.release(release) + source_assets = _release_source_assets(release_value) + source_asset = next( + asset + for asset in source_assets + if asset.platform == current_bundle_platform() + ) + source_path = cache_path(cache_dir, "sources", release, source_asset.name) + download_file( + source_asset.url, + source_path, + expected_sha256=source_asset.sha256, + expected_size=source_asset.size, + ) + bundle = Bundle(source_path) + plan = { + "cache_format": CACHE_FORMAT_VERSION, + "cache_release": cache_release, + "cli_version": str(bundle.codeql.version()), + "pack_fingerprint": bundle.pack_fingerprint(), + "release": release, + "source_assets": [asset.to_dict() for asset in source_assets], + "source_repository": CODEQL_ACTION_REPOSITORY, + "targets": _target_values(bundle), + } + write_json(output, plan) + click.echo(str(output)) + + +@main.command("build") +@click.option( + "--plan", + "plan_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option("--target", required=True) +@click.option( + "--output-dir", + type=click.Path(file_okay=False, path_type=Path), + required=True, +) +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, +) +@click.option( + "--cache-size", + type=click.IntRange(min=1), + default=DEFAULT_COMPILATION_CACHE_SIZE_MB, + show_default=True, +) +@click.option("--threads", type=int, default=0) +@click.option("--ram", type=click.IntRange(min=1)) +def build_cache( + plan_path: Path, + target: str, + output_dir: Path, + cache_dir: Path, + cache_size: int, + threads: int, + ram: Optional[int], +) -> None: + """Build and verify one cache target from a release plan.""" + plan = _read_plan(plan_path) + target_value = _find_plan_target(plan, target) + asset_name = _cache_asset_name(target_value["language"]) + output_dir.mkdir(parents=True, exist_ok=True) + archive_path = output_dir / asset_name + metadata_path = output_dir / f"{asset_name}.metadata.json" + checksum_path = output_dir / f"{asset_name}.sha256" + + if archive_path.exists() and metadata_path.exists() and checksum_path.exists(): + try: + metadata = _read_json(metadata_path) + digest = sha256_file(archive_path) + checksum_parts = checksum_path.read_text().split() + if ( + metadata.get("release") == plan["release"] + and metadata.get("target") == target + and checksum_parts + and checksum_parts[0] == digest + ): + _emit_json(_built_asset_value(archive_path, metadata), None) + return + except (click.ClickException, OSError) as error: + logger.warning(f"Rebuilding invalid cache output: {error}") + + bundle = _bundle_from_plan(plan, cache_dir) + actual_target, query_packs = _find_bundle_target(bundle, target) + work_cache = ( + output_dir + / ".work" + / plan["release"] + / target_value["language"] + / "cache" + ) + work_cache.mkdir(parents=True, exist_ok=True) + compilation = bundle.codeql.query_compile( + (pack.path.parent for pack in query_packs), + work_cache, + bundle.bundle_path, + threads=threads, + ram=ram, + compilation_cache_size=cache_size, + ) + logger.debug(compilation.stderr) + + representative_query = _representative_query(query_packs) + verification = bundle.codeql.query_compile( + [representative_query], + work_cache, + bundle.bundle_path, + threads=threads, + ram=ram, + compilation_cache_size=cache_size, + ) + if "Compilation cache hit" not in verification.stderr: + raise click.ClickException( + f"Cache verification for {target} did not report a compilation cache hit." + ) + + metadata = { + "cache_format": CACHE_FORMAT_VERSION, + "cli_version": plan["cli_version"], + "language": target_value["language"], + "query_packs": target_value["query_packs"], + "release": plan["release"], + "target": actual_target.config.name, + } + with TemporaryDirectory() as temporary_directory: + temporary_metadata = Path(temporary_directory) / "metadata.json" + write_json(temporary_metadata, metadata) + with tarfile.open(archive_path, mode="w:gz") as archive: + archive.add(work_cache, arcname="cache") + archive.add(temporary_metadata, arcname="metadata.json") + + if archive_path.stat().st_size > MAX_RELEASE_ASSET_SIZE: + archive_path.unlink() + raise click.ClickException( + f"Cache asset {asset_name} exceeds GitHub's release asset size limit." + ) + write_json(metadata_path, metadata) + digest = sha256_file(archive_path) + checksum_path.write_text(f"{digest} {asset_name}\n") + _emit_json(_built_asset_value(archive_path, metadata), None) + + +@main.command("verify") +@click.option( + "--plan", + "plan_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option("--target", required=True) +@click.option( + "--cache", + "cache_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, +) +def verify_cache( + plan_path: Path, target: str, cache_path: Path, cache_dir: Path +) -> None: + """Verify a built cache against the current platform's upstream bundle.""" + plan = _read_plan(plan_path) + bundle = _bundle_from_plan(plan, cache_dir) + _verify_cache_with_bundle(plan, bundle, target, cache_path) + click.echo(f"Verified {target} on {current_bundle_platform()}.") + + +@main.command("verify-all") +@click.option( + "--plan", + "plan_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option( + "--assets-dir", + type=click.Path(exists=True, file_okay=False, path_type=Path), + required=True, +) +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, +) +def verify_all(plan_path: Path, assets_dir: Path, cache_dir: Path) -> None: + """Verify every cache in a release plan on the current platform.""" + plan = _read_plan(plan_path) + bundle = _bundle_from_plan(plan, cache_dir) + for target in plan["targets"]: + cache_path = assets_dir / _cache_asset_name(target["language"]) + if not cache_path.is_file(): + raise click.ClickException(f"Missing cache asset {cache_path}.") + _verify_cache_with_bundle(plan, bundle, target["target"], cache_path) + click.echo( + f"Verified {target['target']} on {current_bundle_platform()}." + ) + + +@main.command("verify-entry") +@click.option( + "--entry", + "entry_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, +) +def verify_entry(entry_path: Path, cache_dir: Path) -> None: + """Download and verify every public cache asset in a catalog entry.""" + bundle = _read_bundle_entry(entry_path) + manager = CompilationCacheManager(bundle, cache_dir) + for target in sorted(bundle.compilation_caches): + manager.cache_for(target) + click.echo(f"Downloaded and verified {target}.") + + +def _verify_cache_with_bundle( + plan: dict[str, Any], + bundle: Bundle, + target: str, + cache_path: Path, +) -> None: + _, query_packs = _find_bundle_target(bundle, target) + with TemporaryDirectory() as temporary_directory: + extracted_path = Path(temporary_directory) + safe_extract_tar(cache_path, extracted_path) + metadata = _read_json(extracted_path / "metadata.json") + if ( + metadata.get("release") != plan["release"] + or metadata.get("target") != target + or metadata.get("cache_format") != CACHE_FORMAT_VERSION + ): + raise click.ClickException( + f"Cache {cache_path} metadata does not match {plan['release']}:{target}." + ) + verification = bundle.codeql.query_compile( + [_representative_query(query_packs)], + extracted_path / "cache", + bundle.bundle_path, + threads=0, + ) + if "Compilation cache hit" not in verification.stderr: + raise click.ClickException( + f"Cache {cache_path} did not report a compilation cache hit." + ) + + +@main.command("catalog-entry") +@click.option( + "--plan", + "plan_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option( + "--assets-dir", + type=click.Path(exists=True, file_okay=False, path_type=Path), + required=True, +) +@click.option("--repository", default="advanced-security/codeql-bundle") +@click.option( + "--validated-platform", + "validated_platforms", + multiple=True, + required=True, + type=click.Choice(["linux64", "osx64", "win64"]), +) +@click.option( + "--output", + type=click.Path(dir_okay=False, path_type=Path), + required=True, +) +def catalog_entry( + plan_path: Path, + assets_dir: Path, + repository: str, + validated_platforms: tuple[str, ...], + output: Path, +) -> None: + """Create a catalog entry for verified release assets.""" + plan = _read_plan(plan_path) + compilation_caches = {} + for target in plan["targets"]: + asset_name = _cache_asset_name(target["language"]) + asset_path = assets_dir / asset_name + if not asset_path.is_file(): + raise click.ClickException(f"Missing cache asset {asset_path}.") + asset = ReleaseAsset( + name=asset_name, + url=( + f"https://github.com/{repository}/releases/download/" + f"{plan['cache_release']}/{asset_name}" + ), + sha256=sha256_file(asset_path), + size=asset_path.stat().st_size, + ) + compilation_caches[target["target"]] = CompilationCache( + language=target["language"], + query_packs=tuple(target["query_packs"]), + asset=asset, + ) + + bundle = SupportedBundle( + release=plan["release"], + cli_version=plan["cli_version"], + source_repository=plan["source_repository"], + pack_fingerprint=plan["pack_fingerprint"], + source_assets=tuple( + SourceAsset.from_dict(asset) for asset in plan["source_assets"] + ), + cache_release=plan["cache_release"], + cache_format=CACHE_FORMAT_VERSION, + compilation_caches=compilation_caches, + validated_platforms=tuple(validated_platforms), + ) + BundleCatalog.from_dict( + {"schema_version": 1, "bundles": [bundle.to_dict()]} + ) + write_json(output, bundle.to_dict()) + click.echo(str(output)) + + +@main.command("update-catalog") +@click.option( + "--catalog", + "catalog_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option( + "--entry", + "entry_path", + type=click.Path(exists=True, dir_okay=False, path_type=Path), + required=True, +) +@click.option( + "--output", + type=click.Path(dir_okay=False, path_type=Path), +) +def update_catalog( + catalog_path: Path, entry_path: Path, output: Optional[Path] +) -> None: + """Insert or replace a verified entry in the authoritative catalog.""" + catalog = CatalogLoader().load(str(catalog_path)) + bundle = _read_bundle_entry(entry_path) + destination = output or catalog_path + catalog.updated(bundle).write(destination) + click.echo(str(destination)) + + +def _release_source_assets(release: dict[str, Any]) -> tuple[SourceAsset, ...]: + assets = {asset["name"]: asset for asset in release.get("assets", [])} + result = [] + for platform_name in SOURCE_PLATFORMS: + name = source_asset_name(platform_name) + asset = assets.get(name) + if asset is None: + raise click.ClickException( + f"Upstream release {release['tag_name']} has no {name}." + ) + digest = github_asset_digest(asset) + if digest is None: + checksum_asset = assets.get(f"{name}.checksum.txt") + if checksum_asset is None: + raise click.ClickException( + f"Upstream release asset {name} has no SHA-256 digest." + ) + digest = download_checksum( + checksum_asset["browser_download_url"], expected_name=name + ) + result.append( + SourceAsset( + name=name, + url=asset["browser_download_url"], + sha256=digest, + size=asset["size"], + platform=platform_name, + ) + ) + return tuple(result) + + +def _target_values(bundle: Bundle) -> list[dict[str, Any]]: + targets = get_compilation_cache_targets(bundle.get_bundle_packs()) + values = [] + languages = set() + for target, query_packs in sorted( + targets.items(), key=lambda item: item[0].config.name + ): + language = target.config.get_pack_name().removesuffix("-all") + if language in languages: + raise CacheException( + f"Multiple compilation cache targets map to language {language}." + ) + languages.add(language) + values.append( + { + "language": language, + "query_packs": [ + f"{pack.config.name}@{pack.config.version}" + for pack in sorted(query_packs, key=lambda item: item.config.name) + ], + "target": target.config.name, + } + ) + return values + + +def _bundle_from_plan(plan: dict[str, Any], cache_dir: Path) -> Bundle: + validate_release(plan["release"]) + platform_name = current_bundle_platform() + asset = next( + ( + SourceAsset.from_dict(value) + for value in plan["source_assets"] + if value["platform"] == platform_name + ), + None, + ) + if asset is None: + raise click.ClickException( + f"Release plan has no source bundle for {platform_name}." + ) + validate_remote_url(asset.url) + source_path = cache_path( + cache_dir, "sources", plan["release"], asset.name + ) + download_file( + asset.url, + source_path, + expected_sha256=asset.sha256, + expected_size=asset.size, + ) + bundle = Bundle(source_path) + fingerprint = compute_pack_fingerprint( + str(bundle.codeql.version()), bundle.get_bundle_packs() + ) + if ( + str(bundle.codeql.version()) != plan["cli_version"] + or fingerprint != plan["pack_fingerprint"] + ): + raise click.ClickException( + f"Source bundle for {plan['release']} does not match its release plan." + ) + return bundle + + +def _find_bundle_target( + bundle: Bundle, target_name: str +) -> tuple[ResolvedCodeQLPack, list[ResolvedCodeQLPack]]: + targets = get_compilation_cache_targets(bundle.get_bundle_packs()) + for target, query_packs in targets.items(): + if target.config.name == target_name: + return target, query_packs + raise click.ClickException(f"Bundle has no cache target {target_name}.") + + +def _find_plan_target(plan: dict[str, Any], target_name: str) -> dict[str, Any]: + for target in plan["targets"]: + if target["target"] == target_name: + return target + raise click.ClickException(f"Release plan has no cache target {target_name}.") + + +def _representative_query(query_packs: list[ResolvedCodeQLPack]) -> Path: + for query_pack in sorted(query_packs, key=lambda pack: pack.config.name): + for query in sorted(query_pack.path.parent.rglob("*.ql")): + relative_path = query.relative_to(query_pack.path.parent) + if ( + ".codeql" not in relative_path.parts + and ".cache" not in relative_path.parts + ): + return query + raise click.ClickException("No standard query was found for cache verification.") + + +def _cache_release_tag(release: str) -> str: + return f"codeql-compilation-cache-{release.removeprefix('codeql-bundle-')}" + + +def _cache_asset_name(language: str) -> str: + return f"codeql-compilation-cache-{language}.tar.gz" + + +def _built_asset_value( + archive_path: Path, metadata: dict[str, Any] +) -> dict[str, Any]: + return { + "asset": archive_path.name, + "language": metadata["language"], + "sha256": sha256_file(archive_path), + "size": archive_path.stat().st_size, + "target": metadata["target"], + } + + +def _path_size(path: Path) -> int: + if path.is_symlink() or path.is_file(): + try: + return path.stat(follow_symlinks=False).st_size + except FileNotFoundError: + return 0 + total = 0 + for candidate in path.rglob("*"): + if candidate.is_file() and not candidate.is_symlink(): + try: + total += candidate.stat().st_size + except FileNotFoundError: + continue + return total + + +def _path_modified_time(path: Path) -> float: + try: + modified = path.stat(follow_symlinks=False).st_mtime + except FileNotFoundError: + return 0 + if path.is_dir() and not path.is_symlink(): + for candidate in path.rglob("*"): + try: + modified = max( + modified, + candidate.stat(follow_symlinks=False).st_mtime, + ) + except FileNotFoundError: + continue + return modified + + +def _read_json(path: Path) -> dict[str, Any]: + try: + with path.open() as file: + return json.load(file) + except (OSError, json.JSONDecodeError) as error: + raise click.ClickException(f"Failed to read {path}: {error}") from error + + +def _read_plan(path: Path) -> dict[str, Any]: + value = _read_json(path) + try: + if ( + type(value["cache_format"]) is not int + or value["cache_format"] != CACHE_FORMAT_VERSION + ): + raise ValueError( + f"unsupported cache format {value['cache_format']}" + ) + validate_release(value["release"]) + validate_cache_release(value["cache_release"]) + if not isinstance(value["cli_version"], str): + raise ValueError("invalid CLI version") + Version(value["cli_version"]) + if not re.fullmatch(r"[0-9a-f]{64}", value["pack_fingerprint"]): + raise ValueError("invalid pack fingerprint") + if value["source_repository"] != CODEQL_ACTION_REPOSITORY: + raise ValueError("invalid source repository") + + source_platforms = set() + for source_value in value["source_assets"]: + source = SourceAsset.from_dict(source_value) + if source.name != source_asset_name(source.platform): + raise ValueError( + f"unexpected source asset {source.name} for {source.platform}" + ) + if ( + not re.fullmatch(r"[0-9a-f]{64}", source.sha256) + or source.size < 1 + ): + raise ValueError(f"invalid source asset {source.name}") + validate_remote_url(source.url) + source_platforms.add(source.platform) + if ( + source_platforms != set(SOURCE_PLATFORMS) + or len(value["source_assets"]) != len(SOURCE_PLATFORMS) + ): + raise ValueError("incomplete source platform inventory") + + target_names = set() + languages = set() + if not isinstance(value["targets"], list) or not value["targets"]: + raise ValueError("empty cache target inventory") + for target in value["targets"]: + if ( + not re.fullmatch( + r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + target["target"], + ) + or not re.fullmatch(r"[a-z][a-z0-9-]*", target["language"]) + or not isinstance(target["query_packs"], list) + or not target["query_packs"] + or not all( + isinstance(query_pack, str) and "@" in query_pack + for query_pack in target["query_packs"] + ) + ): + raise ValueError("invalid cache target") + target_names.add(target["target"]) + languages.add(target["language"]) + if ( + len(target_names) != len(value["targets"]) + or len(languages) != len(value["targets"]) + ): + raise ValueError("duplicate cache target") + except (KeyError, TypeError, ValueError, CacheException) as error: + raise click.ClickException( + f"Invalid release plan {path}: {error}" + ) from error + return value + + +def _read_bundle_entry(path: Path) -> SupportedBundle: + catalog = BundleCatalog.from_dict( + {"schema_version": 1, "bundles": [_read_json(path)]} + ) + return catalog.bundles[0] + + +def _emit_json(value: dict[str, Any], output: Optional[Path]) -> None: + if output is not None: + write_json(output, value) + else: + click.echo(json.dumps(value, sort_keys=True)) + + +def run() -> None: + try: + main() + except CacheException as error: + click.echo(f"Error: {error}", err=True) + raise SystemExit(1) from error + + +if __name__ == "__main__": + run() diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index db6372d..d8d8c1b 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -11,6 +11,14 @@ from pathlib import Path from codeql_bundle.helpers.codeql import CodeQLException from codeql_bundle.helpers.bundle import CustomBundle, BundleException, BundlePlatform +from codeql_bundle.cache import ( + BundleCatalog, + BundleSourceResolver, + CacheException, + CatalogLoader, + CompilationCacheManager, + default_cache_dir, +) from typing import List, Optional import sys import logging @@ -24,8 +32,8 @@ "--bundle", "bundle_path", required=True, - help="Path to a CodeQL bundle downloaded from https://github.com/github/codeql-action/releases", - type=click.Path(exists=True, path_type=Path), + help="Path, URL, or github/codeql-action release tag for the source CodeQL bundle", + type=click.STRING, ) @click.option( "-o", @@ -78,10 +86,27 @@ type=click.INT, help="Use this many threads to compile queries.", ) +@click.option( + "--cache-dir", + type=click.Path(path_type=Path), + default=default_cache_dir, + show_default=True, + help="Directory used for downloaded bundles and compilation caches.", +) +@click.option( + "--cache-manifest", + type=click.STRING, + help="Path or URL of a supported bundle manifest.", +) +@click.option( + "--no-compilation-cache", + is_flag=True, + help="Do not resolve, download, or use published compilation caches.", +) @click.argument("packs", nargs=-1, required=True) def main( - bundle_path: Path, + bundle_path: str, output: Path, workspace: Path, no_precompile: bool, @@ -90,6 +115,9 @@ def main( code_scanning_config: Optional[Path], additional_data_config: Optional[Path], threads: Optional[int], + cache_dir: Path, + cache_manifest: Optional[str], + no_compilation_cache: bool, packs: List[str], ) -> None: @@ -105,16 +133,39 @@ def main( ) workspace = Path(os.path.abspath(workspace)) + cache_dir = Path(os.path.abspath(cache_dir)) if workspace.name == "codeql-workspace.yml": workspace = workspace.parent - logger.info( - f"Creating custom bundle of {bundle_path} using CodeQL pack(s) in workspace {workspace}" - ) - try: - bundle = CustomBundle(bundle_path, workspace) + use_compilation_cache = not no_compilation_cache and not no_precompile + catalog = ( + CatalogLoader(cache_dir).load(cache_manifest) + if use_compilation_cache + else BundleCatalog.empty() + ) + source = BundleSourceResolver(catalog, cache_dir).resolve( + bundle_path, platform + ) + logger.info( + f"Creating custom bundle of {source.path} using CodeQL pack(s) in workspace {workspace}" + ) + bundle = CustomBundle(source.path, workspace) + supported_bundle = source.supported_bundle + if ( + use_compilation_cache + and supported_bundle is None + and source.path.is_dir() + ): + supported_bundle = catalog.find_fingerprint( + str(bundle.codeql.version()), bundle.pack_fingerprint() + ) + bundle.compilation_cache_manager = CompilationCacheManager( + supported_bundle, + cache_dir, + enabled=use_compilation_cache, + ) # options for custom bundle bundle.disable_precompilation = no_precompile bundle.threads = threads @@ -181,6 +232,9 @@ def main( except BundleException as e: logger.fatal(f"Failed to build custom bundle with reason: '{e}'") sys.exit(1) + except CacheException as e: + logger.fatal(f"Failed to resolve bundle cache with reason: '{e}'") + sys.exit(1) if __name__ == "__main__": diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 2a7d205..24f2975 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -2,7 +2,7 @@ from pathlib import Path from tempfile import TemporaryDirectory import tarfile -from typing import List, cast, Callable, Optional +from typing import Iterable, List, cast, Callable, Optional from collections import defaultdict import shutil import yaml @@ -17,6 +17,11 @@ from graphlib import TopologicalSorter import platform import concurrent.futures +from codeql_bundle.cache import ( + CompilationCacheManager, + compute_pack_fingerprint, + safe_extract_tar, +) logger = logging.getLogger(__name__) @@ -145,6 +150,50 @@ def inner(pack_to_be_resolved: CodeQLPack) -> ResolvedCodeQLPack: return builder() +def is_dependent_on( + pack: ResolvedCodeQLPack, + other: ResolvedCodeQLPack, + visited: Optional[set[ResolvedCodeQLPack]] = None, +) -> bool: + if other in pack.dependencies: + return True + visited = set() if visited is None else visited + if pack in visited: + return False + visited.add(pack) + return any( + is_dependent_on(dependency, other, visited) + for dependency in pack.dependencies + ) + + +def get_compilation_cache_targets( + packs: Iterable[ResolvedCodeQLPack], +) -> dict[ResolvedCodeQLPack, List[ResolvedCodeQLPack]]: + packs = list(packs) + query_packs = [ + pack + for pack in packs + if pack.kind == CodeQLPackKind.QUERY_PACK + and pack.config.get_scope() == "codeql" + ] + targets = {} + for pack in packs: + if ( + pack.config.library + and pack.config.get_scope() == "codeql" + and pack.config.get_pack_name().endswith("-all") + ): + dependent_query_packs = [ + query_pack + for query_pack in query_packs + if is_dependent_on(query_pack, pack) + ] + if dependent_query_packs: + targets[pack] = dependent_query_packs + return targets + + @verify(UNIQUE) class BundlePlatform(Enum): LINUX = 1 @@ -188,8 +237,7 @@ def __init__(self, bundle_path: Path) -> None: logging.info( f"Unpacking provided bundle {bundle_path} to {self.tmp_dir.name}." ) - file = tarfile.open(bundle_path) - file.extractall(self.tmp_dir.name) + safe_extract_tar(bundle_path, Path(self.tmp_dir.name)) self.bundle_path = Path(self.tmp_dir.name) / "codeql" else: raise BundleException("Invalid CodeQL bundle path") @@ -260,6 +308,9 @@ def __del__(self) -> None: def get_bundle_packs(self) -> List[ResolvedCodeQLPack]: return self.bundle_packs + def pack_fingerprint(self) -> str: + return compute_pack_fingerprint(str(self.codeql.version()), self.bundle_packs) + def supports_platform(self, platform: BundlePlatform) -> bool: return platform in self.platforms @@ -287,8 +338,14 @@ def threads(self, value: int): self.codeql.threads= value class CustomBundle(Bundle): - def __init__(self, bundle_path: Path, workspace_path: Path = Path.cwd()) -> None: + def __init__( + self, + bundle_path: Path, + workspace_path: Path = Path.cwd(), + compilation_cache_manager: Optional[CompilationCacheManager] = None, + ) -> None: Bundle.__init__(self, bundle_path) + self.compilation_cache_manager = compilation_cache_manager packs: List[CodeQLPack] = self.codeql.pack_ls(workspace_path) # Perform a sanity check on the packs in the workspace. @@ -394,12 +451,7 @@ def add_to_graph( if not pack in processed_packs: add_to_graph(pack, processed_packs, std_lib_deps) - def is_dependent_on( - pack: ResolvedCodeQLPack, other: ResolvedCodeQLPack - ) -> bool: - return other in pack.dependencies or any( - map(lambda p: is_dependent_on(p, other), pack.dependencies) - ) + query_pack_caches: dict[ResolvedCodeQLPack, List[Path]] = defaultdict(list) # Add the stdlib and its dependencies to properly sort the customization packs before the other packs. for pack, deps in std_lib_deps.items(): @@ -408,15 +460,27 @@ def is_dependent_on( ) pack_sorter.add(pack, *deps) # Add the standard query packs that rely transitively on the stdlib. - for query_pack in [ + dependent_query_packs = [ p for p in self.bundle_packs if p.kind == CodeQLPackKind.QUERY_PACK and is_dependent_on(p, pack) - ]: + ] + for query_pack in dependent_query_packs: logger.debug( f"Adding standard query pack {query_pack.config.name}@{str(query_pack.config.version)} to dependency graph" ) pack_sorter.add(query_pack, pack) + if ( + self.compilation_cache_manager is not None + and not self.disable_precompilation + and dependent_query_packs + ): + compilation_cache = self.compilation_cache_manager.cache_for( + pack.config.name + ) + if compilation_cache is not None: + for query_pack in dependent_query_packs: + query_pack_caches[query_pack].append(compilation_cache) def bundle_customization_pack(customization_pack: ResolvedCodeQLPack): logging.info( @@ -435,6 +499,7 @@ def bundle_customization_pack(customization_pack: ResolvedCodeQLPack): qlpack_spec["dependencies"] = {} with customization_pack_copy.path.open("w") as fd: yaml.dump(qlpack_spec, fd) + customization_pack_copy.get_lock_file_path().unlink(missing_ok=True) logging.debug( f"Bundling the customization pack {customization_pack_copy.config.name} at {customization_pack_copy.path}" @@ -622,7 +687,8 @@ def bundle_query_pack(pack: ResolvedCodeQLPack): # Recompile the query pack with the assumption that all its dependencies are now in the bundle. self.codeql.pack_create( pack_copy, self.bundle_path / "qlpacks", self.bundle_path, - disable_precompilation=self.disable_precompilation + disable_precompilation=self.disable_precompilation, + compilation_caches=query_pack_caches[pack], ) else: logging.info(f"Bundling the query pack {pack.config.name}.") diff --git a/codeql_bundle/helpers/codeql.py b/codeql_bundle/helpers/codeql.py index c65849e..3f052af 100644 --- a/codeql_bundle/helpers/codeql.py +++ b/codeql_bundle/helpers/codeql.py @@ -6,6 +6,7 @@ import yaml from dataclasses import dataclass, fields, field import logging +import os logger = logging.getLogger(__name__) @@ -92,6 +93,37 @@ def _exec(self, command: str, *args: str) -> subprocess.CompletedProcess[str]: text=True ) + def _exec_streaming( + self, command: str, *args: str + ) -> subprocess.CompletedProcess[str]: + command_args = [f"{self.codeql_path}", command, *args] + logger.debug( + f"Running CodeQL command: {command} with arguments: {' '.join(args)}" + ) + try: + process = subprocess.Popen( + command_args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + except OSError as error: + raise CodeQLException( + f"Failed to run {command_args}: {error}" + ) from error + output = [] + if process.stdout is None: + raise CodeQLException(f"Failed to capture output from {command_args}!") + for line in process.stdout: + output.append(line) + logger.info(line.rstrip()) + return subprocess.CompletedProcess( + command_args, + process.wait(), + stdout="", + stderr="".join(output), + ) + def version(self) -> Version: if self._version != None: return self._version @@ -141,8 +173,8 @@ def pack_bundle( pack: CodeQLPack, output_path: Path, *additional_packs: Path, - disable_precompilation = False - ): + disable_precompilation: bool = False, + ) -> None: if not pack.config.library: raise CodeQLException(f"Cannot bundle non-library pack {pack.config.name}!") @@ -154,7 +186,9 @@ def pack_bundle( ) if len(additional_packs) > 0: - args.append(f"--additional-packs={':'.join(map(str,additional_packs))}") + args.append( + f"--additional-packs={os.pathsep.join(map(str, additional_packs))}" + ) cp = self._exec( "pack", *args, @@ -170,8 +204,9 @@ def pack_create( pack: CodeQLPack, output_path: Path, *additional_packs: Path, - disable_precompilation = False - ): + disable_precompilation: bool = False, + compilation_caches: Iterable[Path] = (), + ) -> None: if pack.config.library: raise CodeQLException(f"Cannot bundle non-query pack {pack.config.name}!") @@ -191,8 +226,12 @@ def pack_create( if self.supports_qlx(): args.append("--qlx") + for compilation_cache in compilation_caches: + args.append(f"--compilation-cache={compilation_cache}") if len(additional_packs) > 0: - args.append(f"--additional-packs={':'.join(map(str,additional_packs))}") + args.append( + f"--additional-packs={os.pathsep.join(map(str, additional_packs))}" + ) cp = self._exec( "pack", *args, @@ -202,6 +241,36 @@ def pack_create( if cp.returncode != 0: raise CodeQLException(f"Failed to run {cp.args} command! {cp.stderr}") + + def query_compile( + self, + queries: Iterable[Path], + compilation_cache: Path, + *additional_packs: Path, + threads: int = 0, + ram: Optional[int] = None, + compilation_cache_size: Optional[int] = None, + ) -> subprocess.CompletedProcess[str]: + args = [ + "compile", + "--keep-going", + f"--threads={threads}", + "--no-default-compilation-cache", + f"--compilation-cache={compilation_cache}", + "--verbosity=progress+++", + ] + if ram is not None: + args.append(f"--ram={ram}") + if compilation_cache_size is not None: + args.append(f"--compilation-cache-size={compilation_cache_size}") + if additional_packs: + args.append( + f"--additional-packs={os.pathsep.join(map(str, additional_packs))}" + ) + cp = self._exec_streaming("query", *args, "--", *map(str, queries)) + if cp.returncode != 0: + raise CodeQLException(f"Failed to run {cp.args} command! {cp.stderr}") + return cp def resolve_languages(self) -> set[str]: cp = self._exec("resolve", "languages", "--format=json") diff --git a/codeql_bundle/supported-codeql-bundles.json b/codeql_bundle/supported-codeql-bundles.json new file mode 100644 index 0000000..87910cb --- /dev/null +++ b/codeql_bundle/supported-codeql-bundles.json @@ -0,0 +1,5 @@ +{ + "$schema": "./supported-codeql-bundles.schema.json", + "schema_version": 1, + "bundles": [] +} diff --git a/codeql_bundle/supported-codeql-bundles.schema.json b/codeql_bundle/supported-codeql-bundles.schema.json new file mode 100644 index 0000000..d69afd4 --- /dev/null +++ b/codeql_bundle/supported-codeql-bundles.schema.json @@ -0,0 +1,188 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/advanced-security/codeql-bundle/main/codeql_bundle/supported-codeql-bundles.schema.json", + "title": "Supported CodeQL bundles", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "bundles" + ], + "properties": { + "$schema": { + "type": "string" + }, + "schema_version": { + "const": 1 + }, + "bundles": { + "type": "array", + "items": { + "$ref": "#/$defs/bundle" + } + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "asset": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "url", + "sha256", + "size" + ], + "properties": { + "name": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "url": { + "type": "string", + "format": "uri" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "size": { + "type": "integer", + "minimum": 1 + } + } + }, + "sourceAsset": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "url", + "sha256", + "size", + "platform" + ], + "properties": { + "name": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "url": { + "type": "string", + "format": "uri" + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "size": { + "type": "integer", + "minimum": 1 + }, + "platform": { + "enum": [ + "all", + "linux64", + "osx64", + "win64" + ] + } + } + }, + "cache": { + "type": "object", + "additionalProperties": false, + "required": [ + "language", + "query_packs", + "asset" + ], + "properties": { + "language": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]*$" + }, + "query_packs": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "asset": { + "$ref": "#/$defs/asset" + } + } + }, + "bundle": { + "type": "object", + "additionalProperties": false, + "required": [ + "release", + "cli_version", + "source_repository", + "pack_fingerprint", + "source_assets", + "cache_release", + "cache_format", + "compilation_caches", + "validated_platforms" + ], + "properties": { + "release": { + "type": "string", + "pattern": "^codeql-bundle-v[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "cli_version": { + "type": "string", + "minLength": 1 + }, + "source_repository": { + "type": "string", + "pattern": "^[^/]+/[^/]+$" + }, + "pack_fingerprint": { + "$ref": "#/$defs/sha256" + }, + "source_assets": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/sourceAsset" + } + }, + "cache_release": { + "type": "string", + "pattern": "^codeql-compilation-cache-v[0-9]+\\.[0-9]+\\.[0-9]+(?:-[A-Za-z0-9._-]+)?$" + }, + "cache_format": { + "const": 1 + }, + "compilation_caches": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" + }, + "additionalProperties": { + "$ref": "#/$defs/cache" + } + }, + "validated_platforms": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "enum": [ + "linux64", + "osx64", + "win64" + ] + } + } + } + } + } +} diff --git a/pyproject.toml b/pyproject.toml index 036f1d2..664a87d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "codeql-bundle" -version = "0.4.2" +version = "0.5.0" description = "Tool to create custom CodeQL bundles" authors = ["Remco Vermeulen "] readme = "README.md" @@ -17,6 +17,7 @@ jsonschema = "^4.22.0" [tool.poetry.scripts] codeql-bundle = 'codeql_bundle.cli:main' +codeql-bundle-cache = 'codeql_bundle.cache_cli:run' [build-system] requires = ["poetry-core"] diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..748e757 --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,526 @@ +from contextlib import contextmanager +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from io import BytesIO +from pathlib import Path +from tempfile import TemporaryDirectory +from threading import Thread +import json +import tarfile +import unittest +from unittest.mock import patch + +from semantic_version import Version + +from codeql_bundle.cache import ( + CACHE_FORMAT_VERSION, + BundleCatalog, + BundleSourceResolver, + CacheException, + CatalogException, + CatalogLoader, + CompilationCache, + CompilationCacheManager, + DownloadException, + GitHubReleaseClient, + ReleaseAsset, + SourceAsset, + SupportedBundle, + compute_pack_fingerprint, + current_bundle_platform, + safe_extract_tar, + sha256_file, + source_platform_for_request, +) +from codeql_bundle.helpers.codeql import CodeQLPack, CodeQLPackConfig + + +class CountingHandler(SimpleHTTPRequestHandler): + requests = 0 + + def do_GET(self) -> None: + type(self).requests += 1 + super().do_GET() + + def log_message(self, format: str, *args: object) -> None: + pass + + +@contextmanager +def serve(directory: Path): + CountingHandler.requests = 0 + server = ThreadingHTTPServer( + ("127.0.0.1", 0), partial(CountingHandler, directory=str(directory)) + ) + thread = Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join() + server.server_close() + + +def bundle_with_assets( + source: SourceAsset, cache: ReleaseAsset +) -> SupportedBundle: + return SupportedBundle( + release="codeql-bundle-v1.2.3", + cli_version="1.2.3", + source_repository="github/codeql-action", + pack_fingerprint="0" * 64, + source_assets=(source,), + cache_release="codeql-compilation-cache-v1.2.3", + cache_format=CACHE_FORMAT_VERSION, + compilation_caches={ + "codeql/cpp-all": CompilationCache( + language="cpp", + query_packs=("codeql/cpp-queries@1.2.3",), + asset=cache, + ) + }, + validated_platforms=(current_bundle_platform(),), + ) + + +class CatalogTests(unittest.TestCase): + def test_missing_packaged_resources_fall_back_to_empty_catalog(self) -> None: + with TemporaryDirectory() as directory: + loader = CatalogLoader(Path(directory)) + with patch( + "codeql_bundle.cache.CatalogLoader._read_url", + side_effect=CatalogException("offline"), + ), patch( + "codeql_bundle.cache.files", + side_effect=FileNotFoundError("not frozen"), + ): + catalog = loader.load() + + self.assertEqual((), catalog.bundles) + self.assertFalse( + (Path(directory) / "catalog" / "supported-codeql-bundles.json").exists() + ) + + def test_catalog_round_trip_and_update(self) -> None: + source = SourceAsset( + name="codeql-bundle.tar.gz", + url="https://example.test/codeql-bundle.tar.gz", + sha256="1" * 64, + size=100, + platform="all", + ) + cache = ReleaseAsset( + name="codeql-compilation-cache-cpp.tar.gz", + url="https://example.test/codeql-compilation-cache-cpp.tar.gz", + sha256="2" * 64, + size=50, + ) + bundle = bundle_with_assets(source, cache) + + with TemporaryDirectory() as directory: + path = Path(directory) / "catalog.json" + BundleCatalog([]).updated(bundle).write(path) + loaded = CatalogLoader().load(str(path)) + + self.assertEqual(bundle, loaded.find_release(bundle.release)) + self.assertEqual( + bundle, + loaded.find_source_digest(source.sha256)[0], + ) + + def test_catalog_rejects_invalid_digest(self) -> None: + value = { + "schema_version": 1, + "bundles": [ + { + "release": "codeql-bundle-v1.2.3", + "cli_version": "1.2.3", + "source_repository": "github/codeql-action", + "pack_fingerprint": "invalid", + "source_assets": [], + "cache_release": "cache", + "cache_format": 1, + "compilation_caches": {}, + "validated_platforms": [], + } + ], + } + with self.assertRaises(CatalogException): + BundleCatalog.from_dict(value) + + def test_catalog_rejects_unsafe_asset_name_and_url(self) -> None: + source = SourceAsset( + name="codeql-bundle.tar.gz", + url="https://example.test/codeql-bundle.tar.gz", + sha256="1" * 64, + size=100, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url="https://example.test/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + value = { + "schema_version": 1, + "bundles": [bundle_with_assets(source, cache).to_dict()], + } + value["bundles"][0]["source_assets"][0]["name"] = "../../outside" + with self.assertRaises(CatalogException): + BundleCatalog.from_dict(value) + + value["bundles"][0]["source_assets"][0]["name"] = "bundle.tar.gz" + value["bundles"][0]["source_assets"][0]["url"] = "not a URL" + with self.assertRaises(CatalogException): + BundleCatalog.from_dict(value) + + def test_pack_fingerprint_is_path_independent(self) -> None: + config = CodeQLPackConfig( + name="codeql/example-all", + version=Version("1.0.0"), + library=True, + ) + first = CodeQLPack(path=Path("/one/qlpack.yml"), config=config) + second = CodeQLPack(path=Path("/two/qlpack.yml"), config=config) + self.assertEqual( + compute_pack_fingerprint("2.0.0", [first]), + compute_pack_fingerprint("2.0.0", [second]), + ) + + def test_pack_fingerprint_changes_with_source(self) -> None: + with TemporaryDirectory() as directory: + pack_path = Path(directory) / "qlpack.yml" + pack_path.write_text("name: codeql/example-all\n") + source_path = pack_path.parent / "Example.qll" + source_path.write_text("class Example extends string {}\n") + pack = CodeQLPack( + path=pack_path, + config=CodeQLPackConfig( + name="codeql/example-all", + version=Version("1.0.0"), + library=True, + ), + ) + first = compute_pack_fingerprint("2.0.0", [pack]) + source_path.write_text("class Changed extends string {}\n") + second = compute_pack_fingerprint("2.0.0", [pack]) + + self.assertNotEqual(first, second) + + +class ArtifactTests(unittest.TestCase): + def test_empty_catalog_does_not_hash_local_archive(self) -> None: + with TemporaryDirectory() as directory: + archive = Path(directory) / "codeql-bundle.tar.gz" + archive.write_bytes(b"bundle") + with patch( + "codeql_bundle.cache.sha256_file", + side_effect=AssertionError("unexpected hash"), + ): + resolved = BundleSourceResolver( + BundleCatalog.empty(), Path(directory) / "cache" + ).resolve(str(archive)) + + self.assertIsNone(resolved.digest) + self.assertIsNone(resolved.supported_bundle) + + def test_release_source_is_runnable_on_current_platform(self) -> None: + current = current_bundle_platform() + other = next( + platform + for platform in ("linux64", "osx64", "win64") + if platform != current + ) + self.assertEqual("all", source_platform_for_request(())) + self.assertEqual(current, source_platform_for_request((current,))) + self.assertEqual("all", source_platform_for_request((other,))) + self.assertEqual( + "all", source_platform_for_request(("linux64", "win64")) + ) + + def test_local_archive_matches_catalog_digest(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + archive = root / "codeql-bundle.tar.gz" + archive.write_bytes(b"bundle") + source = SourceAsset( + name=archive.name, + url="https://example.test/codeql-bundle.tar.gz", + sha256=sha256_file(archive), + size=archive.stat().st_size, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url="https://example.test/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + bundle = bundle_with_assets(source, cache) + resolved = BundleSourceResolver( + BundleCatalog([bundle]), root / "downloads" + ).resolve(str(archive)) + + self.assertEqual(bundle, resolved.supported_bundle) + + def test_url_download_is_reused(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive = served / "codeql-bundle.tar.gz" + archive.write_bytes(b"bundle") + with serve(served) as base_url: + source = SourceAsset( + name=archive.name, + url=f"{base_url}/{archive.name}", + sha256=sha256_file(archive), + size=archive.stat().st_size, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url=f"{base_url}/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + resolver = BundleSourceResolver( + BundleCatalog([bundle_with_assets(source, cache)]), + root / "downloads", + ) + first = resolver.resolve(source.url) + second = resolver.resolve(source.url) + + self.assertEqual(first.path, second.path) + self.assertEqual(1, CountingHandler.requests) + + def test_arbitrary_url_uses_a_safe_local_archive_name(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive = served / "bundle with spaces.tar.gz" + archive.write_bytes(b"bundle") + with serve(served) as base_url: + resolved = BundleSourceResolver( + BundleCatalog([]), root / "downloads" + ).resolve(f"{base_url}/bundle%20with%20spaces.tar.gz") + + self.assertEqual("codeql-bundle.tar.gz", resolved.path.name) + self.assertEqual(archive.read_bytes(), resolved.path.read_bytes()) + + def test_release_uses_checksum_asset_when_digest_is_unavailable(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive = served / f"codeql-bundle-{current_bundle_platform()}.tar.gz" + archive.write_bytes(b"bundle") + checksum = served / f"{archive.name}.checksum.txt" + checksum.write_text(f"{sha256_file(archive)} {archive.name}\n") + with serve(served) as base_url: + release = { + "tag_name": "codeql-bundle-v1.2.3", + "assets": [ + { + "browser_download_url": f"{base_url}/{archive.name}", + "digest": None, + "name": archive.name, + "size": archive.stat().st_size, + }, + { + "browser_download_url": f"{base_url}/{checksum.name}", + "name": checksum.name, + }, + ], + } + + class ReleaseClient(GitHubReleaseClient): + def release(self, tag: str) -> dict[str, object]: + return release + + resolved = BundleSourceResolver( + BundleCatalog([]), + root / "downloads", + release_client=ReleaseClient(), + ).resolve( + "codeql-bundle-v1.2.3", + [current_bundle_platform()], + ) + + self.assertEqual(sha256_file(archive), resolved.digest) + + def test_safe_extract_rejects_parent_path(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + archive_path = root / "unsafe.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + member = tarfile.TarInfo("../escape") + contents = b"unsafe" + member.size = len(contents) + archive.addfile(member, BytesIO(contents)) + + with self.assertRaises((CacheException, tarfile.TarError)): + safe_extract_tar(archive_path, root / "output") + self.assertFalse((root / "escape").exists()) + + def test_safe_extract_fallback_rejects_escaping_hard_link(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + archive_path = root / "unsafe-link.tar.gz" + with tarfile.open(archive_path, "w:gz") as archive: + member = tarfile.TarInfo("nested/link") + member.type = tarfile.LNKTYPE + member.linkname = "../outside" + archive.addfile(member) + + data_filter = getattr(tarfile, "data_filter", None) + if data_filter is not None: + delattr(tarfile, "data_filter") + try: + with self.assertRaises(CacheException): + safe_extract_tar(archive_path, root / "output") + finally: + if data_filter is not None: + setattr(tarfile, "data_filter", data_filter) + + +class CompilationCacheManagerTests(unittest.TestCase): + def test_listed_bundle_requires_validated_platform(self) -> None: + source = SourceAsset( + name="codeql-bundle.tar.gz", + url="https://example.test/codeql-bundle.tar.gz", + sha256="1" * 64, + size=100, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url="https://example.test/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + bundle = bundle_with_assets(source, cache) + manager = CompilationCacheManager( + bundle, platform_name="unsupported-platform" + ) + with self.assertRaises(CacheException): + manager.cache_for("codeql/cpp-all") + + disabled_manager = CompilationCacheManager( + bundle, + enabled=False, + platform_name="unsupported-platform", + ) + self.assertIsNone(disabled_manager.cache_for("codeql/cpp-all")) + + def test_listed_bundle_requires_target_cache(self) -> None: + source = SourceAsset( + name="codeql-bundle.tar.gz", + url="https://example.test/codeql-bundle.tar.gz", + sha256="1" * 64, + size=100, + platform="all", + ) + cache = ReleaseAsset( + name="cache.tar.gz", + url="https://example.test/cache.tar.gz", + sha256="2" * 64, + size=10, + ) + manager = CompilationCacheManager(bundle_with_assets(source, cache)) + with self.assertRaises(CacheException): + manager.cache_for("codeql/java-all") + + def test_cache_is_downloaded_verified_and_reused(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive_path = served / "cache.tar.gz" + self._write_cache_archive( + archive_path, "codeql-bundle-v1.2.3", "codeql/cpp-all" + ) + with serve(served) as base_url: + source = SourceAsset( + name="codeql-bundle.tar.gz", + url=f"{base_url}/codeql-bundle.tar.gz", + sha256="1" * 64, + size=100, + platform="all", + ) + cache = ReleaseAsset( + name=archive_path.name, + url=f"{base_url}/{archive_path.name}", + sha256=sha256_file(archive_path), + size=archive_path.stat().st_size, + ) + manager = CompilationCacheManager( + bundle_with_assets(source, cache), root / "downloads" + ) + first = manager.cache_for("codeql/cpp-all") + second = manager.cache_for("codeql/cpp-all") + + self.assertEqual(first, second) + self.assertEqual(b"cached", (first / "entry").read_bytes()) + self.assertEqual(1, CountingHandler.requests) + + def test_cache_checksum_mismatch_fails(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + served = root / "served" + served.mkdir() + archive_path = served / "cache.tar.gz" + self._write_cache_archive( + archive_path, "codeql-bundle-v1.2.3", "codeql/cpp-all" + ) + with serve(served) as base_url: + source = SourceAsset( + name="codeql-bundle.tar.gz", + url=f"{base_url}/codeql-bundle.tar.gz", + sha256="1" * 64, + size=100, + platform="all", + ) + cache = ReleaseAsset( + name=archive_path.name, + url=f"{base_url}/{archive_path.name}", + sha256="f" * 64, + size=archive_path.stat().st_size, + ) + manager = CompilationCacheManager( + bundle_with_assets(source, cache), root / "downloads" + ) + with self.assertRaises(DownloadException): + manager.cache_for("codeql/cpp-all") + + @staticmethod + def _write_cache_archive( + path: Path, release: str, target: str + ) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + cache = root / "cache" + cache.mkdir() + (cache / "entry").write_bytes(b"cached") + (root / "metadata.json").write_text( + json.dumps( + { + "cache_format": CACHE_FORMAT_VERSION, + "cli_version": "1.2.3", + "language": "cpp", + "query_packs": ["codeql/cpp-queries@1.2.3"], + "release": release, + "target": target, + } + ) + ) + with tarfile.open(path, "w:gz") as archive: + archive.add(cache, arcname="cache") + archive.add(root / "metadata.json", arcname="metadata.json") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py new file mode 100644 index 0000000..9bbe664 --- /dev/null +++ b/tests/test_cache_cli.py @@ -0,0 +1,146 @@ +from pathlib import Path +import json +import os +import time +import unittest + +from click.testing import CliRunner + +from codeql_bundle.cache import CatalogLoader +from codeql_bundle.cache_cli import main + + +class CacheCliTests(unittest.TestCase): + def test_prune_removes_only_expired_cache_entries(self) -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + cache_dir = Path("cache") + old_entry = cache_dir / "sources" / "old" + recent_entry = cache_dir / "sources" / "recent" + old_entry.mkdir(parents=True) + recent_entry.mkdir(parents=True) + (old_entry / "bundle").write_bytes(b"old") + (recent_entry / "bundle").write_bytes(b"recent") + old_time = time.time() - 40 * 24 * 60 * 60 + os.utime(old_entry / "bundle", (old_time, old_time)) + os.utime(old_entry, (old_time, old_time)) + + result = runner.invoke( + main, + [ + "prune", + "--cache-dir", + str(cache_dir), + "--max-age-days", + "30", + ], + ) + + self.assertEqual(0, result.exit_code, result.output) + self.assertFalse(old_entry.exists()) + self.assertTrue(recent_entry.exists()) + + def test_build_rejects_invalid_release_plan(self) -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + plan_path = Path("plan.json") + plan_path.write_text("{}") + result = runner.invoke( + main, + [ + "build", + "--plan", + str(plan_path), + "--target", + "codeql/cpp-all", + "--output-dir", + "dist", + ], + ) + + self.assertNotEqual(0, result.exit_code) + self.assertIn("Invalid release plan", result.output) + + def test_catalog_entry_can_be_added_to_catalog(self) -> None: + runner = CliRunner() + with runner.isolated_filesystem(): + root = Path.cwd() + assets = root / "assets" + assets.mkdir() + cache_asset = assets / "codeql-compilation-cache-cpp.tar.gz" + cache_asset.write_bytes(b"cache") + plan = { + "cache_format": 1, + "cache_release": "codeql-compilation-cache-v1.2.3", + "cli_version": "1.2.3", + "pack_fingerprint": "0" * 64, + "release": "codeql-bundle-v1.2.3", + "source_assets": [ + { + "name": name, + "platform": platform, + "sha256": str(index) * 64, + "size": 100, + "url": f"https://example.test/{name}", + } + for index, (platform, name) in enumerate( + [ + ("all", "codeql-bundle.tar.gz"), + ("linux64", "codeql-bundle-linux64.tar.gz"), + ("osx64", "codeql-bundle-osx64.tar.gz"), + ("win64", "codeql-bundle-win64.tar.gz"), + ], + start=1, + ) + ], + "source_repository": "github/codeql-action", + "targets": [ + { + "language": "cpp", + "query_packs": ["codeql/cpp-queries@1.2.3"], + "target": "codeql/cpp-all", + } + ], + } + plan_path = root / "plan.json" + plan_path.write_text(json.dumps(plan)) + catalog_path = root / "catalog.json" + catalog_path.write_text( + json.dumps({"schema_version": 1, "bundles": []}) + ) + entry_path = root / "entry.json" + + result = runner.invoke( + main, + [ + "catalog-entry", + "--plan", + str(plan_path), + "--assets-dir", + str(assets), + "--validated-platform", + "linux64", + "--output", + str(entry_path), + ], + ) + self.assertEqual(0, result.exit_code, result.output) + + result = runner.invoke( + main, + [ + "update-catalog", + "--catalog", + str(catalog_path), + "--entry", + str(entry_path), + ], + ) + self.assertEqual(0, result.exit_code, result.output) + + catalog = CatalogLoader().load(str(catalog_path)) + self.assertIsNotNone(catalog.find_release("codeql-bundle-v1.2.3")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cache_targets.py b/tests/test_cache_targets.py new file mode 100644 index 0000000..9b7d7de --- /dev/null +++ b/tests/test_cache_targets.py @@ -0,0 +1,51 @@ +from pathlib import Path +import unittest + +from semantic_version import Version + +from codeql_bundle.helpers.bundle import ( + CodeQLPackKind, + ResolvedCodeQLPack, + get_compilation_cache_targets, +) +from codeql_bundle.helpers.codeql import CodeQLPackConfig + + +def pack( + name: str, + kind: CodeQLPackKind, + dependencies: list[ResolvedCodeQLPack] | None = None, +) -> ResolvedCodeQLPack: + return ResolvedCodeQLPack( + path=Path(name.replace("/", "-")) / "qlpack.yml", + config=CodeQLPackConfig( + library=kind != CodeQLPackKind.QUERY_PACK, + name=name, + version=Version("1.0.0"), + ), + kind=kind, + dependencies=dependencies or [], + ) + + +class CacheTargetTests(unittest.TestCase): + def test_query_packs_are_grouped_by_standard_library(self) -> None: + standard_library = pack("codeql/cpp-all", CodeQLPackKind.LIBRARY_PACK) + query_pack = pack( + "codeql/cpp-queries", + CodeQLPackKind.QUERY_PACK, + [standard_library], + ) + unrelated_library = pack( + "codeql/util", CodeQLPackKind.LIBRARY_PACK + ) + + targets = get_compilation_cache_targets( + [standard_library, query_pack, unrelated_library] + ) + + self.assertEqual({standard_library: [query_pack]}, targets) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_codeql.py b/tests/test_codeql.py new file mode 100644 index 0000000..c8da49a --- /dev/null +++ b/tests/test_codeql.py @@ -0,0 +1,76 @@ +from pathlib import Path +import subprocess +import unittest + +from semantic_version import Version + +from codeql_bundle.helpers.codeql import CodeQL, CodeQLPack, CodeQLPackConfig + + +class RecordingCodeQL(CodeQL): + def __init__(self) -> None: + super().__init__(Path("codeql")) + self._version = Version("2.26.1") + self.commands: list[list[str]] = [] + + def _exec( + self, command: str, *args: str + ) -> subprocess.CompletedProcess[str]: + self.commands.append([command, *args]) + return subprocess.CompletedProcess( + ["codeql", command, *args], + 0, + stdout="", + stderr="", + ) + + def _exec_streaming( + self, command: str, *args: str + ) -> subprocess.CompletedProcess[str]: + return self._exec(command, *args) + + +class CodeQLCommandTests(unittest.TestCase): + def test_pack_create_uses_only_explicit_compilation_caches(self) -> None: + codeql = RecordingCodeQL() + pack = CodeQLPack( + path=Path("query-pack/qlpack.yml"), + config=CodeQLPackConfig( + name="example/queries", + version=Version("1.0.0"), + ), + ) + + codeql.pack_create( + pack, + Path("output"), + compilation_caches=[Path("cpp-cache"), Path("shared-cache")], + ) + + command = codeql.commands[-1] + self.assertIn("--no-default-compilation-cache", command) + self.assertIn("--compilation-cache=cpp-cache", command) + self.assertIn("--compilation-cache=shared-cache", command) + + def test_query_compile_builds_an_explicit_cache(self) -> None: + codeql = RecordingCodeQL() + + codeql.query_compile( + [Path("queries")], + Path("compilation-cache"), + Path("bundle"), + threads=4, + ram=8192, + compilation_cache_size=4096, + ) + + command = codeql.commands[-1] + self.assertIn("--no-default-compilation-cache", command) + self.assertIn("--compilation-cache=compilation-cache", command) + self.assertIn("--compilation-cache-size=4096", command) + self.assertIn("--threads=4", command) + self.assertIn("--ram=8192", command) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/workspace/cpp/aa/qlpack.yml b/tests/workspace/cpp/aa/qlpack.yml index d7dbaac..c885849 100644 --- a/tests/workspace/cpp/aa/qlpack.yml +++ b/tests/workspace/cpp/aa/qlpack.yml @@ -4,4 +4,4 @@ warnOnImplicitThis: false name: test/aa version: 0.0.1 dependencies: - "codeql/cpp-all": "^0.8.0" + "codeql/cpp-all": "*" diff --git a/tests/workspace/cpp/foo-customizations/qlpack.yml b/tests/workspace/cpp/foo-customizations/qlpack.yml index 3007eab..3c4b217 100644 --- a/tests/workspace/cpp/foo-customizations/qlpack.yml +++ b/tests/workspace/cpp/foo-customizations/qlpack.yml @@ -2,4 +2,4 @@ library: True name: foo/cpp-customizations version: 0.0.1 dependencies: - "codeql/cpp-all": "^0.8.0" \ No newline at end of file + "codeql/cpp-all": "*" \ No newline at end of file