Cache CI container dependency layers in GHCR.

.github/workflows/test.yml

@@ -13,9 +13,15 @@ on:
       matrix_env:
         required: true
         type: "string"
+      cache_write:
+        description: "Push Docker layer cache to GHCR. Only enable from nightly; PR GITHUB_TOKEN cannot push packages."
+        required: false
+        type: "boolean"
+        default: false
 
 env:
   UV_SYSTEM_PYTHON: "1"
+  DOCKER_CACHE_WRITE: "${{ inputs.cache_write }}"
 
 defaults:
   run:
@@ -93,6 +99,17 @@ jobs:
         run: |
           echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV
 
+      - name: "Set up Docker Buildx"
+        uses: "docker/setup-buildx-action@v4"
+
+      - name: "Login to GHCR"
+        if: ${{ inputs.cache_write }}
+        uses: "docker/login-action@v4"
+        with:
+          registry: "ghcr.io"
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GITHUB_TOKEN }}"
+
       - name: "Prepare Scenario Definition"
         run: |
           .github/workflows/scripts/before_install.sh

CHANGES/2237.feature

@@ -0,0 +1,2 @@
+Cache CI container dependency layers in GHCR, warmed by nightly builds.
+

templates/github/.ci/ansible/Containerfile.j2.copy

@@ -1,4 +1,12 @@
 FROM {{ image.ci_base }}
+
+# Third-party deps are compiled on the host into a lockfile that does not include
+# the PR wheel, so this layer can be reused across CI runs.
+COPY ./{{ plugin_name }}/.ci/assets/third-party-requirements.txt /tmp/third-party-requirements.txt
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --upgrade setuptools wheel && \
+    uv pip install -r /tmp/third-party-requirements.txt
+
 {%- if image.webserver_snippet %}
 
 ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf
@@ -9,19 +17,8 @@ ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snip
 ADD ./{{ item.origin }} {{ item.destination }}
 {%- endfor %}
 
-# This MUST be the ONLY package install inside the container.
-RUN --mount=type=cache,target=/root/.cache/uv uv pip install --upgrade setuptools wheel && \
-  uv pip install {{ image.source }}
-{%- if image.upperbounds | default(false) -%}
-{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt
-{%- endif -%}
-{%- if image.lowerbounds | default(false) -%}
-{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt
-{%- endif -%}
-{%- if image.ci_requirements | default(false) -%}
-{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt
-{%- endif -%}
-{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt
+# The local wheel changes every CI run; extras and git deps are already in the lockfile.
+RUN uv pip install --no-deps {{ image.wheel }}
 
 {% if pulp_env is defined and pulp_env %}
 {% for key, value in pulp_env.items() %}

templates/github/.ci/ansible/build_container.yaml

@@ -12,28 +12,28 @@
         dest: "Containerfile"
 
     - name: "Build pulp image"
-      # We build from the ../.. (parent dir of pulpcore git repo) Docker build
-      # "context" so that repos like pulp-smash are accessible to Docker
-      # build. So that PR branches can be used via relative paths.
-      #
-      # We default to using the docker build / podman buildah cache, for
-      # 1-off-builds and CI purposes (which has no cache across CI runs.)
-      # Run build.yaml with -e cache=false if your builds are using outdated
-      # layers.
-      ansible.builtin.command:
-        argv:
-          - "docker"
-          - "build"
-          - "--network"
-          - "host"
-          - "--no-cache={{ not cache | default(true) | bool }}"
-          - "-t"
-          - "{{ image.name }}:{{ image.tag }}"
-          - "-f"
-          - "{{ playbook_dir }}/Containerfile"
-          - "../../.."
-
-    - name: "Clean image cache"
-      community.docker.docker_prune:
-        images: true
-...
+      # Build from the parent of the plugin checkout so extra repos can be ADD'd
+      # via relative paths. GitHub Actions imports/exports layer cache via GHCR
+      # (nightly writes, PRs only read). Local builds use the engine's own cache.
+      # Pass -e cache=false to force a full rebuild.
+      ansible.builtin.shell:
+        cmd: |
+          set -eu
+          extra=()
+          if [ "{{ cache | default(true) | bool }}" = "True" ]; then
+            if [ "${GITHUB_ACTIONS:-}" = "true" ]; then
+              extra+=(--cache-from "type=registry,ref={{ image.cache_ref }},ignore-error=true")
+              if [ "{{ image.cache_write | default(false) | bool }}" = "True" ]; then
+                extra+=(--cache-to "type=registry,ref={{ image.cache_ref }},mode=max,image-manifest=true,oci-mediatypes=true")
+              fi
+            fi
+          else
+            extra+=(--no-cache)
+          fi
+          docker buildx build --network host --load \
+            -t "{{ image.name }}:{{ image.tag }}" \
+            -f "{{ playbook_dir }}/Containerfile" \
+            "${extra[@]}" \
+            ../../..
+      args:
+        executable: "/bin/bash"

templates/github/.github/workflows/nightly.yml.j2

@@ -37,8 +37,12 @@ jobs:
 
   test:
     needs: "build"
+    permissions:
+      contents: "read"
+      packages: "write"
     uses: "./.github/workflows/test.yml"
     with:
+      cache_write: true
       matrix_env: |
         {{ matrix_env(performance=true) | from_yaml | tojson }}
 

templates/github/.github/workflows/scripts/before_install.sh.j2

@@ -19,22 +19,37 @@ if [ -f .github/workflows/scripts/pre_before_install.sh ]; then
 fi
 
 COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
-COMPONENT_SOURCE="./{{ plugin_name }}/dist/{{ plugin_name | snake }}-${COMPONENT_VERSION}-py3-none-any.whl"
+WHEEL_NAME="{{ plugin_name | snake }}-${COMPONENT_VERSION}-py3-none-any.whl"
+HOST_WHEEL="./dist/${WHEEL_NAME}"
+DOCKER_WHEEL="./{{ plugin_name }}/dist/${WHEEL_NAME}"
 
-{%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %}
+COMPILE_PKGS=("${HOST_WHEEL}")
 {%- if test_s3 %}
 if [ "$TEST" = "s3" ]; then
-  COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue"
+  {%- if plugin_name == "pulpcore" %}
+  COMPILE_PKGS=("${HOST_WHEEL}[s3]")
+  {%- else %}
+  COMPILE_PKGS+=("pulpcore[s3]")
+  {%- endif %}
+  COMPILE_PKGS+=("git+https://github.com/gerrod3/botocore.git@fix-100-continue")
 fi
 {%- endif %}
 {%- if test_azure %}
 if [ "$TEST" = "azure" ]; then
-  COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[azure,uvloop]"
+  {%- if plugin_name == "pulpcore" %}
+  COMPILE_PKGS=("${HOST_WHEEL}[azure,uvloop]")
+  {%- else %}
+  COMPILE_PKGS+=("pulpcore[azure,uvloop]")
+  {%- endif %}
 fi
 {%- endif %}
 {%- if test_gcp %}
 if [ "$TEST" = "gcp" ]; then
-  COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[google]"
+  {%- if plugin_name == "pulpcore" %}
+  COMPILE_PKGS=("${HOST_WHEEL}[google]")
+  {%- else %}
+  COMPILE_PKGS+=("pulpcore[google]")
+  {%- endif %}
 fi
 {%- endif %}
 
@@ -45,6 +60,28 @@ if [[ "$TEST" = "lowerbounds" ]]; then
   python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt
 fi
 
+# Pin third-party deps without the PR wheel so the image's first RUN can cache.
+COMPILE_ARGS=(
+  --no-header
+  --no-annotate
+  --no-emit-package "{{ plugin_name | snake }}"
+  --python-version "{{ python_version }}"
+  --python-platform linux
+  -o .ci/assets/third-party-requirements.txt
+  -c .ci/assets/ci_constraints.txt
+)
+if [[ "$TEST" = "pulp" ]]; then
+  COMPILE_ARGS+=(-c upperbounds_constraints.txt)
+fi
+if [[ "$TEST" = "lowerbounds" ]]; then
+  COMPILE_ARGS+=(-c lowerbounds_constraints.txt)
+fi
+if [[ -s ci_requirements.txt ]]; then
+  COMPILE_ARGS+=(-r ci_requirements.txt)
+fi
+printf '%s\n' "${COMPILE_PKGS[@]}" > .ci/assets/third-party-requirements.in
+uv pip compile "${COMPILE_ARGS[@]}" .ci/assets/third-party-requirements.in
+
 # Compose the scenario definition.
 mkdir -p .ci/ansible/vars
 
@@ -66,7 +103,9 @@ image:
   {% else %}
   ci_base: "{{ ci_base_image + ":latest" }}"
   {% endif -%}
-  source: "${COMPONENT_SOURCE}"
+  wheel: "${DOCKER_WHEEL}"
+  cache_ref: "ghcr.io/{{ github_org }}/{{ plugin_name | dash }}-ci-cache:${TEST}"
+  cache_write: "${DOCKER_CACHE_WRITE:-false}"
   ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
   upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
   lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)

templates/github/.github/workflows/scripts/before_script.sh.j2

@@ -30,6 +30,10 @@ echo
 echo "# Containerfile:"
 tail -v -n +1 .ci/ansible/Containerfile
 
+echo
+echo "# Third-party requirements:"
+tail -v -n +1 .ci/assets/third-party-requirements.txt
+
 echo
 echo "# Constraints Files:"
 # They need not even exist.

templates/github/.github/workflows/test.yml.j2

@@ -15,9 +15,15 @@ on:
       matrix_env:
         required: true
         type: "string"
+      cache_write:
+        description: "Push Docker layer cache to GHCR. Only enable from nightly; PR GITHUB_TOKEN cannot push packages."
+        required: false
+        type: "boolean"
+        default: false
 
 env:
   UV_SYSTEM_PYTHON: "1"
+  DOCKER_CACHE_WRITE: "{{ '${{ inputs.cache_write }}' }}"
 
 defaults:
   run:
@@ -74,6 +80,17 @@ jobs:
 
       {{ setup_env() | indent(6) }}
 
+      - name: "Set up Docker Buildx"
+        uses: "docker/setup-buildx-action@v4"
+
+      - name: "Login to GHCR"
+        if: "{{ '${{ inputs.cache_write }}' }}"
+        uses: "docker/login-action@v4"
+        with:
+          registry: "ghcr.io"
+          username: "{{ '${{ github.actor }}' }}"
+          password: "{{ '${{ secrets.GITHUB_TOKEN }}' }}"
+
       {{ run_script(name="Prepare Scenario Definition", file="before_install.sh") | indent(6) }}
 
       {{ run_script(name="Install", file="install.sh") | indent(6) }}