Fix UK private-data credential deployment

.env.example

@@ -17,7 +17,9 @@ POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=policyengine_github_token
 OPENAI_API_KEY=policyengine_openai_api_key
 
 # Token for Hugging Face models 
-HUGGING_FACE_TOKEN=policyengine_huggingface_token
+# PolicyEngine Core compatibility variable. Managed deployments bind the
+# PE_UK_PRIVATE_HF_READ_TOKEN credential to this process-level name.
+HUGGING_FACE_TOKEN=pe_uk_private_hf_read_token_value
 
 # Redis is required for budget-window economy requests and other API cache
 # paths. Configure local development explicitly as described in README.md;

.github/scripts/cloud_run_env.sh

@@ -39,7 +39,7 @@ cloud_run_set_defaults() {
   CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-}"
   CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}"
   CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}"
-  CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}"
+  CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET:-pe-uk-private-hf-read-token:latest}"
   CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}"
   CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}"
   CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}"
@@ -76,7 +76,7 @@ cloud_run_set_defaults() {
   export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET
   export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET
   export CLOUD_RUN_OPENAI_API_KEY_SECRET
-  export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET
+  export CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET
   export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT

.github/scripts/deploy_cloud_run_candidate.sh

@@ -62,7 +62,9 @@ secret_vars=(
   "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}"
   "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET}"
   "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}"
-  "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}"
+  # PolicyEngine Core reads HUGGING_FACE_TOKEN. The stored credential and every
+  # deployment input use the purpose-specific PE_UK_PRIVATE_HF_READ_TOKEN name.
+  "HUGGING_FACE_TOKEN=${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET}"
   "RUNTIME_CACHE_URL=${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET}"
   "RUNTIME_CACHE_CA_CERT=${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET}"
 )

.github/scripts/resolve_cloud_run_candidate_state.sh

@@ -78,6 +78,32 @@ image="$(jq -er '
   | select(type == "string" and contains("@sha256:"))
 ' <<<"${revision_json}")"
 
+expected_hf_secret_resource="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET%:*}"
+expected_hf_secret_name="${expected_hf_secret_resource##*/}"
+expected_hf_secret_version="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET##*:}"
+actual_hf_secret_binding="$(jq -cer '
+  [
+    .spec.containers[0].env[]?
+    | select(.name == "HUGGING_FACE_TOKEN")
+    | {
+        name: .valueFrom.secretKeyRef.name,
+        version: .valueFrom.secretKeyRef.key
+      }
+  ]
+  | if length == 1 then .[0]
+    else error("HUGGING_FACE_TOKEN must have exactly one secret binding")
+    end
+' <<<"${revision_json}")"
+actual_hf_secret_name="$(jq -er '.name' <<<"${actual_hf_secret_binding}")"
+actual_hf_secret_version="$(jq -er '.version' <<<"${actual_hf_secret_binding}")"
+if [[ "${actual_hf_secret_name}" != "${expected_hf_secret_name}" \
+  || "${actual_hf_secret_version}" != "${expected_hf_secret_version}" ]]; then
+  printf 'Revision %s binds HUGGING_FACE_TOKEN to %s:%s; expected %s:%s\n' \
+    "${revision}" "${actual_hf_secret_name}" "${actual_hf_secret_version}" \
+    "${expected_hf_secret_name}" "${expected_hf_secret_version}" >&2
+  exit 2
+fi
+
 deployment_selector_count=0
 for selector in \
   ROUTE_IMPL_HEALTH \

.github/scripts/sync_cloud_run_secrets.sh

@@ -4,6 +4,21 @@ set -euo pipefail
 set +x
 
 CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}"
+CLOUD_RUN_REGION="${CLOUD_RUN_REGION:-us-central1}"
+GCLOUD_BIN="${GCLOUD_BIN:-gcloud}"
+include_database_password=0
+case "${1:-}" in
+  "") ;;
+  --include-database-password) include_database_password=1 ;;
+  *)
+    echo "::error::Unknown argument: $1" >&2
+    exit 2
+    ;;
+esac
+if [[ "$#" -gt 1 ]]; then
+  echo "::error::Expected at most one argument." >&2
+  exit 2
+fi
 
 require_env() {
   local env_name="$1"
@@ -17,35 +32,119 @@ sync_secret() {
   local env_name="$1"
   local secret_name="$2"
   local secret_value="${!env_name:-}"
+  local expected_hash
+  local stored_hash=""
 
   if [[ -z "${secret_value}" ]]; then
     echo "::error::Missing required GitHub secret ${env_name}."
     exit 1
   fi
 
-  if ! gcloud secrets describe "${secret_name}" \
+  if ! "${GCLOUD_BIN}" secrets describe "${secret_name}" \
     --project "${CLOUD_RUN_PROJECT}" >/dev/null 2>&1; then
-    gcloud secrets create "${secret_name}" \
+    "${GCLOUD_BIN}" secrets create "${secret_name}" \
       --project "${CLOUD_RUN_PROJECT}" \
       --replication-policy automatic
   fi
 
-  printf '%s' "${secret_value}" | gcloud secrets versions add \
-    "${secret_name}" \
-    --project "${CLOUD_RUN_PROJECT}" \
-    --data-file=- >/dev/null
+  expected_hash="$(printf '%s' "${secret_value}" | sha256sum | cut -d ' ' -f 1)"
+  if stored_hash="$(
+    "${GCLOUD_BIN}" secrets versions access latest \
+      --secret "${secret_name}" \
+      --project "${CLOUD_RUN_PROJECT}" 2>/dev/null \
+      | sha256sum \
+      | cut -d ' ' -f 1
+  )" && [[ "${stored_hash}" == "${expected_hash}" ]]; then
+    echo "Secret Manager already matches ${env_name}; no version added."
+  else
+    printf '%s' "${secret_value}" | "${GCLOUD_BIN}" secrets versions add \
+      "${secret_name}" \
+      --project "${CLOUD_RUN_PROJECT}" \
+      --data-file=- >/dev/null
+    echo "Synchronized ${env_name} to Secret Manager secret ${secret_name}."
+  fi
+
+  verify_secret_value "${env_name}" "${secret_name}"
+  grant_secret_access "${secret_name}"
+  unset secret_value expected_hash stored_hash
+}
+
+verify_secret_value() {
+  local env_name="$1"
+  local secret_name="$2"
+  local expected_value="${!env_name:-}"
+  local expected_hash
+  local stored_hash
+
+  expected_hash="$(printf '%s' "${expected_value}" | sha256sum | cut -d ' ' -f 1)"
+  stored_hash="$(
+    "${GCLOUD_BIN}" secrets versions access latest \
+      --secret "${secret_name}" \
+      --project "${CLOUD_RUN_PROJECT}" \
+      | sha256sum \
+      | cut -d ' ' -f 1
+  )"
+  if [[ "${stored_hash}" != "${expected_hash}" ]]; then
+    echo "::error::Secret Manager value verification failed for ${secret_name}." >&2
+    return 1
+  fi
+  unset stored_hash expected_hash expected_value
+  echo "Verified ${env_name} in Secret Manager without printing its value."
+}
+
+grant_secret_access() {
+  local secret_name="$1"
+  local member
 
-  gcloud secrets add-iam-policy-binding "${secret_name}" \
+  member="serviceAccount:${runtime_service_account}"
+
+  "${GCLOUD_BIN}" secrets add-iam-policy-binding "${secret_name}" \
     --project "${CLOUD_RUN_PROJECT}" \
-    --member "serviceAccount:${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \
+    --member "${member}" \
     --role roles/secretmanager.secretAccessor >/dev/null
 
-  echo "Synced ${env_name} to Secret Manager secret ${secret_name}."
+  if ! "${GCLOUD_BIN}" secrets get-iam-policy "${secret_name}" \
+    --project "${CLOUD_RUN_PROJECT}" \
+    --format=json \
+    | jq -e --arg member "${member}" '
+        any(
+          .bindings[]?;
+          .role == "roles/secretmanager.secretAccessor"
+          and ((.members // []) | index($member) != null)
+        )
+      ' >/dev/null; then
+    echo "::error::Secret Manager access verification failed for ${CLOUD_RUN_SERVICE}." >&2
+    return 1
+  fi
+  echo "Verified ${CLOUD_RUN_SERVICE} can read ${secret_name}."
 }
 
-require_env CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT
+require_env CLOUD_RUN_SERVICE
+
+runtime_service_account="$(
+  "${GCLOUD_BIN}" run services describe "${CLOUD_RUN_SERVICE}" \
+    --project "${CLOUD_RUN_PROJECT}" \
+    --region "${CLOUD_RUN_REGION}" \
+    --platform managed \
+    --format='value(spec.template.spec.serviceAccountName)'
+)"
+if [[ -z "${runtime_service_account}" ]]; then
+  echo "::error::Cloud Run service ${CLOUD_RUN_SERVICE} has no runtime service account." >&2
+  exit 1
+fi
 
-sync_secret POLICYENGINE_DB_PASSWORD policyengine-api-prod-db-password
+if [[ "${include_database_password}" -eq 1 ]]; then
+  require_env CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET
+  db_password_secret_version="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET##*:}"
+  db_password_secret_resource="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET%:*}"
+  db_password_secret_name="${db_password_secret_resource##*/}"
+  if [[ "${db_password_secret_version}" != "latest" \
+    || -z "${db_password_secret_name}" ]]; then
+    echo "::error::CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET must name a :latest secret version." >&2
+    exit 1
+  fi
+  sync_secret POLICYENGINE_DB_PASSWORD "${db_password_secret_name}"
+fi
 sync_secret POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN policyengine-api-prod-github-microdata-token
 sync_secret OPENAI_API_KEY policyengine-api-prod-openai-api-key
-sync_secret HUGGING_FACE_TOKEN policyengine-api-prod-hugging-face-token
+sync_secret PE_UK_PRIVATE_HF_READ_TOKEN pe-uk-private-hf-read-token

.github/scripts/validate_cloud_run_deploy_env.sh

@@ -29,7 +29,7 @@ cloud_run_require_env \
   CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \
   CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET \
   CLOUD_RUN_OPENAI_API_KEY_SECRET \
-  CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \
+  CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET \
   CLOUD_RUN_RUNTIME_CACHE_URL_SECRET \
   CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET \
   CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT \

.github/workflows/pr.yml

@@ -145,7 +145,9 @@ jobs:
         run: pytest tests/env_variables/test_environment_variables.py
         env:
           POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
-          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+          # Compatibility name required by PolicyEngine Core. The GitHub
+          # credential name describes its provider, data, and read-only scope.
+          HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
           POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
   test:
     name: Test
@@ -184,7 +186,7 @@ jobs:
           POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
           POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+          HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5
         with:

.github/workflows/push.yml

@@ -240,7 +240,8 @@ jobs:
           POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
           POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+          # Compatibility name required by PolicyEngine Core.
+          HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
           GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }}
           GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }}
           GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }}
@@ -254,6 +255,14 @@ jobs:
         uses: "google-github-actions/setup-gcloud@v2"
       - name: Install jq
         run: sudo apt-get install -y jq
+      # The staging database password is managed independently in Secret Manager
+      # and differs from the repository-level production database password.
+      - name: Synchronize shared Cloud Run runtime secrets
+        env:
+          POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
+        run: bash .github/scripts/sync_cloud_run_secrets.sh
       - name: Capture current Cloud Run staging state
         id: previous
         run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT"
@@ -488,6 +497,13 @@ jobs:
         uses: "google-github-actions/setup-gcloud@v2"
       - name: Install jq
         run: sudo apt-get install -y jq
+      - name: Synchronize complete Cloud Run runtime secret batch
+        env:
+          POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
+          POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
+        run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password
       - name: Capture current Cloud Run production state
         id: previous
         run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT"

.github/workflows/sync-cloud-run-secrets.yml

@@ -2,6 +2,15 @@ name: Sync Cloud Run secrets
 
 on:
   workflow_dispatch:
+    inputs:
+      deployment_environment:
+        description: GitHub and Cloud Run environment to synchronize
+        required: true
+        default: production
+        type: choice
+        options:
+          - staging
+          - production
 
 concurrency:
   group: cloud-run-secret-sync
@@ -10,7 +19,7 @@ jobs:
   sync-cloud-run-secrets:
     name: Sync GitHub secrets to Secret Manager
     runs-on: ubuntu-latest
-    environment: production
+    environment: ${{ inputs.deployment_environment }}
     permissions:
       contents: read
       id-token: write
@@ -29,12 +38,25 @@ jobs:
           service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}"
       - name: Set up GCloud
         uses: "google-github-actions/setup-gcloud@v2"
-      - name: Sync runtime secrets
+      - name: Install jq
+        run: sudo apt-get install -y jq
+      - name: Sync shared staging runtime secrets
+        if: inputs.deployment_environment == 'staging'
         env:
           CLOUD_RUN_PROJECT: policyengine-api
-          CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: ${{ secrets.GCP_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT }}
-          POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
+          CLOUD_RUN_SERVICE: policyengine-api-staging
           POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-          HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}
+          PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
         run: bash .github/scripts/sync_cloud_run_secrets.sh
+      - name: Sync complete production runtime secret batch
+        if: inputs.deployment_environment == 'production'
+        env:
+          CLOUD_RUN_PROJECT: policyengine-api
+          CLOUD_RUN_SERVICE: policyengine-api
+          CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }}
+          POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}
+          POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}
+        run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password

README.md

@@ -43,7 +43,9 @@ make setup-env
 - `POLICYENGINE_DB_INSTANCE_CONNECTION_NAME`
 - `POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN`
 - `OPENAI_API_KEY`
-- `HUGGING_FACE_TOKEN`
+- `HUGGING_FACE_TOKEN` (the local PolicyEngine Core compatibility variable;
+  managed deployments source it from the purpose-specific GitHub secret
+  `PE_UK_PRIVATE_HF_READ_TOKEN`)
 
 The database settings must resolve to an explicit durable development MySQL
 database (or an authorized Cloud SQL development target). `FLASK_DEBUG` does

changelog.d/uk-private-hf-read-token.fixed.md

@@ -0,0 +1 @@
+Use a dedicated read-only Hugging Face credential for private UK runtime data, synchronize GitHub-owned runtime secrets before Cloud Run deployment, and verify their values and access bindings.