diff --git a/.env.example b/.env.example index b43e2dc22..dfd01ccb3 100644 --- a/.env.example +++ b/.env.example @@ -17,7 +17,9 @@ POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=policyengine_github_token OPENAI_API_KEY=policyengine_openai_api_key # Token for Hugging Face models -HUGGING_FACE_TOKEN=policyengine_huggingface_token +# PolicyEngine Core compatibility variable. Managed deployments bind the +# PE_UK_PRIVATE_HF_READ_TOKEN credential to this process-level name. +HUGGING_FACE_TOKEN=pe_uk_private_hf_read_token_value # Redis is required for budget-window economy requests and other API cache # paths. Configure local development explicitly as described in README.md; diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index c97672628..3a0743fec 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -39,7 +39,7 @@ cloud_run_set_defaults() { CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET:-}" CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET="${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET:-policyengine-api-prod-github-microdata-token:latest}" CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}" - CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}" + CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET:-pe-uk-private-hf-read-token:latest}" CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}" CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}" CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}" @@ -76,7 +76,7 @@ cloud_run_set_defaults() { export CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET export CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET export CLOUD_RUN_OPENAI_API_KEY_SECRET - export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET + export CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index 2e5d80773..827403ce0 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -62,7 +62,9 @@ secret_vars=( "POLICYENGINE_DB_PASSWORD=${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET}" "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=${CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET}" "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}" - "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}" + # PolicyEngine Core reads HUGGING_FACE_TOKEN. The stored credential and every + # deployment input use the purpose-specific PE_UK_PRIVATE_HF_READ_TOKEN name. + "HUGGING_FACE_TOKEN=${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET}" "RUNTIME_CACHE_URL=${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET}" "RUNTIME_CACHE_CA_CERT=${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET}" ) diff --git a/.github/scripts/resolve_cloud_run_candidate_state.sh b/.github/scripts/resolve_cloud_run_candidate_state.sh index 334a42b66..60f3b49c7 100755 --- a/.github/scripts/resolve_cloud_run_candidate_state.sh +++ b/.github/scripts/resolve_cloud_run_candidate_state.sh @@ -78,6 +78,32 @@ image="$(jq -er ' | select(type == "string" and contains("@sha256:")) ' <<<"${revision_json}")" +expected_hf_secret_resource="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET%:*}" +expected_hf_secret_name="${expected_hf_secret_resource##*/}" +expected_hf_secret_version="${CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET##*:}" +actual_hf_secret_binding="$(jq -cer ' + [ + .spec.containers[0].env[]? + | select(.name == "HUGGING_FACE_TOKEN") + | { + name: .valueFrom.secretKeyRef.name, + version: .valueFrom.secretKeyRef.key + } + ] + | if length == 1 then .[0] + else error("HUGGING_FACE_TOKEN must have exactly one secret binding") + end +' <<<"${revision_json}")" +actual_hf_secret_name="$(jq -er '.name' <<<"${actual_hf_secret_binding}")" +actual_hf_secret_version="$(jq -er '.version' <<<"${actual_hf_secret_binding}")" +if [[ "${actual_hf_secret_name}" != "${expected_hf_secret_name}" \ + || "${actual_hf_secret_version}" != "${expected_hf_secret_version}" ]]; then + printf 'Revision %s binds HUGGING_FACE_TOKEN to %s:%s; expected %s:%s\n' \ + "${revision}" "${actual_hf_secret_name}" "${actual_hf_secret_version}" \ + "${expected_hf_secret_name}" "${expected_hf_secret_version}" >&2 + exit 2 +fi + deployment_selector_count=0 for selector in \ ROUTE_IMPL_HEALTH \ diff --git a/.github/scripts/sync_cloud_run_secrets.sh b/.github/scripts/sync_cloud_run_secrets.sh index a83b3599c..3ee45fa82 100644 --- a/.github/scripts/sync_cloud_run_secrets.sh +++ b/.github/scripts/sync_cloud_run_secrets.sh @@ -4,6 +4,21 @@ set -euo pipefail set +x CLOUD_RUN_PROJECT="${CLOUD_RUN_PROJECT:-policyengine-api}" +CLOUD_RUN_REGION="${CLOUD_RUN_REGION:-us-central1}" +GCLOUD_BIN="${GCLOUD_BIN:-gcloud}" +include_database_password=0 +case "${1:-}" in + "") ;; + --include-database-password) include_database_password=1 ;; + *) + echo "::error::Unknown argument: $1" >&2 + exit 2 + ;; +esac +if [[ "$#" -gt 1 ]]; then + echo "::error::Expected at most one argument." >&2 + exit 2 +fi require_env() { local env_name="$1" @@ -17,35 +32,119 @@ sync_secret() { local env_name="$1" local secret_name="$2" local secret_value="${!env_name:-}" + local expected_hash + local stored_hash="" if [[ -z "${secret_value}" ]]; then echo "::error::Missing required GitHub secret ${env_name}." exit 1 fi - if ! gcloud secrets describe "${secret_name}" \ + if ! "${GCLOUD_BIN}" secrets describe "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" >/dev/null 2>&1; then - gcloud secrets create "${secret_name}" \ + "${GCLOUD_BIN}" secrets create "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" \ --replication-policy automatic fi - printf '%s' "${secret_value}" | gcloud secrets versions add \ - "${secret_name}" \ - --project "${CLOUD_RUN_PROJECT}" \ - --data-file=- >/dev/null + expected_hash="$(printf '%s' "${secret_value}" | sha256sum | cut -d ' ' -f 1)" + if stored_hash="$( + "${GCLOUD_BIN}" secrets versions access latest \ + --secret "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" 2>/dev/null \ + | sha256sum \ + | cut -d ' ' -f 1 + )" && [[ "${stored_hash}" == "${expected_hash}" ]]; then + echo "Secret Manager already matches ${env_name}; no version added." + else + printf '%s' "${secret_value}" | "${GCLOUD_BIN}" secrets versions add \ + "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --data-file=- >/dev/null + echo "Synchronized ${env_name} to Secret Manager secret ${secret_name}." + fi + + verify_secret_value "${env_name}" "${secret_name}" + grant_secret_access "${secret_name}" + unset secret_value expected_hash stored_hash +} + +verify_secret_value() { + local env_name="$1" + local secret_name="$2" + local expected_value="${!env_name:-}" + local expected_hash + local stored_hash + + expected_hash="$(printf '%s' "${expected_value}" | sha256sum | cut -d ' ' -f 1)" + stored_hash="$( + "${GCLOUD_BIN}" secrets versions access latest \ + --secret "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + | sha256sum \ + | cut -d ' ' -f 1 + )" + if [[ "${stored_hash}" != "${expected_hash}" ]]; then + echo "::error::Secret Manager value verification failed for ${secret_name}." >&2 + return 1 + fi + unset stored_hash expected_hash expected_value + echo "Verified ${env_name} in Secret Manager without printing its value." +} + +grant_secret_access() { + local secret_name="$1" + local member - gcloud secrets add-iam-policy-binding "${secret_name}" \ + member="serviceAccount:${runtime_service_account}" + + "${GCLOUD_BIN}" secrets add-iam-policy-binding "${secret_name}" \ --project "${CLOUD_RUN_PROJECT}" \ - --member "serviceAccount:${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \ + --member "${member}" \ --role roles/secretmanager.secretAccessor >/dev/null - echo "Synced ${env_name} to Secret Manager secret ${secret_name}." + if ! "${GCLOUD_BIN}" secrets get-iam-policy "${secret_name}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --format=json \ + | jq -e --arg member "${member}" ' + any( + .bindings[]?; + .role == "roles/secretmanager.secretAccessor" + and ((.members // []) | index($member) != null) + ) + ' >/dev/null; then + echo "::error::Secret Manager access verification failed for ${CLOUD_RUN_SERVICE}." >&2 + return 1 + fi + echo "Verified ${CLOUD_RUN_SERVICE} can read ${secret_name}." } -require_env CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT +require_env CLOUD_RUN_SERVICE + +runtime_service_account="$( + "${GCLOUD_BIN}" run services describe "${CLOUD_RUN_SERVICE}" \ + --project "${CLOUD_RUN_PROJECT}" \ + --region "${CLOUD_RUN_REGION}" \ + --platform managed \ + --format='value(spec.template.spec.serviceAccountName)' +)" +if [[ -z "${runtime_service_account}" ]]; then + echo "::error::Cloud Run service ${CLOUD_RUN_SERVICE} has no runtime service account." >&2 + exit 1 +fi -sync_secret POLICYENGINE_DB_PASSWORD policyengine-api-prod-db-password +if [[ "${include_database_password}" -eq 1 ]]; then + require_env CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET + db_password_secret_version="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET##*:}" + db_password_secret_resource="${CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET%:*}" + db_password_secret_name="${db_password_secret_resource##*/}" + if [[ "${db_password_secret_version}" != "latest" \ + || -z "${db_password_secret_name}" ]]; then + echo "::error::CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET must name a :latest secret version." >&2 + exit 1 + fi + sync_secret POLICYENGINE_DB_PASSWORD "${db_password_secret_name}" +fi sync_secret POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN policyengine-api-prod-github-microdata-token sync_secret OPENAI_API_KEY policyengine-api-prod-openai-api-key -sync_secret HUGGING_FACE_TOKEN policyengine-api-prod-hugging-face-token +sync_secret PE_UK_PRIVATE_HF_READ_TOKEN pe-uk-private-hf-read-token diff --git a/.github/scripts/validate_cloud_run_deploy_env.sh b/.github/scripts/validate_cloud_run_deploy_env.sh index 8647f1235..1f1e32451 100755 --- a/.github/scripts/validate_cloud_run_deploy_env.sh +++ b/.github/scripts/validate_cloud_run_deploy_env.sh @@ -29,7 +29,7 @@ cloud_run_require_env \ CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \ CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET \ CLOUD_RUN_OPENAI_API_KEY_SECRET \ - CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \ + CLOUD_RUN_PE_UK_PRIVATE_HF_READ_TOKEN_SECRET \ CLOUD_RUN_RUNTIME_CACHE_URL_SECRET \ CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET \ CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT \ diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index dee90cc71..9171eba74 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -145,7 +145,9 @@ jobs: run: pytest tests/env_variables/test_environment_variables.py env: POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + # Compatibility name required by PolicyEngine Core. The GitHub + # credential name describes its provider, data, and read-only scope. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} test: name: Test @@ -184,7 +186,7 @@ jobs: POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 6b817ab14..a602c0138 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -240,7 +240,8 @@ jobs: POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + # Compatibility name required by PolicyEngine Core. + HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} @@ -254,6 +255,14 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" - name: Install jq run: sudo apt-get install -y jq + # The staging database password is managed independently in Secret Manager + # and differs from the repository-level production database password. + - name: Synchronize shared Cloud Run runtime secrets + env: + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh - name: Capture current Cloud Run staging state id: previous run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT" @@ -488,6 +497,13 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" - name: Install jq run: sudo apt-get install -y jq + - name: Synchronize complete Cloud Run runtime secret batch + env: + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password - name: Capture current Cloud Run production state id: previous run: bash .github/scripts/capture_cloud_run_service_state.sh >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/sync-cloud-run-secrets.yml b/.github/workflows/sync-cloud-run-secrets.yml index 77b766777..c6b46f80a 100644 --- a/.github/workflows/sync-cloud-run-secrets.yml +++ b/.github/workflows/sync-cloud-run-secrets.yml @@ -2,6 +2,15 @@ name: Sync Cloud Run secrets on: workflow_dispatch: + inputs: + deployment_environment: + description: GitHub and Cloud Run environment to synchronize + required: true + default: production + type: choice + options: + - staging + - production concurrency: group: cloud-run-secret-sync @@ -10,7 +19,7 @@ jobs: sync-cloud-run-secrets: name: Sync GitHub secrets to Secret Manager runs-on: ubuntu-latest - environment: production + environment: ${{ inputs.deployment_environment }} permissions: contents: read id-token: write @@ -29,12 +38,25 @@ jobs: service_account: "${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" - name: Set up GCloud uses: "google-github-actions/setup-gcloud@v2" - - name: Sync runtime secrets + - name: Install jq + run: sudo apt-get install -y jq + - name: Sync shared staging runtime secrets + if: inputs.deployment_environment == 'staging' env: CLOUD_RUN_PROJECT: policyengine-api - CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: ${{ secrets.GCP_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT }} - POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + CLOUD_RUN_SERVICE: policyengine-api-staging POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} run: bash .github/scripts/sync_cloud_run_secrets.sh + - name: Sync complete production runtime secret batch + if: inputs.deployment_environment == 'production' + env: + CLOUD_RUN_PROJECT: policyengine-api + CLOUD_RUN_SERVICE: policyengine-api + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: ${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }} + POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }} + run: bash .github/scripts/sync_cloud_run_secrets.sh --include-database-password diff --git a/README.md b/README.md index 9f0a4470d..f572bcdf9 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,9 @@ make setup-env - `POLICYENGINE_DB_INSTANCE_CONNECTION_NAME` - `POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN` - `OPENAI_API_KEY` -- `HUGGING_FACE_TOKEN` +- `HUGGING_FACE_TOKEN` (the local PolicyEngine Core compatibility variable; + managed deployments source it from the purpose-specific GitHub secret + `PE_UK_PRIVATE_HF_READ_TOKEN`) The database settings must resolve to an explicit durable development MySQL database (or an authorized Cloud SQL development target). `FLASK_DEBUG` does diff --git a/changelog.d/uk-private-hf-read-token.fixed.md b/changelog.d/uk-private-hf-read-token.fixed.md new file mode 100644 index 000000000..50f27be5a --- /dev/null +++ b/changelog.d/uk-private-hf-read-token.fixed.md @@ -0,0 +1 @@ +Use a dedicated read-only Hugging Face credential for private UK runtime data, synchronize GitHub-owned runtime secrets before Cloud Run deployment, and verify their values and access bindings. diff --git a/docs/engineering/uk-private-hugging-face-credential.md b/docs/engineering/uk-private-hugging-face-credential.md new file mode 100644 index 000000000..1bc1c6134 --- /dev/null +++ b/docs/engineering/uk-private-hugging-face-credential.md @@ -0,0 +1,65 @@ +# UK private-data Hugging Face credential + +API and simulation deployments use one read-only Hugging Face credential for +the private UK runtime data. Its managed name is +`PE_UK_PRIVATE_HF_READ_TOKEN`. + +## Required access + +The credential must be a fine-grained token named +`pe-uk-private-hf-read-token` with read-only access to: + +- the model repository `policyengine/policyengine-uk-data-private`; +- the dataset repository `policyengine/populace-uk-private`. + +It must not have repository write access. The deployed API and simulation +paths do not use `policyengine/populace-uk-staging`; build and publication +credentials for that repository are separate. + +## API v1 path + +The selected-repository GitHub organization secret +`PE_UK_PRIVATE_HF_READ_TOKEN` is copied to the Google Secret Manager resource +`pe-uk-private-hf-read-token`. It is one member of the GitHub-owned runtime +secret batch, alongside the environment-specific database password and the +shared GitHub microdata and OpenAI credentials. + +Each staging and production deployment synchronizes the shared three-secret +batch before deploying its Cloud Run candidate. Production additionally +synchronizes the existing GitHub-owned database password to +`policyengine-api-prod-db-password`. Staging deliberately leaves +`policyengine-api-staging-db-password` untouched: its value differs from the +production password and its canonical value is managed directly in Secret +Manager rather than duplicated in GitHub. Synchronization compares SHA-256 +digests without logging either value and creates a new secret version only when +the value changed. It then reads back and verifies every synchronized value. + +The synchronization script resolves the runtime identity from the deployed +`policyengine-api-staging` or `policyengine-api` service. It grants that identity +`roles/secretmanager.secretAccessor` on every resource synchronized for that +environment and verifies each IAM binding. A manually dispatched workflow +performs the corresponding staging or production operation when credentials +need to be rotated independently of a deployment. + +Cloud Run exposes the resource as `HUGGING_FACE_TOKEN` because PolicyEngine +Core reads that compatibility environment variable. `HUGGING_FACE_TOKEN` is +not the name of the stored GitHub or Google Cloud credential. + +The pull-request environment check verifies the token identity, confirms that +it has no write permission, and checks both required private repositories. The +Cloud Run candidate resolver verifies that the exact deployed revision binds +`HUGGING_FACE_TOKEN` to `pe-uk-private-hf-read-token:latest`, without printing +the credential. + +The simulation-entry Cloud Run service does not receive this credential. It +submits requests to Modal and does not download model data. + +## Rotation and retirement + +Provision the replacement credential before merging workflow changes. Deploy +and verify staging before production. Keep the previous Google Secret Manager +resource during the rollback period; then disable its version before deleting +the resource. + +Do not remove the organization secret named `HUGGING_FACE_TOKEN` as part of +this migration. Other PolicyEngine repositories still consume it. diff --git a/tests/env_variables/test_environment_variables.py b/tests/env_variables/test_environment_variables.py index 64cc2176b..5dde7a2d2 100644 --- a/tests/env_variables/test_environment_variables.py +++ b/tests/env_variables/test_environment_variables.py @@ -3,6 +3,13 @@ import requests HUGGING_FACE_API_URL = "https://huggingface.co/api/whoami-v2" +HUGGING_FACE_UK_DATA_MODEL_URL = ( + "https://huggingface.co/api/models/policyengine/policyengine-uk-data-private" +) +HUGGING_FACE_UK_POPULATION_DATASET_URL = ( + "https://huggingface.co/api/datasets/policyengine/populace-uk-private" +) +EXPECTED_HUGGING_FACE_TOKEN_NAME = "pe-uk-private-hf-read-token" GITHUB_API_URL = "https://api.github.com/user" @@ -32,6 +39,48 @@ def test_hugging_face_token(self): assert token_validation_response.status_code == 200, ( f"Invalid HUGGING_FACE_TOKEN: {token_validation_response.text}" ) + token_details = token_validation_response.json()["auth"]["accessToken"] + assert token_details["displayName"] == EXPECTED_HUGGING_FACE_TOKEN_NAME + assert token_details["role"] == "fineGrained" + fine_grained = token_details["fineGrained"] + granted_permissions = set(fine_grained.get("global", [])) + granted_permissions.update( + permission + for scope in fine_grained["scoped"] + for permission in scope["permissions"] + ) + assert "repo.content.read" in granted_permissions + assert not any("write" in permission for permission in granted_permissions) + + headers = {"Authorization": f"Bearer {token}"} + uk_data_response = requests.get( + HUGGING_FACE_UK_DATA_MODEL_URL, + headers=headers, + timeout=10, + ) + assert uk_data_response.status_code == 200, ( + "The API runtime token cannot read " + "policyengine/policyengine-uk-data-private: " + f"{uk_data_response.text}" + ) + uk_data_files = { + sibling["rfilename"] for sibling in uk_data_response.json()["siblings"] + } + assert "local_authority_weights.h5" in uk_data_files + + populace_response = requests.get( + HUGGING_FACE_UK_POPULATION_DATASET_URL, + headers=headers, + timeout=10, + ) + assert populace_response.status_code == 200, ( + "The shared API and simulation runtime token cannot read " + f"policyengine/populace-uk-private: {populace_response.text}" + ) + populace_files = { + sibling["rfilename"] for sibling in populace_response.json()["siblings"] + } + assert "populace_uk_2023.h5" in populace_files @pytest.mark.skipif( do_not_run_in_debug(), diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 653e4ea98..ec1b334f4 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -35,7 +35,14 @@ "policyengine-api-prod-github-microdata-token:latest" ), "OPENAI_API_KEY": "policyengine-api-prod-openai-api-key:latest", - "HUGGING_FACE_TOKEN": "policyengine-api-prod-hugging-face-token:latest", + "HUGGING_FACE_TOKEN": "pe-uk-private-hf-read-token:latest", +} +SYNCED_SHARED_CLOUD_RUN_SECRETS = { + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ( + "policyengine-api-prod-github-microdata-token" + ), + "OPENAI_API_KEY": "policyengine-api-prod-openai-api-key", + "PE_UK_PRIVATE_HF_READ_TOKEN": "pe-uk-private-hf-read-token", } RAW_CLOUD_RUN_SECRET_VALUES = ( "raw-db-secret-value", @@ -103,7 +110,7 @@ def _required_runtime_env() -> dict[str, str]: "POLICYENGINE_DB_PASSWORD": "raw-db-secret-value", "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": ("raw-github-secret-value"), "OPENAI_API_KEY": "raw-openai-secret-value", - "HUGGING_FACE_TOKEN": "raw-hf-secret-value", + "PE_UK_PRIVATE_HF_READ_TOKEN": "raw-hf-secret-value", "SIMULATION_ENTRYPOINT_URL": "https://simulation.example.test", "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test", "SIM_ENTRYPOINT": "cloud_run_simulation_entrypoint", @@ -122,9 +129,13 @@ def _required_runtime_env() -> dict[str, str]: } -def _run_script(path: str, env: dict[str, str]) -> subprocess.CompletedProcess[str]: +def _run_script( + path: str, + env: dict[str, str], + *args: str, +) -> subprocess.CompletedProcess[str]: return subprocess.run( - ["bash", path], + ["bash", path, *args], cwd=REPO, env=env, text=True, @@ -164,6 +175,12 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: ), }, "candidate_cloud_sql": PRODUCTION_CLOUD_SQL_INSTANCE, + "candidate_secret_env": { + "HUGGING_FACE_TOKEN": { + "name": "pe-uk-private-hf-read-token", + "key": "latest", + } + }, "updates": [], } ), @@ -221,6 +238,15 @@ def _fake_gcloud(tmp_path: Path) -> tuple[Path, Path]: for name, value in state.get( "candidate_env", {} ).items() + ] + + [ + { + "name": name, + "valueFrom": {"secretKeyRef": reference}, + } + for name, reference in state.get( + "candidate_secret_env", {} + ).items() ], } ] @@ -267,6 +293,91 @@ def _fake_gcloud_env(gcloud_path: Path, state_path: Path) -> dict[str, str]: ) +def _fake_secret_sync_gcloud(tmp_path: Path) -> tuple[Path, Path]: + state_path = tmp_path / "secret-sync-state.json" + state_path.write_text( + json.dumps( + { + "services": { + STAGING_CLOUD_RUN_SERVICE: ( + "policyengine-api-cr-staging@policyengine-api.iam." + "gserviceaccount.com" + ), + PRODUCTION_CLOUD_RUN_SERVICE: ( + "policyengine-api-cr-runtime@policyengine-api.iam." + "gserviceaccount.com" + ), + }, + "secrets": { + "policyengine-api-staging-db-password": "db-value", + "policyengine-api-prod-db-password": "db-value", + "policyengine-api-prod-openai-api-key": "old-openai-value", + "pe-uk-private-hf-read-token": "hf-value", + }, + "iam": {}, + "version_adds": [], + } + ), + encoding="utf-8", + ) + gcloud_path = tmp_path / "gcloud-secret-sync" + gcloud_path.write_text( + """#!/usr/bin/env python3 +import json +import os +import sys +from pathlib import Path + +state_path = Path(os.environ["FAKE_SECRET_SYNC_STATE"]) +state = json.loads(state_path.read_text(encoding="utf-8")) +args = sys.argv[1:] + +if args[:3] == ["run", "services", "describe"]: + print(state["services"][args[3]]) +elif args[:2] == ["secrets", "describe"]: + raise SystemExit(0 if args[2] in state["secrets"] else 1) +elif args[:2] == ["secrets", "create"]: + state["secrets"][args[2]] = "" +elif args[:3] == ["secrets", "versions", "access"]: + secret_name = args[args.index("--secret") + 1] + if secret_name not in state["secrets"]: + raise SystemExit(1) + print(state["secrets"][secret_name], end="") +elif args[:3] == ["secrets", "versions", "add"]: + secret_name = args[3] + state["secrets"][secret_name] = sys.stdin.read() + state["version_adds"].append(secret_name) +elif args[:2] == ["secrets", "add-iam-policy-binding"]: + secret_name = args[2] + member = args[args.index("--member") + 1] + members = state["iam"].setdefault(secret_name, []) + if member not in members: + members.append(member) +elif args[:2] == ["secrets", "get-iam-policy"]: + secret_name = args[2] + print( + json.dumps( + { + "bindings": [ + { + "role": "roles/secretmanager.secretAccessor", + "members": state["iam"].get(secret_name, []), + } + ] + } + ) + ) +else: + raise SystemExit(f"unexpected gcloud arguments: {args}") + +state_path.write_text(json.dumps(state), encoding="utf-8") +""", + encoding="utf-8", + ) + gcloud_path.chmod(0o755) + return gcloud_path, state_path + + def _run_simulation_version_guard( versions_response: dict, *args: str, @@ -1220,6 +1331,24 @@ def test_resolve_cloud_run_candidate_records_exact_ready_revision_and_image(tmp_ ] +def test_resolve_cloud_run_candidate_rejects_wrong_hugging_face_secret(tmp_path): + gcloud_path, state_path = _fake_gcloud(tmp_path) + state = json.loads(state_path.read_text(encoding="utf-8")) + state["candidate_secret_env"]["HUGGING_FACE_TOKEN"]["name"] = ( + "policyengine-api-prod-hugging-face-token" + ) + state_path.write_text(json.dumps(state), encoding="utf-8") + + result = _run_script( + ".github/scripts/resolve_cloud_run_candidate_state.sh", + _fake_gcloud_env(gcloud_path, state_path), + ) + + assert result.returncode == 2 + assert "binds HUGGING_FACE_TOKEN" in result.stderr + assert "expected pe-uk-private-hf-read-token:latest" in result.stderr + + def test_resolve_cloud_run_candidate_rejects_changed_revision(tmp_path): gcloud_path, state_path = _fake_gcloud(tmp_path) @@ -1979,6 +2108,16 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_commands(): staging_deploy_start, ) staging_deploy = cloud_run_staging[staging_deploy_start:staging_deploy_end] + production_deploy_start = cloud_run_production.index( + "- name: Deploy tagged Cloud Run candidate" + ) + production_deploy_end = cloud_run_production.index( + "- name: Resolve exact Cloud Run production candidate", + production_deploy_start, + ) + production_deploy = cloud_run_production[ + production_deploy_start:production_deploy_end + ] raw_secret_envs = ( "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}", ( @@ -1986,15 +2125,15 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_commands(): "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" ), "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", - "HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}", + "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}", ) for raw_secret_env in raw_secret_envs: assert raw_secret_env not in staging_deploy - assert raw_secret_env not in cloud_run_production + assert raw_secret_env not in production_deploy -def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): +def test_push_workflow_syncs_environment_specific_secrets_before_deployment(): workflow = _push_workflow() cloud_run_staging = _workflow_job_block(workflow, "deploy-cloud-run-staging") cloud_run_production = _workflow_job_block(workflow, "deploy-cloud-run-candidate") @@ -2004,20 +2143,58 @@ def test_push_workflow_release_test_step_is_the_only_raw_secret_consumer(): test_step_start, ) release_test_step = cloud_run_staging[test_step_start:test_step_end] + staging_sync_start = cloud_run_staging.index( + "- name: Synchronize shared Cloud Run runtime secrets" + ) + staging_sync_end = cloud_run_staging.index( + "- name: Capture current Cloud Run staging state", + staging_sync_start, + ) + staging_sync_step = cloud_run_staging[staging_sync_start:staging_sync_end] + production_sync_start = cloud_run_production.index( + "- name: Synchronize complete Cloud Run runtime secret batch" + ) + production_sync_end = cloud_run_production.index( + "- name: Capture current Cloud Run production state", + production_sync_start, + ) + production_sync_step = cloud_run_production[ + production_sync_start:production_sync_end + ] - raw_secret_envs = ( + release_test_secret_envs = ( "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}", ( "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: " "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" ), "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", - "HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}", + "HUGGING_FACE_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}", ) - for raw_secret_env in raw_secret_envs: + shared_sync_secret_envs = ( + ( + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: " + "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" + ), + "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", + ("PE_UK_PRIVATE_HF_READ_TOKEN: ${{ secrets.PE_UK_PRIVATE_HF_READ_TOKEN }}"), + ) + for raw_secret_env in release_test_secret_envs: assert release_test_step.count(raw_secret_env) == 1 - assert cloud_run_staging.count(raw_secret_env) == 1 - assert raw_secret_env not in cloud_run_production + for raw_secret_env in shared_sync_secret_envs: + assert staging_sync_step.count(raw_secret_env) == 1 + assert production_sync_step.count(raw_secret_env) == 1 + database_secret_env = ( + "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}" + ) + assert database_secret_env not in staging_sync_step + assert production_sync_step.count(database_secret_env) == 1 + assert "run: bash .github/scripts/sync_cloud_run_secrets.sh\n" in staging_sync_step + assert "--include-database-password" not in staging_sync_step + assert ( + "run: bash .github/scripts/sync_cloud_run_secrets.sh " + "--include-database-password" in production_sync_step + ) def test_sync_cloud_run_secrets_workflow_is_manual_and_environment_gated(): @@ -2026,7 +2203,10 @@ def test_sync_cloud_run_secrets_workflow_is_manual_and_environment_gated(): assert "workflow_dispatch:" in workflow assert "pull_request:" not in workflow assert "push:" not in workflow - assert "environment: production" in workflow + assert "deployment_environment:" in workflow + assert "- staging" in workflow + assert "- production" in workflow + assert "environment: ${{ inputs.deployment_environment }}" in workflow assert "id-token: write" in workflow assert "github.ref != 'refs/heads/master'" in workflow assert "google-github-actions/auth@v2" in workflow @@ -2044,12 +2224,129 @@ def test_sync_cloud_run_secrets_workflow_writes_expected_secret_versions(): assert "run: bash .github/scripts/sync_cloud_run_secrets.sh" in workflow assert "set +x" in script assert "--data-file=-" in script - assert "gcloud secrets add-iam-policy-binding" in script + assert "Secret Manager already matches" in script + assert "secrets versions access latest" in script + assert "secrets add-iam-policy-binding" in script + assert "secrets get-iam-policy" in script + assert "run services describe" in script assert "roles/secretmanager.secretAccessor" in script - for env_name, secret_ref in CLOUD_RUN_SECRET_MAPPINGS.items(): - secret_name = secret_ref.removesuffix(":latest") + assert ( + "CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET: " + "${{ vars.CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET }}" in workflow + ) + assert 'sync_secret POLICYENGINE_DB_PASSWORD "${db_password_secret_name}"' in script + for env_name, secret_name in SYNCED_SHARED_CLOUD_RUN_SECRETS.items(): assert f"{env_name}: ${{{{ secrets.{env_name} }}}}" in workflow assert f"sync_secret {env_name} {secret_name}" in script + assert "verify_secret_value" in script + assert "grant_secret_access" in script + + +def test_sync_cloud_run_secrets_is_idempotent_and_grants_complete_batch(tmp_path): + gcloud_path, state_path = _fake_secret_sync_gcloud(tmp_path) + secret_values = { + "POLICYENGINE_DB_PASSWORD": "db-value", + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN": "microdata-value", + "OPENAI_API_KEY": "openai-value", + "PE_UK_PRIVATE_HF_READ_TOKEN": "hf-value", + } + env = _script_env( + GCLOUD_BIN=str(gcloud_path), + FAKE_SECRET_SYNC_STATE=str(state_path), + CLOUD_RUN_PROJECT="policyengine-api", + CLOUD_RUN_REGION="us-central1", + CLOUD_RUN_SERVICE=PRODUCTION_CLOUD_RUN_SERVICE, + CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET=( + "policyengine-api-prod-db-password:latest" + ), + **secret_values, + ) + + first_result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + "--include-database-password", + ) + assert first_result.returncode == 0, first_result.stderr + first_state = json.loads(state_path.read_text(encoding="utf-8")) + assert first_state["version_adds"] == [ + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + ] + assert first_state["secrets"] == { + "policyengine-api-staging-db-password": "db-value", + "policyengine-api-prod-db-password": "db-value", + "policyengine-api-prod-github-microdata-token": "microdata-value", + "policyengine-api-prod-openai-api-key": "openai-value", + "pe-uk-private-hf-read-token": "hf-value", + } + expected_member = ( + "serviceAccount:policyengine-api-cr-runtime@policyengine-api.iam." + "gserviceaccount.com" + ) + synchronized_secret_names = { + "policyengine-api-prod-db-password", + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + "pe-uk-private-hf-read-token", + } + assert set(first_state["iam"]) == synchronized_secret_names + assert all(members == [expected_member] for members in first_state["iam"].values()) + assert all(value not in first_result.stdout for value in secret_values.values()) + + second_result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + "--include-database-password", + ) + assert second_result.returncode == 0, second_result.stderr + second_state = json.loads(state_path.read_text(encoding="utf-8")) + assert second_state["version_adds"] == first_state["version_adds"] + + +def test_sync_cloud_run_staging_batch_preserves_distinct_database_secret(tmp_path): + gcloud_path, state_path = _fake_secret_sync_gcloud(tmp_path) + env = _script_env( + GCLOUD_BIN=str(gcloud_path), + FAKE_SECRET_SYNC_STATE=str(state_path), + CLOUD_RUN_PROJECT="policyengine-api", + CLOUD_RUN_REGION="us-central1", + CLOUD_RUN_SERVICE=STAGING_CLOUD_RUN_SERVICE, + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN="microdata-value", + OPENAI_API_KEY="openai-value", + PE_UK_PRIVATE_HF_READ_TOKEN="hf-value", + ) + + result = _run_script( + ".github/scripts/sync_cloud_run_secrets.sh", + env, + ) + assert result.returncode == 0, result.stderr + state = json.loads(state_path.read_text(encoding="utf-8")) + assert state["secrets"]["policyengine-api-staging-db-password"] == "db-value" + assert "policyengine-api-staging-db-password" not in state["iam"] + assert set(state["iam"]) == { + "policyengine-api-prod-github-microdata-token", + "policyengine-api-prod-openai-api-key", + "pe-uk-private-hf-read-token", + } + + +def test_hugging_face_credential_uses_purpose_specific_managed_names(): + workflow_sources = "\n".join( + ( + _pr_workflow(), + _push_workflow(), + _sync_secrets_workflow(), + _sync_secrets_script(), + (REPO / ".github/scripts/cloud_run_env.sh").read_text(encoding="utf-8"), + ) + ) + + assert "secrets.HUGGING_FACE_TOKEN" not in workflow_sources + assert "secrets.PE_UK_PRIVATE_HF_READ_TOKEN" in workflow_sources + assert "policyengine-api-prod-hugging-face-token" not in workflow_sources + assert "pe-uk-private-hf-read-token" in workflow_sources def test_sync_cloud_run_secrets_script_is_shell_syntax_valid():