From 0d85fc73dce98d742e355f797be39385cfdd5d46 Mon Sep 17 00:00:00 2001 From: Artyom Borissov Date: Wed, 5 Aug 2026 14:04:32 +0200 Subject: [PATCH 1/7] feat: [ansible/tasks] adding on-boot reindex in an even of collation version change --- .../files/refresh-collation-version.service | 22 +++ ansible/files/refresh_collation.sh | 177 ++++++++++++++++++ ansible/tasks/setup-postgres.yml | 16 ++ 3 files changed, 215 insertions(+) create mode 100644 ansible/files/refresh-collation-version.service create mode 100644 ansible/files/refresh_collation.sh diff --git a/ansible/files/refresh-collation-version.service b/ansible/files/refresh-collation-version.service new file mode 100644 index 000000000..34a57cae5 --- /dev/null +++ b/ansible/files/refresh-collation-version.service @@ -0,0 +1,22 @@ +[Unit] +Description=Refresh collation versions and reindex affected indexes +Documentation=file:///usr/local/bin/refresh_collation.sh +Requires=postgresql.service +After=postgresql.service +# Symmetric ordering: hold the traffic-facing services until the gate finishes, +# without editing their unit files. Ordering only applies to units that exist and +# are started in the same boot, so listing an absent one is harmless. +Before=pgbouncer.service postgrest.service gotrue.service envoy.service + +[Service] +Type=oneshot +RemainAfterExit=yes +# Rely on the same generous ceiling as postgresql.service; plain REINDEX INDEX is +# atomic per index, so a kill mid-run just leaves the remainder to the backstop. +TimeoutStartSec=86400 +# Leading '-' plus the script's own `exit 0` enforce fail-open: this gate must +# never block boot. +ExecStart=-/usr/local/bin/refresh_collation.sh + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/refresh_collation.sh b/ansible/files/refresh_collation.sh new file mode 100644 index 000000000..d75f85e27 --- /dev/null +++ b/ansible/files/refresh_collation.sh @@ -0,0 +1,177 @@ +#! /usr/bin/env bash + +## Boot-time collation gate: runs once after postgresql.service and BEFORE the +## traffic-facing services (ordered via systemd). When a new AMI ships an upgraded +## glibc and/or ICU over an existing data volume, it reindexes every affected +## index and then refreshes the recorded collation versions, so no warnings or +## deferred reindexing remain once traffic is admitted. Plain (non-concurrent) +## REINDEX is used since no clients are connected yet; reindex-before-refresh also +## makes the bare REFRESH safe for both providers (libc 'c' and ICU 'i'). +## +## FAIL-OPEN: per-statement errors are logged and skipped and the script exits 0, +## so the gate never wedges boot. The platform refreshCollationVersion task +## (REINDEX CONCURRENTLY under traffic) is the backstop for oversized indexes. + +set -uo pipefail # deliberately NOT -e: we handle errors per statement (fail-open) + +# pg_database_collation_actual_version / pg_collation_actual_version were added in +# PG15 — below that there is nothing to compare against. +MIN_SERVER_VERSION_NUM=150000 + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] refresh_collation: $1" +} + +# Match the connection style used by the pg_upgrade scripts' run_sql. +run_sql() { + psql -h localhost -U supabase_admin -v ON_ERROR_STOP=1 --no-psqlrc "$@" +} + +# Retry a command a few times (Postgres should already be up via +# After=postgresql.service, but guard against a slow socket at boot). +retry() { + local attempts=$1 + shift + local i=0 + until "$@"; do + i=$((i + 1)) + if [ "$i" -ge "$attempts" ]; then + return 1 + fi + sleep 1 + done + return 0 +} + +# libc always; ICU always. Reindex-before-refresh makes ICU safe. +PROVIDERS="'c', 'i'" + +# Indexes in the CURRENT database whose columns depend on a version-mismatched +# collation (explicit or the database default). Leaf indexes ('i') in user +# schemas only: partitioned parents ('I') have no storage, and system catalogs +# index version-less collations so they are immune. Emits %I.%I-quoted names. +affected_indexes_sql() { + cat < 'template0' order by datname;") + + # c. Database-default versions (shared catalog) — after every database's + # default-collated indexes have been reindexed above. + while IFS= read -r stmt; do + [ -z "$stmt" ] && continue + log "refresh (db default) :: $stmt" + if ! run_sql -d postgres -c "$stmt"; then + log "WARN database-default refresh failed :: $stmt (continuing, fail-open)" + fi + done < <(run_sql -d postgres -Atq -c "$(database_default_refresh_sql)") + + log "done" +} + +main +exit 0 diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index c4d065451..002d38ed1 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -252,9 +252,18 @@ loop: - 'database-optimizations.service' - 'postgresql_config/postgresql.service' + - 'refresh-collation-version.service' loop_control: loop_var: 'systemd_svc_item' +- name: copy boot-time collation refresh script + ansible.builtin.copy: + dest: '/usr/local/bin/refresh_collation.sh' + group: 'root' + mode: '0755' + owner: 'root' + src: 'files/refresh_collation.sh' + - name: initialize pg required state become: true ansible.builtin.file: @@ -324,6 +333,13 @@ enabled: true name: 'postgresql' +- name: Enable boot-time collation refresh gate + when: stage2 + ansible.builtin.systemd_service: + daemon_reload: true + enabled: true + name: 'refresh-collation-version' + - name: Add lang and locale items to .bashrc when: not stage2 ansible.builtin.lineinfile: From b6e41b5a1849c4a15e1d3513bcfd150bd97d5e9c Mon Sep 17 00:00:00 2001 From: Artyom Borissov Date: Thu, 6 Aug 2026 12:24:00 +0200 Subject: [PATCH 2/7] feat: [ansible/files] moved refresh collation into adminapi static scripts --- .../pg_upgrade_scripts}/refresh_collation.sh | 28 ++++++++++--------- ansible/files/adminapi.sudoers.conf | 1 + .../files/refresh-collation-version.service | 22 --------------- ansible/tasks/internal/admin-api.yml | 1 + ansible/tasks/setup-postgres.yml | 16 ----------- 5 files changed, 17 insertions(+), 51 deletions(-) rename ansible/files/{ => admin_api_scripts/pg_upgrade_scripts}/refresh_collation.sh (83%) mode change 100644 => 100755 delete mode 100644 ansible/files/refresh-collation-version.service diff --git a/ansible/files/refresh_collation.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh old mode 100644 new mode 100755 similarity index 83% rename from ansible/files/refresh_collation.sh rename to ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh index d75f85e27..e5843f1fb --- a/ansible/files/refresh_collation.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh @@ -1,16 +1,16 @@ #! /usr/bin/env bash -## Boot-time collation gate: runs once after postgresql.service and BEFORE the -## traffic-facing services (ordered via systemd). When a new AMI ships an upgraded -## glibc and/or ICU over an existing data volume, it reindexes every affected -## index and then refreshes the recorded collation versions, so no warnings or -## deferred reindexing remain once traffic is admitted. Plain (non-concurrent) -## REINDEX is used since no clients are connected yet; reindex-before-refresh also -## makes the bare REFRESH safe for both providers (libc 'c' and ICU 'i'). +## Collation refresh helper: when a new AMI ships an upgraded glibc and/or ICU +## over an existing data volume, it reindexes every affected index and then +## refreshes the recorded collation versions, so no warnings or deferred +## reindexing remain. REINDEX ... CONCURRENTLY is used so the rebuild is safe to +## run while clients are connected; reindex-before-refresh also makes the bare +## REFRESH safe for both providers (libc 'c' and ICU 'i'). ## -## FAIL-OPEN: per-statement errors are logged and skipped and the script exits 0, -## so the gate never wedges boot. The platform refreshCollationVersion task -## (REINDEX CONCURRENTLY under traffic) is the backstop for oversized indexes. +## FAIL-OPEN: per-statement errors are logged and skipped and the script exits 0. +## A failed CONCURRENTLY rebuild may leave an INVALID leftover index (_ccnew); +## the platform refreshCollationVersion task is the backstop that retries those +## and any oversized indexes. set -uo pipefail # deliberately NOT -e: we handle errors per statement (fail-open) @@ -27,8 +27,7 @@ run_sql() { psql -h localhost -U supabase_admin -v ON_ERROR_STOP=1 --no-psqlrc "$@" } -# Retry a command a few times (Postgres should already be up via -# After=postgresql.service, but guard against a slow socket at boot). +# Retry a command a few times to guard against a not-yet-ready Postgres socket. retry() { local attempts=$1 shift @@ -123,10 +122,13 @@ process_database() { # a. Reindex affected indexes BEFORE refreshing (PostgreSQL ALTER COLLATION # docs) so a stale index is never left masked by an updated catalog. + # CONCURRENTLY keeps the table writable throughout; each statement runs on + # its own connection (autocommit) since REINDEX CONCURRENTLY cannot run + # inside a transaction block. while IFS= read -r index; do [ -z "$index" ] && continue log "reindex $db :: $index" - if ! run_sql -d "$db" -c "reindex index $index;"; then + if ! run_sql -d "$db" -c "reindex index concurrently $index;"; then log "WARN reindex failed on $db :: $index (continuing, fail-open)" fi done < <(run_sql -d "$db" -Atq -c "$(affected_indexes_sql)") diff --git a/ansible/files/adminapi.sudoers.conf b/ansible/files/adminapi.sudoers.conf index e6d27bd10..ab10665a3 100644 --- a/ansible/files/adminapi.sudoers.conf +++ b/ansible/files/adminapi.sudoers.conf @@ -14,6 +14,7 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st %adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/check.sh %adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/common.sh %adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/pgsodium_getkey.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/refresh_collation.sh %adminapi ALL= NOPASSWD: /usr/bin/systemctl daemon-reload # pgBackRest wrapper scripts: constrained helpers called by supabase-admin-agent. # pgdata-chown runs as root (default); pgdata-signal runs as postgres so it can diff --git a/ansible/files/refresh-collation-version.service b/ansible/files/refresh-collation-version.service deleted file mode 100644 index 34a57cae5..000000000 --- a/ansible/files/refresh-collation-version.service +++ /dev/null @@ -1,22 +0,0 @@ -[Unit] -Description=Refresh collation versions and reindex affected indexes -Documentation=file:///usr/local/bin/refresh_collation.sh -Requires=postgresql.service -After=postgresql.service -# Symmetric ordering: hold the traffic-facing services until the gate finishes, -# without editing their unit files. Ordering only applies to units that exist and -# are started in the same boot, so listing an absent one is harmless. -Before=pgbouncer.service postgrest.service gotrue.service envoy.service - -[Service] -Type=oneshot -RemainAfterExit=yes -# Rely on the same generous ceiling as postgresql.service; plain REINDEX INDEX is -# atomic per index, so a kill mid-run just leaves the remainder to the backstop. -TimeoutStartSec=86400 -# Leading '-' plus the script's own `exit 0` enforce fail-open: this gate must -# never block boot. -ExecStart=-/usr/local/bin/refresh_collation.sh - -[Install] -WantedBy=multi-user.target diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index 6affb840c..39d43569b 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -66,6 +66,7 @@ - { file: "prepare.sh" } - { file: "pgsodium_getkey.sh" } - { file: "common.sh" } + - { file: "refresh_collation.sh" } - name: adminapi - create service file template: diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 002d38ed1..c4d065451 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -252,18 +252,9 @@ loop: - 'database-optimizations.service' - 'postgresql_config/postgresql.service' - - 'refresh-collation-version.service' loop_control: loop_var: 'systemd_svc_item' -- name: copy boot-time collation refresh script - ansible.builtin.copy: - dest: '/usr/local/bin/refresh_collation.sh' - group: 'root' - mode: '0755' - owner: 'root' - src: 'files/refresh_collation.sh' - - name: initialize pg required state become: true ansible.builtin.file: @@ -333,13 +324,6 @@ enabled: true name: 'postgresql' -- name: Enable boot-time collation refresh gate - when: stage2 - ansible.builtin.systemd_service: - daemon_reload: true - enabled: true - name: 'refresh-collation-version' - - name: Add lang and locale items to .bashrc when: not stage2 ansible.builtin.lineinfile: From 05b28884f077e36cd9e0d2a5dfd44f11b72ebf45 Mon Sep 17 00:00:00 2001 From: Artyom Borissov Date: Fri, 7 Aug 2026 11:53:32 +0200 Subject: [PATCH 3/7] fix: [ansible/files] addressing comments --- .../publish-nix-pgupgrade-scripts.yml | 1 + .../pg_upgrade_scripts/refresh_collation.sh | 271 +++++++++++++----- audit-specs/baselines/baseline.yml | 6 + 3 files changed, 204 insertions(+), 74 deletions(-) diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 810bfb8bb..6202a1efd 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -8,6 +8,7 @@ on: paths: - '.github/workflows/publish-nix-pgupgrade-scripts.yml' - 'ansible/vars.yml' + - 'ansible/files/admin_api_scripts/pg_upgrade_scripts/**' workflow_dispatch: inputs: postgresVersion: diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh index e5843f1fb..ac9957691 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh @@ -1,33 +1,47 @@ #! /usr/bin/env bash -## Collation refresh helper: when a new AMI ships an upgraded glibc and/or ICU -## over an existing data volume, it reindexes every affected index and then -## refreshes the recorded collation versions, so no warnings or deferred -## reindexing remain. REINDEX ... CONCURRENTLY is used so the rebuild is safe to -## run while clients are connected; reindex-before-refresh also makes the bare -## REFRESH safe for both providers (libc 'c' and ICU 'i'). +## Rebuilds affected indexes and refreshes recorded collation versions after an +## AMI ships new glibc/ICU over existing data. Invoked on demand by adminapi (not +## a boot service); the exit code is the signal: 0 = done / nothing to do, +## 1 = something failed. Per-statement errors don't abort — we fix what we can, +## then exit non-zero (like the sibling pg_upgrade scripts). Reindex runs BEFORE +## refresh so a stale index is never masked by an updated catalog; a failed reindex +## skips that database's refresh to preserve the signal. ## -## FAIL-OPEN: per-statement errors are logged and skipped and the script exits 0. -## A failed CONCURRENTLY rebuild may leave an INVALID leftover index (_ccnew); -## the platform refreshCollationVersion task is the backstop that retries those -## and any oversized indexes. +## SECURITY: object names never reach the shell. Enumeration returns integer OIDs +## only; DDL is built and run server-side via format('%I',...) + \gexec, so a +## hostile collation/index name (any user with CREATE on a schema) can't inject a +## psql meta-command (\!) to get a root shell via this script's sudoers grant. -set -uo pipefail # deliberately NOT -e: we handle errors per statement (fail-open) +set -uo pipefail # deliberately NOT -e: per-statement errors are handled inline -# pg_database_collation_actual_version / pg_collation_actual_version were added in -# PG15 — below that there is nothing to compare against. +# pg_database_collation_actual_version() + datcollversion are PG15+; nothing to +# compare against below that. (The collation-level function is older, but this +# database-level gate sets the floor.) MIN_SERVER_VERSION_NUM=150000 +# libc always; ICU always. Reindex-before-refresh makes ICU safe. +PROVIDERS="'c', 'i'" + +# Caps REINDEX CONCURRENTLY's brief locks so one idle-in-transaction client can't +# block it forever. Passed via PGOPTIONS, not a ;-joined SET — that would open a +# transaction block, which REINDEX CONCURRENTLY refuses to run inside. +REINDEX_LOCK_TIMEOUT_MS=2000 + +# Set by any enumeration/reindex/refresh failure; becomes the exit code. +SCRIPT_FAILED=0 + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] refresh_collation: $1" } -# Match the connection style used by the pg_upgrade scripts' run_sql. +# ON_ERROR_STOP makes psql exit non-zero on any SQL error (incl. inside \gexec) — +# that is how per-statement failure is detected below. run_sql() { psql -h localhost -U supabase_admin -v ON_ERROR_STOP=1 --no-psqlrc "$@" } -# Retry a command a few times to guard against a not-yet-ready Postgres socket. +# Retry to guard against a not-yet-ready Postgres socket. retry() { local attempts=$1 shift @@ -42,14 +56,22 @@ retry() { return 0 } -# libc always; ICU always. Reindex-before-refresh makes ICU safe. -PROVIDERS="'c', 'i'" +# Wrap a db name in dbname='...' (escaping \ and ') so characters special to -d +# parsing (=, spaces, quotes) stay part of a literal name, not a conninfo fragment. +conninfo_for_db() { + local d="$1" + d="${d//\\/\\\\}" + d="${d//\'/\\\'}" + printf "dbname='%s'" "$d" +} + +# --- Enumeration queries: emit integer OIDs only (never object names) ---------- -# Indexes in the CURRENT database whose columns depend on a version-mismatched -# collation (explicit or the database default). Leaf indexes ('i') in user -# schemas only: partitioned parents ('I') have no storage, and system catalogs -# index version-less collations so they are immune. Emits %I.%I-quoted names. -affected_indexes_sql() { +# Leaf indexes ('i') in user schemas depending on a stale collation (explicit or +# the db default). Partitioned parents ('I') have no storage; catalogs use +# version-less collations; temp schemas are skipped (reindexing another session's +# temp index fails). +affected_index_oids_sql() { cat < 'template0' order by datname;")" + rc=$? + if [ "$rc" -ne 0 ]; then + log "WARN could not enumerate databases (psql rc=$rc)" + SCRIPT_FAILED=1 + return + fi while IFS= read -r db; do [ -z "$db" ] && continue process_database "$db" - done < <(run_sql -d postgres -Atq -c "select datname from pg_database where datallowconn and datname <> 'template0' order by datname;") - - # c. Database-default versions (shared catalog) — after every database's - # default-collated indexes have been reindexed above. - while IFS= read -r stmt; do - [ -z "$stmt" ] && continue - log "refresh (db default) :: $stmt" - if ! run_sql -d postgres -c "$stmt"; then - log "WARN database-default refresh failed :: $stmt (continuing, fail-open)" - fi - done < <(run_sql -d postgres -Atq -c "$(database_default_refresh_sql)") + done <<< "$dbs" + + # c. Database-default versions (shared catalog), after every db's default- + # collated indexes were reindexed above. Includes template0. + oids="$(run_sql -d postgres -Atq -c "$(stale_db_default_oids_sql)")" + rc=$? + if [ "$rc" -ne 0 ]; then + log "WARN could not enumerate stale database defaults (psql rc=$rc)" + SCRIPT_FAILED=1 + else + while IFS= read -r oid; do + [ -z "$oid" ] && continue + if ! [[ "$oid" =~ ^[0-9]+$ ]]; then + log "WARN ignoring non-numeric database oid '$oid'" + SCRIPT_FAILED=1 + continue + fi + log "refresh (db default) :: database oid $oid" + if ! refresh_db_default_by_oid "$oid"; then + log "WARN database-default refresh failed :: database oid $oid (continuing)" + SCRIPT_FAILED=1 + fi + done <<< "$oids" + fi - log "done" + log "done (failed=$SCRIPT_FAILED)" } main -exit 0 +exit "$SCRIPT_FAILED" diff --git a/audit-specs/baselines/baseline.yml b/audit-specs/baselines/baseline.yml index 0c7810c62..395366c1d 100644 --- a/audit-specs/baselines/baseline.yml +++ b/audit-specs/baselines/baseline.yml @@ -145,6 +145,12 @@ file: owner: "1006" group: "0" filetype: file + /etc/adminapi/pg_upgrade_scripts/refresh_collation.sh: + exists: true + mode: "0755" + owner: "1006" + group: "0" + filetype: file /etc/alternatives/README: exists: true mode: "0644" From 5132fbf0573206c61e5b42f00e6bd8df7ce52dd5 Mon Sep 17 00:00:00 2001 From: Artyom Borissov Date: Fri, 7 Aug 2026 13:02:13 +0200 Subject: [PATCH 4/7] fix: [ansible/files] shfmt formatting --- .../pg_upgrade_scripts/refresh_collation.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh index ac9957691..1b368ebdd 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh @@ -189,7 +189,7 @@ process_database() { fi while IFS= read -r oid; do [ -z "$oid" ] && continue - if ! [[ "$oid" =~ ^[0-9]+$ ]]; then + if ! [[ $oid =~ ^[0-9]+$ ]]; then log "WARN ignoring non-numeric index oid '$oid' on $db" SCRIPT_FAILED=1 reindex_failed=1 @@ -201,7 +201,7 @@ process_database() { SCRIPT_FAILED=1 reindex_failed=1 fi - done <<< "$oids" + done <<<"$oids" # b. Refresh collation versions — only if every reindex succeeded, else we'd # erase the signal that the index still needs rebuilding. @@ -219,7 +219,7 @@ process_database() { fi while IFS= read -r oid; do [ -z "$oid" ] && continue - if ! [[ "$oid" =~ ^[0-9]+$ ]]; then + if ! [[ $oid =~ ^[0-9]+$ ]]; then log "WARN ignoring non-numeric collation oid '$oid' on $db" SCRIPT_FAILED=1 continue @@ -229,7 +229,7 @@ process_database() { log "WARN collation refresh failed on $db :: collation oid $oid (continuing)" SCRIPT_FAILED=1 fi - done <<< "$oids" + done <<<"$oids" } main() { @@ -250,7 +250,7 @@ main() { # Unknown/non-numeric version must not pass the floor check — skip, don't proceed. local svn svn="$(run_sql -d postgres -Atq -c "select current_setting('server_version_num');")" || svn="" - if ! [[ "$svn" =~ ^[0-9]+$ ]]; then + if ! [[ $svn =~ ^[0-9]+$ ]]; then log "could not read a numeric server_version_num (got '${svn}'); skipping" return 0 fi @@ -270,7 +270,7 @@ main() { while IFS= read -r db; do [ -z "$db" ] && continue process_database "$db" - done <<< "$dbs" + done <<<"$dbs" # c. Database-default versions (shared catalog), after every db's default- # collated indexes were reindexed above. Includes template0. @@ -282,7 +282,7 @@ main() { else while IFS= read -r oid; do [ -z "$oid" ] && continue - if ! [[ "$oid" =~ ^[0-9]+$ ]]; then + if ! [[ $oid =~ ^[0-9]+$ ]]; then log "WARN ignoring non-numeric database oid '$oid'" SCRIPT_FAILED=1 continue @@ -292,7 +292,7 @@ main() { log "WARN database-default refresh failed :: database oid $oid (continuing)" SCRIPT_FAILED=1 fi - done <<< "$oids" + done <<<"$oids" fi log "done (failed=$SCRIPT_FAILED)" From 7e25275c801de940b97514dbadf72c600f35a756 Mon Sep 17 00:00:00 2001 From: Artyom Borissov Date: Tue, 11 Aug 2026 12:34:26 +0200 Subject: [PATCH 5/7] fix: [ansible/files] addressing comments --- .../pg_upgrade_scripts/refresh_collation.sh | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh index 1b368ebdd..aee372fed 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh @@ -2,8 +2,10 @@ ## Rebuilds affected indexes and refreshes recorded collation versions after an ## AMI ships new glibc/ICU over existing data. Invoked on demand by adminapi (not -## a boot service); the exit code is the signal: 0 = done / nothing to do, -## 1 = something failed. Per-statement errors don't abort — we fix what we can, +## a boot service); the exit code is the signal: 0 = work done, or a legitimate +## no-op (replica, pre-PG15); 1 = something failed, OR preconditions could not be +## established (Postgres unreachable) so we don't actually know the collation +## state. Per-statement errors don't abort — we fix what we can, ## then exit non-zero (like the sibling pg_upgrade scripts). Reindex runs BEFORE ## refresh so a stale index is never masked by an updated catalog; a failed reindex ## skips that database's refresh to preserve the signal. @@ -71,6 +73,17 @@ conninfo_for_db() { # the db default). Partitioned parents ('I') have no storage; catalogs use # version-less collations; temp schemas are skipped (reindexing another session's # temp index fails). +# +# SCOPE / KNOWN LIMITATION (follow-up): detection keys off pg_index.indcollation — +# the collations of the index's KEY COLUMNS only. Collation dependencies that live +# elsewhere are NOT detected and NOT rebuilt: partial-index predicates +# (pg_index.indpred), CHECK constraints (pg_constraint), and partition bound +# expressions (pg_class.relpartbound). The refresh step below bumps the recorded +# version for EVERY stale collation regardless of where it is used, so those +# dependencies get stamped "refreshed" without any revalidation. The intended +# follow-up surfaces them via the adminapi advisory channel rather than silently +# refreshing. See: +# https://github.com/supabase/postgres/pull/2343#discussion_r3756738261 affected_index_oids_sql() { cat < Date: Tue, 11 Aug 2026 14:39:34 +0200 Subject: [PATCH 6/7] fix: [ansible/files] adding status file for exclusion indexes to be raised to user --- .../pg_upgrade_scripts/refresh_collation.sh | 106 +++++++++++++++++- 1 file changed, 103 insertions(+), 3 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh index aee372fed..76c715d2e 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh @@ -33,6 +33,12 @@ REINDEX_LOCK_TIMEOUT_MS=2000 # Set by any enumeration/reindex/refresh failure; becomes the exit code. SCRIPT_FAILED=0 +# Advisory snapshot consumed by adminapi (api/refresh_collation.go). Rewritten +# fresh every run; readable by the adminapi user (0644 — index names, not secret). +ADVISORY_FILE="/tmp/collation-refresh-status.json" +ADVISORY_NDJSON="" # per-run temp NDJSON accumulator; set in main() +ADVISORY_DBS="" # per-run temp file of databases that produced advisories + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] refresh_collation: $1" } @@ -84,7 +90,10 @@ conninfo_for_db() { # follow-up surfaces them via the adminapi advisory channel rather than silently # refreshing. See: # https://github.com/supabase/postgres/pull/2343#discussion_r3756738261 -affected_index_oids_sql() { +# Shared WITH clause: collations (affected_coll) and the current db's default +# (affected_default) whose recorded version is stale. Used by both the reindex +# enumeration and the advisory query so their notion of "affected" cannot drift. +_affected_ctes() { cat <>"$ADVISORY_DBS" + return + fi + + # Exclusion/invalid indexes we cannot rebuild automatically: record an advisory, + # do NOT fail, and skip this database's refresh entirely (named collations AND + # the db default in section c) so a stale version is never stamped over an + # un-rebuilt index. The advisory persists until the customer reindexes manually. + local advisories + advisories="$(run_sql -d "$conn" -Atq -c "$(affected_advisory_sql)")" + rc=$? + if [ "$rc" -ne 0 ]; then + log "WARN could not enumerate advisory indexes on $db (psql rc=$rc); skipping refresh to preserve signal" + SCRIPT_FAILED=1 + printf '%s\n' "$db" >>"$ADVISORY_DBS" + return + fi + if [ -n "$advisories" ]; then + printf '%s\n' "$advisories" >>"$ADVISORY_NDJSON" + printf '%s\n' "$db" >>"$ADVISORY_DBS" + log "advisory: $db has exclusion/invalid indexes needing manual REINDEX; skipping this DB's collation refresh" return fi @@ -246,6 +315,13 @@ process_database() { } main() { + # Fresh, readable, empty snapshot up front; overwritten at the end if findings. + printf '[]' >"$ADVISORY_FILE" 2>/dev/null || true + chmod 0644 "$ADVISORY_FILE" 2>/dev/null || true + ADVISORY_NDJSON="$(mktemp)" + ADVISORY_DBS="$(mktemp)" + trap 'rm -f "$ADVISORY_NDJSON" "$ADVISORY_DBS"' EXIT + if ! retry 8 pg_isready -h localhost -U supabase_admin -d postgres; then log "postgres not ready after retries; could not run (reporting failure)" SCRIPT_FAILED=1 @@ -273,7 +349,7 @@ main() { return 0 fi - local dbs rc db oids oid + local dbs rc db oids oid dbname adv_json dbs="$(run_sql -d postgres -Atq -c "select datname from pg_database where datallowconn and datname <> 'template0' order by datname;")" rc=$? if [ "$rc" -ne 0 ]; then @@ -301,6 +377,17 @@ main() { SCRIPT_FAILED=1 continue fi + dbname="$(run_sql -d postgres -Atq -c "select datname from pg_database where oid = $oid;")" + rc=$? + if [ "$rc" -ne 0 ] || [ -z "$dbname" ]; then + log "WARN could not resolve datname for database oid $oid (psql rc=$rc); skipping db-default refresh to avoid masking advisories" + SCRIPT_FAILED=1 + continue + fi + if grep -qxF "$dbname" "$ADVISORY_DBS"; then + log "skipping db-default refresh for $dbname: outstanding collation advisories" + continue + fi log "refresh (db default) :: database oid $oid" if ! refresh_db_default_by_oid "$oid"; then log "WARN database-default refresh failed :: database oid $oid (continuing)" @@ -309,6 +396,19 @@ main() { done <<<"$oids" fi + # Assemble the JSON array snapshot from per-DB NDJSON. Fail-open: any problem + # leaves the "[]" written at entry. jq matches the sibling scripts' JSON tooling. + if [ -s "$ADVISORY_NDJSON" ]; then + if adv_json="$(jq -s '.' "$ADVISORY_NDJSON" 2>/dev/null)"; then + printf '%s\n' "$adv_json" >"${ADVISORY_FILE}.tmp" && + mv "${ADVISORY_FILE}.tmp" "$ADVISORY_FILE" + chmod 0644 "$ADVISORY_FILE" 2>/dev/null || true + else + log "WARN could not assemble advisory JSON; leaving prior snapshot" + SCRIPT_FAILED=1 + fi + fi + log "done (failed=$SCRIPT_FAILED)" } From df871d332759c7107f8f75567d707a7678d3c5ec Mon Sep 17 00:00:00 2001 From: Artyom Borissov Date: Tue, 11 Aug 2026 15:52:23 +0200 Subject: [PATCH 7/7] fix: [ansible/files] shfmt fix --- .../admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh index 76c715d2e..1b464436f 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh @@ -37,7 +37,7 @@ SCRIPT_FAILED=0 # fresh every run; readable by the adminapi user (0644 — index names, not secret). ADVISORY_FILE="/tmp/collation-refresh-status.json" ADVISORY_NDJSON="" # per-run temp NDJSON accumulator; set in main() -ADVISORY_DBS="" # per-run temp file of databases that produced advisories +ADVISORY_DBS="" # per-run temp file of databases that produced advisories log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] refresh_collation: $1"