From d215932e23e18d683c5312e672b37a28c59ba84b Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Fri, 12 Jun 2026 12:28:36 -0400 Subject: [PATCH 01/15] actionlint: Consolidate config files --- .github/actionlint.yaml | 6 +++++- .github/actionlint.yml | 4 ---- 2 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 .github/actionlint.yml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index f238fb7913..9d8ae12db1 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -2,8 +2,12 @@ self-hosted-runner: labels: - aarch64-darwin - aarch64-linux - - blacksmith-32vcpu-ubuntu-2404 + - arm-native-runner - blacksmith-2vcpu-ubuntu-2404 - blacksmith-2vcpu-ubuntu-2404-arm - blacksmith-4vcpu-ubuntu-2404 + - blacksmith-4vcpu-ubuntu-2404-arm + - blacksmith-8vcpu-ubuntu-2404 + - blacksmith-32vcpu-ubuntu-2404 - large-linux-arm + - large-linux-x86 diff --git a/.github/actionlint.yml b/.github/actionlint.yml deleted file mode 100644 index eaf4d7d50d..0000000000 --- a/.github/actionlint.yml +++ /dev/null @@ -1,4 +0,0 @@ -self-hosted-runner: - labels: - - blacksmith-2vcpu-ubuntu-2404-arm - - blacksmith-4vcpu-ubuntu-2404 From 92677a31328995e82d7d6bc2f1b2aa9d9a571125 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Fri, 19 Jun 2026 14:42:27 -0400 Subject: [PATCH 02/15] gha/nix-build: Replace non-breaking-space with normal space Doesn't make sense to have non-breaking space so looks fish to some tools, rightfully so. --- .github/workflows/nix-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 1b5720e829..e3a102487a 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -10,7 +10,7 @@ on: permissions: id-token: write - # required by testinfra-ami-build dependent workflows + # required by testinfra-ami-build dependent workflows contents: write packages: write From d3ee055f8957900d57e8e8b79f02eef3bfaa4018 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Mon, 15 Jun 2026 16:16:11 -0400 Subject: [PATCH 03/15] gha/build-ami: Refactor build-ami calls No need to set EXECUTION_ID var and then use it for a differently named var later, just export the expected name from the beginning. I also sorted the nix/build-ami args for an easier time comparing to Release AMI Nix workflow. --- .github/actions/build-ami/action.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index a9c903c966..cb58d556be 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -49,9 +49,9 @@ runs: id: set-execution-id shell: bash run: | - EXECUTION_ID="${{ github.run_id }}-${{ inputs.postgres_version }}-${{ inputs.arch }}" - echo "EXECUTION_ID=$EXECUTION_ID" >> $GITHUB_ENV - echo "execution_id=$EXECUTION_ID" >> $GITHUB_OUTPUT + PACKER_EXECUTION_ID="${{ github.run_id }}-${{ inputs.postgres_version }}-${{ inputs.arch }}" + echo "PACKER_EXECUTION_ID=$PACKER_EXECUTION_ID" >> $GITHUB_ENV + echo "execution_id=$PACKER_EXECUTION_ID" >> $GITHUB_OUTPUT - name: Generate common-nix.vars.pkr.hcl id: generate-vars @@ -73,10 +73,11 @@ runs: AWS_REGION: ${{ inputs.region }} run: | nix run .#build-ami -- stage1 ${{ inputs.arch }} \ - -var "git-head-version=${{ inputs.git_sha }}" \ - -var "packer-execution-id=${{ env.EXECUTION_ID }}" \ - -var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \ + -var "ami_name=${{ inputs.ami_name_prefix }}" \ -var 'ami_regions=${{ inputs.ami_regions }}' \ + -var "ansible_arguments=-e postgresql_major=${{ inputs.postgres_version }}" \ + -var "git-head-version=${{ inputs.git_sha }}" \ + -var "packer-execution-id=$PACKER_EXECUTION_ID" \ amazon-${{ inputs.arch }}-nix.pkr.hcl - name: Build AMI stage 2 @@ -85,16 +86,15 @@ runs: env: POSTGRES_MAJOR_VERSION: ${{ inputs.postgres_version }} POSTGRES_VERSION: ${{ steps.generate-vars.outputs.version }} - PACKER_EXECUTION_ID: ${{ env.EXECUTION_ID }} AWS_MAX_ATTEMPTS: 10 AWS_RETRY_MODE: adaptive AWS_REGION: ${{ inputs.region }} run: | nix run .#build-ami -- stage2 ${{ inputs.arch }} \ - -var "git-head-version=${{ inputs.git_sha }}" \ - -var "packer-execution-id=${{ env.EXECUTION_ID }}" \ - -var "postgres_major_version=${{ inputs.postgres_version }}" \ -var "ami_name=${{ inputs.ami_name_prefix }}" \ + -var "git-head-version=${{ inputs.git_sha }}" \ -var "git_sha=${{ inputs.git_sha }}" \ -var "instance_type=${{ inputs.instance_type }}" \ + -var "packer-execution-id=$PACKER_EXECUTION_ID" \ + -var "postgres_major_version=${{ inputs.postgres_version }}" \ stage2-nix-psql.pkr.hcl From c33fc31c3d927b7cf850bfc60934af76848de7b7 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Mon, 15 Jun 2026 16:16:11 -0400 Subject: [PATCH 04/15] gha/build-ami: Make ready for Release AMI Nix use Release AMI Nix ends up always building stage1 AMI so lets take in a force input that will pass it along as BUILD_AMI_NIX_FORCE_BUILD. Setting the env is behind an `if` since nix/build-ami will actually treat it as a tri-state. Passing in GIT_SHA too so that nix/build-ami can use it for finding the AMI just like Release AMI Nix does. --- .github/actions/build-ami/action.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index cb58d556be..8261f5442b 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -11,6 +11,9 @@ inputs: arch: description: Architecture to build AMI for (amd64|arm64) required: true + force: + description: Force building stage1 AMI + required: false git_sha: description: 'Git SHA for this build' required: true @@ -71,7 +74,12 @@ runs: AWS_MAX_ATTEMPTS: 10 AWS_RETRY_MODE: adaptive AWS_REGION: ${{ inputs.region }} + GIT_SHA: ${{ inputs.git_sha }} run: | + if [[ -n "${{ inputs.force }}" ]]; then + export BUILD_AMI_NIX_FORCE_BUILD="${{ inputs.force }}" + fi + nix run .#build-ami -- stage1 ${{ inputs.arch }} \ -var "ami_name=${{ inputs.ami_name_prefix }}" \ -var 'ami_regions=${{ inputs.ami_regions }}' \ @@ -89,6 +97,7 @@ runs: AWS_MAX_ATTEMPTS: 10 AWS_RETRY_MODE: adaptive AWS_REGION: ${{ inputs.region }} + GIT_SHA: ${{ inputs.git_sha }} run: | nix run .#build-ami -- stage2 ${{ inputs.arch }} \ -var "ami_name=${{ inputs.ami_name_prefix }}" \ From feda3804b1d506f1c74cd3b1b05b4369d6e81153 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Tue, 16 Jun 2026 12:11:28 -0400 Subject: [PATCH 05/15] gha/build-ami: Add group log outputs These are especially helpful for local actions because GHA does not show the jobs/steps within, it all looks like one "Build AMI" step. --- .github/actions/build-ami/action.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index 8261f5442b..ef3d3e82e6 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -76,6 +76,8 @@ runs: AWS_REGION: ${{ inputs.region }} GIT_SHA: ${{ inputs.git_sha }} run: | + echo "::group::AMI Build: Stage 1" + if [[ -n "${{ inputs.force }}" ]]; then export BUILD_AMI_NIX_FORCE_BUILD="${{ inputs.force }}" fi @@ -88,6 +90,8 @@ runs: -var "packer-execution-id=$PACKER_EXECUTION_ID" \ amazon-${{ inputs.arch }}-nix.pkr.hcl + echo "::endgroup::" + - name: Build AMI stage 2 id: build-stage2 shell: bash @@ -99,6 +103,8 @@ runs: AWS_REGION: ${{ inputs.region }} GIT_SHA: ${{ inputs.git_sha }} run: | + echo "::group::AMI Build: Stage 2" + nix run .#build-ami -- stage2 ${{ inputs.arch }} \ -var "ami_name=${{ inputs.ami_name_prefix }}" \ -var "git-head-version=${{ inputs.git_sha }}" \ @@ -107,3 +113,5 @@ runs: -var "packer-execution-id=$PACKER_EXECUTION_ID" \ -var "postgres_major_version=${{ inputs.postgres_version }}" \ stage2-nix-psql.pkr.hcl + + echo "::endgroup::" From 12bd58d2e2d884541b23c45fad9d7fdbb3b047d0 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 11 Jun 2026 17:43:24 -0400 Subject: [PATCH 06/15] nix/build-ami: Use build-ami nix hash for INPUT_HASH This way when we get a hit we know its for exact same inputs. Changes to build-ami script can cause a different AMI to have been built so should be part of the INPUT_HASH calculation. Using $0 takes into account any ${packerSources} changes due to normal nix behavior. --- nix/packages/build-ami.nix | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/nix/packages/build-ami.nix b/nix/packages/build-ami.nix index f6edd54df0..48138aba77 100644 --- a/nix/packages/build-ami.nix +++ b/nix/packages/build-ami.nix @@ -1,10 +1,11 @@ { lib, stdenv, - writeShellApplication, - packer, awscli2, + coreutils, jq, + packer, + writeShellApplication, ... }: @@ -41,14 +42,13 @@ writeShellApplication { name = "build-ami"; runtimeInputs = [ - packer awscli2 + coreutils jq + packer ]; text = '' - set -euo pipefail - set -x # Parse required parameters @@ -63,11 +63,17 @@ writeShellApplication { amd64 | arm64) ;; *) echo "Error: Invalid arch '$ARCH'. Must be 'amd64' or 'arm64'" >&2 && exit 1 ;; esac + + if [[ $0 != "''${BASH_SOURCE[0]}" ]]; then + echo "This file is not to be sourced" >&2 + exit 1 + fi + INPUT_HASH=$(realpath "$0") + INPUT_HASH=''${INPUT_HASH#/nix/store/} + INPUT_HASH=''${INPUT_HASH%%-*} shift 2 REGION="''${AWS_REGION:-ap-southeast-1}" - PACKER_SOURCES="${packerSources}" - INPUT_HASH=$(basename "$PACKER_SOURCES" | cut -d- -f1) find_stage1_ami() { set +e @@ -122,7 +128,7 @@ writeShellApplication { echo "No cached AMI found" - cd "$PACKER_SOURCES" + cd ${packerSources} packer init "$@" packer build \ -var-file="development-$ARCH.vars.pkr.hcl" \ From 05bbb68623686132c0944c66745640e2071c00f2 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Mon, 15 Jun 2026 16:16:11 -0400 Subject: [PATCH 07/15] nix/build-ami: Make usable for Release AMI Nix workflow Release AMI Nix *always* builds stage1 so lets make sure we have a way to behave similarly. Also need to make sure we find the correct AMI via architecture and sourceSha. Also sync'ing up the filters used to find stage-1, added architecture and sourceSha. The architecture tag filter should be obviously ok/no-op and sourceSha was already always being passed via build-ami action so all the AMIs should have it as long as it was built in CI. It seems redundant to have both sourceSha and inputHash filters since inputHash should always be better than sourceSha, when content changes inputHash changes and sourceSha is obviously different but a different sourceSha could be from changes to unrelated files. I'll take extra builds for now though just to get some clarity here, will make it better in the near future. --- nix/packages/build-ami.nix | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/nix/packages/build-ami.nix b/nix/packages/build-ami.nix index 48138aba77..95a5410644 100644 --- a/nix/packages/build-ami.nix +++ b/nix/packages/build-ami.nix @@ -77,14 +77,24 @@ writeShellApplication { find_stage1_ami() { set +e + local arch + case $ARCH in + amd64) arch=x86_64 ;; + arm64) arch=arm64 ;; + esac + local filters=( + "Name=architecture,Values=$arch" + "Name=state,Values=available" + "Name=tag:inputHash,Values=$INPUT_HASH" + "Name=tag:postgresVersion,Values=$POSTGRES_VERSION-stage1" + "Name=tag:sourceSha,Values=$GIT_SHA" # This is set by packer via the git-head-version var which is always passed in by the build-ami action + ) + local ami_output ami_output=$(aws ec2 describe-images \ --region "$REGION" \ --owners self \ - --filters \ - "Name=tag:inputHash,Values=$INPUT_HASH" \ - "Name=tag:postgresVersion,Values=$POSTGRES_VERSION-stage1" \ - "Name=state,Values=available" \ + --filters "''${filters[@]}" \ --query 'Images[0].ImageId' \ --output text 2>&1) local exit_code=$? @@ -106,6 +116,17 @@ writeShellApplication { echo "Building stage 1..." echo "Checking for existing AMI..." + if [ -n "''${BUILD_AMI_NIX_FORCE_BUILD:-}" ]; then + if [ "''${BUILD_AMI_NIX_FORCE_BUILD:-}" == true ]; then + echo 'BUILD_AMI_NIX_FORCE_STAGE1 == true ... skip search for stage1 AMI' >&2 + find_stage1_ami() { + return + } + else + echo 'BUILD_AMI_NIX_FORCE_STAGE1 != true ... will search for stage1 AMI' >&2 + fi + fi + AMI_ID=$(find_stage1_ami) if [ -n "$AMI_ID" ]; then echo "Found existing AMI: $AMI_ID" @@ -166,8 +187,8 @@ writeShellApplication { packer build \ -var-file="development-$ARCH.vars.pkr.hcl" \ -var-file="common-nix.vars.pkr.hcl" \ - -var "source_ami=$STAGE1_AMI_ID" \ -var "region=$REGION" \ + -var "source_ami=$STAGE1_AMI_ID" \ "$@" if [ -n "''${PACKER_EXECUTION_ID:-}" ]; then From 2ed96112c2b8a8a6c791764290c7a81d6d22e8b6 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 17 Jun 2026 16:11:51 -0400 Subject: [PATCH 08/15] nix/build-ami: Change how packer build handles errors In CI it will abort so that the workflow can grab ec2 console output, hopefully catching any kernel messages there. Otherwise set to ask if stdout is a tty so that the caller can poke around if they want to debug. If stdout is not a tty then we will go with the default. --- .github/actions/build-ami/action.yml | 2 +- nix/packages/build-ami.nix | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/actions/build-ami/action.yml b/.github/actions/build-ami/action.yml index ef3d3e82e6..83602334da 100644 --- a/.github/actions/build-ami/action.yml +++ b/.github/actions/build-ami/action.yml @@ -79,7 +79,7 @@ runs: echo "::group::AMI Build: Stage 1" if [[ -n "${{ inputs.force }}" ]]; then - export BUILD_AMI_NIX_FORCE_BUILD="${{ inputs.force }}" + export BUILD_AMI_NIX_FORCE_BUILD_STAGE1="${{ inputs.force }}" fi nix run .#build-ami -- stage1 ${{ inputs.arch }} \ diff --git a/nix/packages/build-ami.nix b/nix/packages/build-ami.nix index 95a5410644..a64136a867 100644 --- a/nix/packages/build-ami.nix +++ b/nix/packages/build-ami.nix @@ -73,6 +73,16 @@ writeShellApplication { INPUT_HASH=''${INPUT_HASH%%-*} shift 2 + export PACKER_LOG=''${PACKER_LOG:-''${RUNNER_DEBUG:-0}} + on_error=ask + if ''${CI:-false}; then + echo "::notice::Setting packer build -on-error=abort since this is CI, this is different than non-CI runs!" + on_error=abort + elif ! [[ -t 0 ]]; then + echo "stdin is not a tty, so running packer build -on-error=cleanup (default) since there's no one to ask!" >&2 + on_error=cleanup + fi + REGION="''${AWS_REGION:-ap-southeast-1}" find_stage1_ami() { @@ -116,14 +126,14 @@ writeShellApplication { echo "Building stage 1..." echo "Checking for existing AMI..." - if [ -n "''${BUILD_AMI_NIX_FORCE_BUILD:-}" ]; then - if [ "''${BUILD_AMI_NIX_FORCE_BUILD:-}" == true ]; then - echo 'BUILD_AMI_NIX_FORCE_STAGE1 == true ... skip search for stage1 AMI' >&2 + if [ -n "''${BUILD_AMI_NIX_FORCE_BUILD_STAGE1:-}" ]; then + if [ "''${BUILD_AMI_NIX_FORCE_BUILD_STAGE1:-}" == true ]; then + echo 'BUILD_AMI_NIX_FORCE_BUILD_STAGE1 == true ... skip search for stage1 AMI' >&2 find_stage1_ami() { return } else - echo 'BUILD_AMI_NIX_FORCE_STAGE1 != true ... will search for stage1 AMI' >&2 + echo 'BUILD_AMI_NIX_FORCE_BUILD_STAGE1 != true ... will search for stage1 AMI' >&2 fi fi @@ -151,7 +161,7 @@ writeShellApplication { cd ${packerSources} packer init "$@" - packer build \ + packer build -on-error=$on_error \ -var-file="development-$ARCH.vars.pkr.hcl" \ -var "input-hash=$INPUT_HASH" \ -var "postgres-version=$POSTGRES_VERSION" \ @@ -184,7 +194,7 @@ writeShellApplication { echo "Found stage 1 AMI: $STAGE1_AMI_ID" packer init stage2-nix-psql.pkr.hcl - packer build \ + packer build -on-error=$on_error \ -var-file="development-$ARCH.vars.pkr.hcl" \ -var-file="common-nix.vars.pkr.hcl" \ -var "region=$REGION" \ From 500c236e281f84e3b37e0174bd920d9420173311 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Mon, 15 Jun 2026 16:16:11 -0400 Subject: [PATCH 09/15] gha/ami-release-nix: Refactor for readability This is mostly just clean ups, it seems better to add things to the env pre/post build instead of having to use the annoyingly long output syntax or re-derive/parse from matrix values or packer files. I renamed the arch specific matrix values to something that reads better, arch makes more sense as a single value representing the architecture and target is a common name for gha matrix grouping. I changed most uses of the various postgres versions from using matrix value or getting from packer file to env vars. This way we use the same name consistently throughout the file and the names are more explicit. Its easy to guess the difference between POSTGRES_MAJOR_VERSION and POSTGRES_SUPABASE_VERSION compared to postgres_version and PG_VERSION. --- .github/workflows/ami-release-nix.yml | 130 +++++++++++--------------- 1 file changed, 54 insertions(+), 76 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 932d279d94..b6ce407ebc 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -40,22 +40,22 @@ jobs: fail-fast: false matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} - arch: - - name: arm64 + target: + - arch: arm64 runner: blacksmith-2vcpu-ubuntu-2404-arm packer_template: amazon-arm64-nix.pkr.hcl vars_file: development-arm64.vars.pkr.hcl instance_type: c6g.4xlarge nix_system: aarch64-linux ami_arch_filter: arm64 - - name: amd64 + - arch: amd64 runner: blacksmith-2vcpu-ubuntu-2404 packer_template: amazon-amd64-nix.pkr.hcl vars_file: development-amd64.vars.pkr.hcl instance_type: c6i.4xlarge nix_system: x86_64-linux ami_arch_filter: x86_64 - runs-on: ${{ matrix.arch.runner }} + runs-on: ${{ matrix.target.runner }} timeout-minutes: 150 steps: @@ -78,39 +78,35 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - name: Set PostgreSQL version environment variable + - name: Setup pre build env vars run: | echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> "$GITHUB_ENV" - echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}-${{ matrix.arch.name }}" >> "$GITHUB_ENV" + echo "PACKER_EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}-${{ matrix.arch.name }}" >> "$GITHUB_ENV" - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION="$(nix run nixpkgs#yq-go -- -r '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml)" - echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl + POSTGRES_SUPABASE_VERSION="$(nix run nixpkgs#yq-go -- -r '.postgres_release["postgres'"$POSTGRES_MAJOR_VERSION"'"]' ansible/vars.yml)" + echo "postgres-version = \"$POSTGRES_SUPABASE_VERSION\"" > common-nix.vars.pkr.hcl - name: Build AMI stage 1 - env: - POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | GIT_SHA=${{github.sha}} nix run github:supabase/postgres/${GIT_SHA}#packer -- init ${{ matrix.arch.packer_template }} - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "ami_name=supabase-postgres-${{ matrix.arch.ami_arch_filter }}" ${{ matrix.arch.packer_template }} + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${PACKER_EXECUTION_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "ami_name=supabase-postgres-${{ matrix.arch.ami_arch_filter }}" ${{ matrix.arch.packer_template }} - name: Find stage 1 AMI run: | GIT_SHA=${{github.sha}} - - PG_VERSION=$(sed -n 's/postgres-version = "\(.*\)"/\1/p' common-nix.vars.pkr.hcl) REGION="us-east-1" - echo "Looking for stage 1 AMI with postgresVersion=${PG_VERSION}-stage1 and sourceSha=${GIT_SHA} in region ${REGION}" + echo "Looking for stage 1 AMI with postgresVersion=${POSTGRES_SUPABASE_VERSION}-stage1 and sourceSha=${GIT_SHA} in region ${REGION}" STAGE1_AMI_ID=$(aws ec2 describe-images \ --region "$REGION" \ --owners self \ --filters \ - "Name=tag:postgresVersion,Values=${PG_VERSION}-stage1" \ + "Name=tag:postgresVersion,Values=${POSTGRES_SUPABASE_VERSION}-stage1" \ "Name=tag:sourceSha,Values=${GIT_SHA}" \ "Name=state,Values=available" \ "Name=architecture,Values=${{ matrix.arch.ami_arch_filter }}" \ @@ -126,45 +122,33 @@ jobs: echo "STAGE1_AMI_ID=$STAGE1_AMI_ID" >> "$GITHUB_ENV" - name: Build AMI stage 2 - env: - POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | GIT_SHA=${{github.sha}} nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var "instance_type=${{ matrix.arch.instance_type }}" -var "ami_name=supabase-postgres-${{ matrix.arch.ami_arch_filter }}" stage2-nix-psql.pkr.hcl + nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${PACKER_EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var "instance_type=${{ matrix.arch.instance_type }}" -var "ami_name=supabase-postgres-${{ matrix.arch.ami_arch_filter }}" stage2-nix-psql.pkr.hcl - - name: Grab release version - id: process_release_version - run: | - VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - echo "::notice title=AMI Published::Postgres AMI version: $VERSION" + echo "::notice title=AMI Published::Postgres AMI version: $POSTGRES_SUPABASE_VERSION" - - name: Set arch-qualified version - id: arch_version + - name: Setup post build env vars run: | - VERSION="${{ steps.process_release_version.outputs.version }}" - if [ "${{ matrix.arch.name }}" = "amd64" ]; then + if [ "${{ matrix.target.arch }}" = "amd64" ]; then { - echo "version=${VERSION}" - echo "arch_suffix=-x86" - echo "release_tag=${VERSION}-x86" - } >> "$GITHUB_OUTPUT" + echo "ARCH_SUFFIX=-x86" + echo "RELEASE_TAG=${POSTGRES_SUPABASE_VERSION}-x86" + } >>"$GITHUB_ENV" else { - echo "version=${VERSION}" - echo "arch_suffix=" - echo "release_tag=${VERSION}" - } >> "$GITHUB_OUTPUT" + echo "ARCH_SUFFIX=" + echo "RELEASE_TAG=${POSTGRES_SUPABASE_VERSION}" + } >>"$GITHUB_ENV" fi - name: Create nix flake revision tarball run: | GIT_SHA=${{github.sha}} - MAJOR_VERSION=${{ matrix.postgres_version }} - mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" - echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" + mkdir -p "/tmp/pg_upgrade_bin/${POSTGRES_MAJOR_VERSION}" + echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${POSTGRES_MAJOR_VERSION}/nix_flake_version" tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" . - name: configure aws credentials - staging @@ -177,17 +161,17 @@ jobs: run: | cd ansible ansible-playbook -i localhost \ - -e "ami_release_version=${{ steps.arch_version.outputs.version }}" \ + -e "ami_release_version=$POSTGRES_SUPABASE_VERSION" \ -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ - -e "postgres_major_version=${{ matrix.postgres_version }}" \ - -e "arch=${{ matrix.arch.name }}" \ + -e "postgres_major_version=$POSTGRES_MAJOR_VERSION" \ + -e "arch=${{ matrix.target.arch }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 staging run: | - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades${{ steps.arch_version.outputs.arch_suffix }}/postgres/supabase-postgres-${{ steps.arch_version.outputs.version }}/20.04.tar.gz - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades${{ steps.arch_version.outputs.arch_suffix }}/postgres/supabase-postgres-${{ steps.arch_version.outputs.version }}/24.04.tar.gz - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades${{ steps.arch_version.outputs.arch_suffix }}/postgres/supabase-postgres-${{ steps.arch_version.outputs.version }}/upgrade_bundle.tar.gz + aws s3 cp /tmp/pg_binaries.tar.gz "s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades$ARCH_SUFFIX/postgres/supabase-postgres-$POSTGRES_SUPABASE_VERSION/20.04.tar.gz" + aws s3 cp /tmp/pg_binaries.tar.gz "s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades$ARCH_SUFFIX/postgres/supabase-postgres-$POSTGRES_SUPABASE_VERSION/24.04.tar.gz" + aws s3 cp /tmp/pg_binaries.tar.gz "s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades$ARCH_SUFFIX/postgres/supabase-postgres-$POSTGRES_SUPABASE_VERSION/upgrade_bundle.tar.gz" - name: configure aws credentials - prod uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 @@ -199,17 +183,17 @@ jobs: run: | cd ansible ansible-playbook -i localhost \ - -e "ami_release_version=${{ steps.arch_version.outputs.version }}" \ + -e "ami_release_version=$POSTGRES_SUPABASE_VERSION" \ -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ - -e "postgres_major_version=${{ matrix.postgres_version }}" \ - -e "arch=${{ matrix.arch.name }}" \ + -e "postgres_major_version=$POSTGRES_MAJOR_VERSION" \ + -e "arch=${{ matrix.target.arch }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 prod run: | - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades${{ steps.arch_version.outputs.arch_suffix }}/postgres/supabase-postgres-${{ steps.arch_version.outputs.version }}/20.04.tar.gz - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades${{ steps.arch_version.outputs.arch_suffix }}/postgres/supabase-postgres-${{ steps.arch_version.outputs.version }}/24.04.tar.gz - aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades${{ steps.arch_version.outputs.arch_suffix }}/postgres/supabase-postgres-${{ steps.arch_version.outputs.version }}/upgrade_bundle.tar.gz + aws s3 cp /tmp/pg_binaries.tar.gz "s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades$ARCH_SUFFIX/postgres/supabase-postgres-$POSTGRES_SUPABASE_VERSION/20.04.tar.gz" + aws s3 cp /tmp/pg_binaries.tar.gz "s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades$ARCH_SUFFIX/postgres/supabase-postgres-$POSTGRES_SUPABASE_VERSION/24.04.tar.gz" + aws s3 cp /tmp/pg_binaries.tar.gz "s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades$ARCH_SUFFIX/postgres/supabase-postgres-$POSTGRES_SUPABASE_VERSION/upgrade_bundle.tar.gz" - name: GitHub OIDC Auth uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 @@ -229,26 +213,24 @@ jobs: - name: Update nix store path catalog run: | - VERSION="${{ steps.process_release_version.outputs.version }}" GIT_SHA="${{ github.sha }}" - PG_VERSION="${{ matrix.postgres_version }}" - SYSTEM="${{ matrix.arch.nix_system }}" + SYSTEM="${{ matrix.target.nix_system }}" # Get store path for this build - STORE_PATH=$(nix eval --raw ".#psql_${PG_VERSION}/bin.outPath") + STORE_PATH=$(nix eval --raw ".#psql_${POSTGRES_MAJOR_VERSION}/bin.outPath") # Each postgres version gets its own catalog file (no race conditions) - CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${PG_VERSION}-${SYSTEM}.json" + CATALOG_S3="s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-psql_${POSTGRES_MAJOR_VERSION}-${SYSTEM}.json" # Create catalog JSON for this version jq -n \ - --arg ver "$VERSION" \ + --arg ver "$POSTGRES_SUPABASE_VERSION" \ --arg sha "$GIT_SHA" \ --arg sys "$SYSTEM" \ --arg path "$STORE_PATH" \ '{version: $ver, git_sha: $sha, ($sys): $path}' > /tmp/catalog.json - echo "Catalog for psql_${PG_VERSION}:" + echo "Catalog for psql_${POSTGRES_MAJOR_VERSION}:" cat /tmp/catalog.json # Upload catalog @@ -260,17 +242,16 @@ jobs: - name: Create release uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 with: - name: ${{ steps.arch_version.outputs.release_tag }} - tag_name: ${{ steps.arch_version.outputs.release_tag }} - target_commitish: ${{github.sha}} + name: ${{ env.RELEASE_TAG }} + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ github.sha }} - name: Create CLI tag for PG 17 - if: matrix.postgres_version == '17' && matrix.arch.name == 'arm64' && github.event_name != 'workflow_dispatch' + if: matrix.postgres_version == '17' && matrix.target.arch == 'arm64' && github.event_name != 'workflow_dispatch' env: GH_TOKEN: ${{ github.token }} run: | - VERSION="${{ steps.process_release_version.outputs.version }}" - CLI_TAG="v${VERSION}-cli" + CLI_TAG="v${POSTGRES_SUPABASE_VERSION}-cli" echo "Creating CLI tag: ${CLI_TAG}" git tag "${CLI_TAG}" "${{ github.sha }}" git push origin "${CLI_TAG}" @@ -280,29 +261,28 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - VERSION="${{ steps.process_release_version.outputs.version }}" - CLI_TAG="v${VERSION}-cli" + CLI_TAG="v${POSTGRES_SUPABASE_VERSION}-cli" gh workflow run cli-release.yml \ --ref "${CLI_TAG}" \ -f version="${CLI_TAG}" - name: Trigger pg_upgrade_scripts workflow - if: matrix.arch.name == 'arm64' + if: matrix.target.arch == 'arm64' env: GH_TOKEN: ${{ github.token }} run: | gh workflow run publish-nix-pgupgrade-scripts.yml \ --ref "${{ github.ref_name }}" \ - -f postgresVersion="${{ steps.process_release_version.outputs.version }}" + -f postgresVersion="$POSTGRES_SUPABASE_VERSION" - name: Trigger pg_upgrade_bin flake version workflow - if: matrix.arch.name == 'arm64' + if: matrix.target.arch == 'arm64' env: GH_TOKEN: ${{ github.token }} run: | gh workflow run publish-nix-pgupgrade-bin-flake-version.yml \ --ref "${{ github.ref_name }}" \ - -f postgresVersion="${{ steps.process_release_version.outputs.version }}" + -f postgresVersion="$POSTGRES_SUPABASE_VERSION" - name: Slack Notification on Failure if: ${{ failure() }} @@ -324,13 +304,11 @@ jobs: - name: Cleanup resources after build if: ${{ always() }} run: | - EXECUTION_ID="${{ env.EXECUTION_ID }}" - aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids - aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id + aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids + aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - EXECUTION_ID="${{ env.EXECUTION_ID }}" - aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids - aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id + aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids + aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id From 8b9308ae1a056be5af4cffd652c10f885ce61333 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Mon, 15 Jun 2026 16:16:11 -0400 Subject: [PATCH 10/15] gha/ami-release-nix: Use build-ami action No need to repeat ourselves when we have a perfectly good abstraction already. --- .github/workflows/ami-release-nix.yml | 86 ++++++++------------------- 1 file changed, 25 insertions(+), 61 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index b6ce407ebc..283a7bdc5b 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -6,9 +6,11 @@ on: - develop - release/* paths: - - '.github/workflows/ami-release-nix.yml' - - 'common-nix.vars.pkr.hcl' - - 'ansible/vars.yml' + - .github/workflows/ami-release-nix.yml + - ansible/vars.yml + - flake.lock + - flake.nix + - nix/packages/build-ami.nix workflow_dispatch: permissions: @@ -43,18 +45,10 @@ jobs: target: - arch: arm64 runner: blacksmith-2vcpu-ubuntu-2404-arm - packer_template: amazon-arm64-nix.pkr.hcl - vars_file: development-arm64.vars.pkr.hcl instance_type: c6g.4xlarge - nix_system: aarch64-linux - ami_arch_filter: arm64 - arch: amd64 runner: blacksmith-2vcpu-ubuntu-2404 - packer_template: amazon-amd64-nix.pkr.hcl - vars_file: development-amd64.vars.pkr.hcl instance_type: c6i.4xlarge - nix_system: x86_64-linux - ami_arch_filter: x86_64 runs-on: ${{ matrix.target.runner }} timeout-minutes: 150 @@ -78,59 +72,29 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - name: Setup pre build env vars - run: | - echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> "$GITHUB_ENV" - echo "PACKER_EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}-${{ matrix.arch.name }}" >> "$GITHUB_ENV" - - - name: Generate common-nix.vars.pkr.hcl - run: | - POSTGRES_SUPABASE_VERSION="$(nix run nixpkgs#yq-go -- -r '.postgres_release["postgres'"$POSTGRES_MAJOR_VERSION"'"]' ansible/vars.yml)" - echo "postgres-version = \"$POSTGRES_SUPABASE_VERSION\"" > common-nix.vars.pkr.hcl - - - name: Build AMI stage 1 - run: | - GIT_SHA=${{github.sha}} - - nix run github:supabase/postgres/${GIT_SHA}#packer -- init ${{ matrix.arch.packer_template }} - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${PACKER_EXECUTION_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" -var "region=us-east-1" -var 'ami_regions=["us-east-1"]' -var "ami_name=supabase-postgres-${{ matrix.arch.ami_arch_filter }}" ${{ matrix.arch.packer_template }} - - - name: Find stage 1 AMI - run: | - GIT_SHA=${{github.sha}} - REGION="us-east-1" - - echo "Looking for stage 1 AMI with postgresVersion=${POSTGRES_SUPABASE_VERSION}-stage1 and sourceSha=${GIT_SHA} in region ${REGION}" - - STAGE1_AMI_ID=$(aws ec2 describe-images \ - --region "$REGION" \ - --owners self \ - --filters \ - "Name=tag:postgresVersion,Values=${POSTGRES_SUPABASE_VERSION}-stage1" \ - "Name=tag:sourceSha,Values=${GIT_SHA}" \ - "Name=state,Values=available" \ - "Name=architecture,Values=${{ matrix.arch.ami_arch_filter }}" \ - --query 'Images[0].ImageId' \ - --output text) - - if [ -z "$STAGE1_AMI_ID" ] || [ "$STAGE1_AMI_ID" = "None" ]; then - echo "ERROR: Failed to find stage 1 AMI" - exit 1 - fi - - echo "Found stage 1 AMI: $STAGE1_AMI_ID" - echo "STAGE1_AMI_ID=$STAGE1_AMI_ID" >> "$GITHUB_ENV" + - name: Build AMI + id: build-ami + uses: ./.github/actions/build-ami + with: + ami_name_prefix: "supabase-postgres-${{ github.run_id }}-${{ matrix.target.arch }}" + ami_regions: '["us-east-1"]' + arch: ${{ matrix.target.arch }} + force: "true" + git_sha: ${{ github.sha }} + instance_type: ${{ matrix.target.instance_type }} + postgres_version: ${{ matrix.postgres_version }} + region: us-east-1 - - name: Build AMI stage 2 + - name: Setup post build env vars run: | - GIT_SHA=${{github.sha}} - nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl - nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${PACKER_EXECUTION_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var "source_ami=${STAGE1_AMI_ID}" -var-file="${{ matrix.arch.vars_file }}" -var-file="common-nix.vars.pkr.hcl" -var "region=us-east-1" -var "instance_type=${{ matrix.arch.instance_type }}" -var "ami_name=supabase-postgres-${{ matrix.arch.ami_arch_filter }}" stage2-nix-psql.pkr.hcl - + POSTGRES_SUPABASE_VERSION=${{ steps.build-ami.outputs.postgres_release_version }} echo "::notice title=AMI Published::Postgres AMI version: $POSTGRES_SUPABASE_VERSION" + { + echo "PACKER_EXECUTION_ID=${{ steps.build-ami.outputs.execution_id }}" + echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" + echo "POSTGRES_SUPABASE_VERSION=$POSTGRES_SUPABASE_VERSION" + } >>"$GITHUB_ENV" - - name: Setup post build env vars - run: | if [ "${{ matrix.target.arch }}" = "amd64" ]; then { echo "ARCH_SUFFIX=-x86" @@ -214,7 +178,7 @@ jobs: - name: Update nix store path catalog run: | GIT_SHA="${{ github.sha }}" - SYSTEM="${{ matrix.target.nix_system }}" + SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') # Get store path for this build STORE_PATH=$(nix eval --raw ".#psql_${POSTGRES_MAJOR_VERSION}/bin.outPath") From 79fdc3e7a3f73053ba45cdd16844e00bfa26b50a Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Tue, 16 Jun 2026 13:01:35 -0400 Subject: [PATCH 11/15] gha/testinfra: Drop unused args step --- .github/workflows/testinfra-ami-build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index ee71c7951e..9c1c55fae5 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -79,11 +79,6 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - id: args - uses: mikefarah/yq@065b200af9851db0d5132f50bc10b1406ea5c0a8 # v4.50.1 - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' - - run: docker context create builders - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 From 84e88ccae30c0609261f5963fe9f28a109315802 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 17 Jun 2026 07:25:31 -0400 Subject: [PATCH 12/15] gha/testinfra: Move docker setup to after Build AMI This way testinfra and release-ami workflow/logs are more comparable. --- .github/workflows/testinfra-ami-build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index 9c1c55fae5..ce66c31a82 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -79,12 +79,6 @@ jobs: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} - - run: docker context create builders - - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - with: - endpoint: builders - - name: Build AMI id: build-ami uses: ./.github/actions/build-ami @@ -97,6 +91,12 @@ jobs: postgres_version: ${{ matrix.postgres_version }} region: ap-southeast-1 + - run: docker context create builders + + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 + with: + endpoint: builders + - name: Run tests timeout-minutes: 10 env: From 72fbfc219775ed03339598421d99c01273238501 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Tue, 16 Jun 2026 18:36:13 -0400 Subject: [PATCH 13/15] gha: Make ami-release and testinfra-ami look more similar Order things the same, add Debug AWS role secret step, use AWS_REGION env var instead of hard coding the region everywhere... etc just so we can get a better looking diff between them. --- .github/workflows/ami-release-nix.yml | 43 +++++++++++++++-------- .github/workflows/testinfra-ami-build.yml | 31 ++++++++-------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 283a7bdc5b..0cab5865d7 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -18,6 +18,9 @@ permissions: id-token: write actions: write +env: + AWS_REGION: us-east-1 + jobs: prepare: runs-on: blacksmith-4vcpu-ubuntu-2404 @@ -43,12 +46,12 @@ jobs: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} target: - - arch: arm64 - runner: blacksmith-2vcpu-ubuntu-2404-arm - instance_type: c6g.4xlarge - arch: amd64 - runner: blacksmith-2vcpu-ubuntu-2404 instance_type: c6i.4xlarge + runner: blacksmith-2vcpu-ubuntu-2404 + - arch: arm64 + instance_type: c6g.4xlarge + runner: blacksmith-2vcpu-ubuntu-2404-arm runs-on: ${{ matrix.target.runner }} timeout-minutes: 150 @@ -56,11 +59,20 @@ jobs: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - name: aws-creds + - name: Debug AWS role secret + run: | + echo "Checking DEV_AWS_ROLE secret availability..." + if [ -z "${{ secrets.DEV_AWS_ROLE }}" ]; then + echo "❌ DEV_AWS_ROLE is empty or not available" + else + echo "✅ DEV_AWS_ROLE is available" + fi + + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: ${{ env.AWS_REGION }} output-credentials: true role-duration-seconds: 7200 @@ -68,6 +80,7 @@ jobs: uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' + aws-region: ${{ env.AWS_REGION }} env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} @@ -77,13 +90,13 @@ jobs: uses: ./.github/actions/build-ami with: ami_name_prefix: "supabase-postgres-${{ github.run_id }}-${{ matrix.target.arch }}" - ami_regions: '["us-east-1"]' + ami_regions: '["${{ env.AWS_REGION }}"]' arch: ${{ matrix.target.arch }} force: "true" git_sha: ${{ github.sha }} instance_type: ${{ matrix.target.instance_type }} postgres_version: ${{ matrix.postgres_version }} - region: us-east-1 + region: ${{ env.AWS_REGION }} - name: Setup post build env vars run: | @@ -119,7 +132,7 @@ jobs: uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: ${{ env.AWS_REGION }} - name: Upload software manifest to s3 staging run: | @@ -141,7 +154,7 @@ jobs: uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ secrets.PROD_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: ${{ env.AWS_REGION }} - name: Upload software manifest to s3 prod run: | @@ -263,16 +276,16 @@ jobs: uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "us-east-1" + aws-region: ${{ env.AWS_REGION }} - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids - aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id + aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids + aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids - aws ec2 --region us-east-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region us-east-1 delete-volume --volume-id + aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids + aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${PACKER_EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id diff --git a/.github/workflows/testinfra-ami-build.yml b/.github/workflows/testinfra-ami-build.yml index ce66c31a82..142250d1ec 100644 --- a/.github/workflows/testinfra-ami-build.yml +++ b/.github/workflows/testinfra-ami-build.yml @@ -15,6 +15,9 @@ permissions: contents: write id-token: write +env: + AWS_REGION: ap-southeast-1 + jobs: prepare: runs-on: blacksmith-2vcpu-ubuntu-2404 @@ -66,7 +69,7 @@ jobs: uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1 with: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} - aws-region: "ap-southeast-1" + aws-region: ${{ env.AWS_REGION }} output-credentials: true role-duration-seconds: 7200 @@ -74,7 +77,7 @@ jobs: uses: ./.github/actions/nix-install-ephemeral with: push-to-cache: 'true' - aws-region: "ap-southeast-1" + aws-region: ${{ env.AWS_REGION }} env: DEV_AWS_ROLE: ${{ secrets.DEV_AWS_ROLE }} NIX_SIGN_SECRET_KEY: ${{ secrets.NIX_SIGN_SECRET_KEY }} @@ -84,12 +87,12 @@ jobs: uses: ./.github/actions/build-ami with: ami_name_prefix: "supabase-postgres-${{ github.run_id }}-${{ matrix.target.arch }}" - ami_regions: '["ap-southeast-1"]' + ami_regions: '["${{ env.AWS_REGION }}"]' arch: ${{ matrix.target.arch }} git_sha: ${{ github.sha }} instance_type: ${{ matrix.target.instance_type }} postgres_version: ${{ matrix.postgres_version }} - region: ap-southeast-1 + region: ${{ env.AWS_REGION }} - run: docker context create builders @@ -111,17 +114,17 @@ jobs: if: ${{ cancelled() }} run: | EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - INSTANCE_IDS=$(aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) + INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) if [ -n "$INSTANCE_IDS" ]; then echo "Terminating packer build instances: $INSTANCE_IDS" - echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids + echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids else echo "No packer build instances to clean up" fi - VOLUME_IDS=$(aws ec2 --region ap-southeast-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) + VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) if [ -n "$VOLUME_IDS" ]; then echo "Deleting orphaned packer volumes: $VOLUME_IDS" - echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region ap-southeast-1 delete-volume --volume-id + echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id else echo "No orphaned packer volumes to clean up" fi @@ -130,17 +133,17 @@ jobs: if: ${{ always() }} run: | EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" - INSTANCE_IDS=$(aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) + INSTANCE_IDS=$(aws ec2 --region "$AWS_REGION" describe-instances --filters "Name=tag:testinfra-run-id,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text) if [ -n "$INSTANCE_IDS" ]; then echo "Terminating testinfra instances: $INSTANCE_IDS" - echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids || true + echo "$INSTANCE_IDS" | xargs -r aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids || true else echo "No testinfra instances to clean up" fi - VOLUME_IDS=$(aws ec2 --region ap-southeast-1 describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) + VOLUME_IDS=$(aws ec2 --region "$AWS_REGION" describe-volumes --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" "Name=status,Values=available" --query "Volumes[*].VolumeId" --output text) if [ -n "$VOLUME_IDS" ]; then echo "Deleting orphaned packer volumes: $VOLUME_IDS" - echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region ap-southeast-1 delete-volume --volume-id || true + echo "$VOLUME_IDS" | xargs -r -n1 aws ec2 --region "$AWS_REGION" delete-volume --volume-id || true else echo "No orphaned packer volumes to clean up" fi @@ -150,7 +153,7 @@ jobs: run: | EXECUTION_ID="${{ steps.build-ami.outputs.execution_id }}" STAGE2_AMI_IDS=$(aws ec2 describe-images \ - --region ap-southeast-1 \ + --region "$AWS_REGION" \ --owners self \ --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" \ --query 'Images[*].ImageId' \ @@ -159,7 +162,7 @@ jobs: if [ -n "$STAGE2_AMI_IDS" ]; then for ami_id in $STAGE2_AMI_IDS; do echo "Deregistering stage 2 AMI: $ami_id" - aws ec2 deregister-image --region ap-southeast-1 --image-id "$ami_id" || true + aws ec2 deregister-image --region "$AWS_REGION" --image-id "$ami_id" || true done else echo "No stage 2 AMI to clean up" From cb991dfb6792a84840c23d3ffb4dfed9783b0c0d Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 17 Jun 2026 20:24:51 -0400 Subject: [PATCH 14/15] packer/amd64: Undo ssh_pty = false This wasn't really necessary so lets revert. --- amazon-amd64-nix.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index f3d2f46230..bc08d656bd 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -169,7 +169,7 @@ source "amazon-ebssurrogate" "source" { } communicator = "ssh" - ssh_pty = false + ssh_pty = true ssh_username = "ubuntu" ssh_timeout = "5m" From 890a40a0425b551802bf6df5c5c5d29690ed0664 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Fri, 19 Jun 2026 14:34:51 -0400 Subject: [PATCH 15/15] packer: Pin amazon plugin to 1.8.0 This is causing the plugin's cpu usage to go from ~.6% -> ~600%! See https://github.com/hashicorp/packer-plugin-amazon/issues/676 --- amazon-amd64-nix.pkr.hcl | 5 ++++- amazon-arm64-nix.pkr.hcl | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/amazon-amd64-nix.pkr.hcl b/amazon-amd64-nix.pkr.hcl index bc08d656bd..4543955663 100644 --- a/amazon-amd64-nix.pkr.hcl +++ b/amazon-amd64-nix.pkr.hcl @@ -87,7 +87,10 @@ packer { required_plugins { amazon = { source = "github.com/hashicorp/amazon" - version = "~> 1" + # don't use semver for the version since there's no lock files + # can go back when we can have renovate watching this + # see https://github.com/hashicorp/packer-plugin-amazon/issues/676 + version = "1.8.0" } } } diff --git a/amazon-arm64-nix.pkr.hcl b/amazon-arm64-nix.pkr.hcl index ac64fe3503..a5faec3a93 100644 --- a/amazon-arm64-nix.pkr.hcl +++ b/amazon-arm64-nix.pkr.hcl @@ -87,7 +87,10 @@ packer { required_plugins { amazon = { source = "github.com/hashicorp/amazon" - version = "~> 1" + # don't use semver for the version since there's no lock files + # can go back when we can have renovate watching this + # see https://github.com/hashicorp/packer-plugin-amazon/issues/676 + version = "1.8.0" } } }