feat(cli): port db diff and db pull to native TypeScript (CLI-1313)

apps/cli-e2e/src/tests/database-core.e2e.test.ts

@@ -279,7 +279,24 @@ describe("db pull", () => {
     expect(result.stderr).toContain("connect");
   });
 
-  testParity(["db", "pull", "--local"]);
+  // No testParity for `db pull --local`: like `db lint --local` and `test db --local`,
+  // pull connects via the shared utils.ConnectByConfig → pgxv5.Connect path on Go and
+  // the same LegacyDbConnection sql-pg layer on TS. With no local Postgres listening in
+  // the harness, the only reachable path is the connection-failure path, and its stderr
+  // diverges by driver in ways that aren't cosmetic and can't be normalized away.
+  // Both emit Go's leading diagnostic to stderr:
+  //   Connecting to local database...
+  // but the connect-error body and trailing hint still differ by driver. Go (pgx):
+  //   failed to connect to postgres: failed to connect to `host=… user=… database=…`: dial error (dial tcp …: connect: connection refused)
+  //   Make sure your local IP is allowed in Network Restrictions and Network Bans.
+  //   http://…/project/_/database/settings
+  // The TS port (@effect/sql-pg) prints the effect SqlError and the --debug hint:
+  //   failed to connect to postgres: effect/sql/SqlError: PgClient: Failed to connect
+  //   Try rerunning the command with --debug to troubleshoot the error.
+  // The meaningful contract (non-zero exit + a connect error on stderr) is covered by
+  // the behaviour test above. A real connect-path parity test would need a live local
+  // database in the harness. (db dump --local keeps its testParity because it connects
+  // through the pg_dump Docker container, so its stderr matches on both runtimes.)
 });
 
 // ---------------------------------------------------------------------------

apps/cli-go/cmd/db.go

@@ -197,6 +197,76 @@ var (
 		},
 	}
 
+	shadowMode        string
+	shadowTargetLocal bool
+	shadowUsePgDelta  bool
+	shadowSchema      []string
+	shadowProjectRef  string
+
+	// dbShadowCmd is a hidden seam used by the native-TypeScript db diff/pull
+	// commands to provision the throwaway shadow database that the diff "source"
+	// runs against, then leave it running so the TS caller can run the differ
+	// (migra or pg-delta) itself and remove the container afterwards. It prints
+	// three newline-separated lines to stdout: the container id, the source
+	// Postgres URL, and an optional target-override URL (empty unless the
+	// local-target declarative branch redirects the diff target to a second
+	// shadow database). The URLs are emitted WITHOUT the password
+	// (ToPostgresURLWithoutPassword) so we never log a credential to stdout
+	// (CWE-312); the TS caller re-injects the local Postgres password it already
+	// resolves from config.toml, which is the same value the shadow uses. Shadow
+	// provisioning (start.SetupDatabase) is not yet ported, which is why this
+	// stays in Go.
+	dbShadowCmd = &cobra.Command{
+		Use:    "__shadow",
+		Hidden: true,
+		Short:  "Internal: provision a shadow database for the native db diff/pull commands",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// The hidden __shadow command carries none of the db-url/local/linked
+			// target flags, so the root PersistentPreRunE's ParseDatabaseConfig
+			// never loads supabase/config.toml (it only loads when a target flag
+			// is set, internal/utils/flags/db_url.go:46-90). Load it explicitly so
+			// the shadow is provisioned from the project's [db] settings — shadow
+			// port, Postgres version, service baseline, and especially the
+			// password: the native-TS caller injects the config.toml password into
+			// the seam URLs, so the shadow must be created with that same password.
+			fsys := afero.NewOsFs()
+			// On the linked path the native-TS caller passes the resolved project
+			// ref via --project-ref so the shadow is built from the same
+			// remote-merged config the Go monolith uses: LoadConfig seeds
+			// utils.Config.ProjectId from flags.ProjectRef and merges the matching
+			// [remotes.<ref>] block (pkg/config/config.go). Omitted on local/db-url
+			// shadows, which the monolith never remote-merges, so the base config is
+			// used exactly as before.
+			if len(shadowProjectRef) > 0 {
+				flags.ProjectRef = shadowProjectRef
+			}
+			if err := flags.LoadConfig(fsys); err != nil {
+				return err
+			}
+			var src diff.ShadowSource
+			var err error
+			switch shadowMode {
+			case "declarative":
+				src, err = diff.PrepareRawShadow(cmd.Context())
+			case "diff", "":
+				src, err = diff.PrepareShadowSource(cmd.Context(), shadowSchema, shadowTargetLocal, shadowUsePgDelta, fsys)
+			default:
+				return fmt.Errorf("unknown shadow mode: %s", shadowMode)
+			}
+			if err != nil {
+				return err
+			}
+			fmt.Println(src.Container)
+			fmt.Println(utils.ToPostgresURLWithoutPassword(src.Source))
+			if src.TargetOverride != nil {
+				fmt.Println(utils.ToPostgresURLWithoutPassword(*src.TargetOverride))
+			} else {
+				fmt.Println("")
+			}
+			return nil
+		},
+	}
+
 	dbRemoteCmd = &cobra.Command{
 		Hidden: true,
 		Use:    "remote",
@@ -475,6 +545,14 @@ func init() {
 	pullFlags.StringVarP(&dbPassword, "password", "p", "", "Password to your remote Postgres database.")
 	cobra.CheckErr(viper.BindPFlag("DB_PASSWORD", pullFlags.Lookup("password")))
 	dbCmd.AddCommand(dbPullCmd)
+	// Build hidden shadow-provisioning seam command
+	shadowFlags := dbShadowCmd.Flags()
+	shadowFlags.StringVar(&shadowMode, "mode", "diff", "Shadow mode: diff (baseline + migrations) or declarative (bare shadow).")
+	shadowFlags.BoolVar(&shadowTargetLocal, "target-local", false, "Whether the diff target is the local database (enables the declarative-schema branch).")
+	shadowFlags.BoolVar(&shadowUsePgDelta, "use-pg-delta", false, "Whether pg-delta is the active diff engine (selects the declarative-apply path).")
+	shadowFlags.StringSliceVarP(&shadowSchema, "schema", "s", []string{}, "Comma separated list of schema to include.")
+	shadowFlags.StringVar(&shadowProjectRef, "project-ref", "", "Linked project ref, so the shadow merges the matching [remotes.<ref>] config override.")
+	dbCmd.AddCommand(dbShadowCmd)
 	// Build remote command
 	remoteFlags := dbRemoteCmd.PersistentFlags()
 	remoteFlags.StringSliceVarP(&schema, "schema", "s", []string{}, "Comma separated list of schema to include.")

apps/cli-go/cmd/db_schema_declarative.go

@@ -47,6 +47,14 @@ var (
 		Use:   "declarative",
 		Short: "Manage declarative database schemas",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			// The hidden __catalog seam forwards the resolved linked ref via
+			// --project-ref so the catalog is built from the remote-merged config.
+			// Seed flags.ProjectRef before LoadConfig (which keys the [remotes.<ref>]
+			// merge off Config.ProjectId = flags.ProjectRef); this command never runs
+			// LoadProjectRef, so SUPABASE_PROJECT_ID env alone would not merge.
+			if len(pgdeltaCatalogProjectRef) > 0 {
+				flags.ProjectRef = pgdeltaCatalogProjectRef
+			}
 			if err := flags.LoadConfig(afero.NewOsFs()); err != nil {
 				return err
 			}

apps/cli-go/cmd/pgdelta_catalog.go

@@ -11,6 +11,13 @@ import (
 // pgdeltaCatalogMode selects which catalog the hidden seam command produces.
 var pgdeltaCatalogMode string
 
+// pgdeltaCatalogProjectRef is the resolved linked project ref, forwarded by the
+// native-TypeScript seam so the catalog is built from the remote-merged config.
+// The declarative group's PersistentPreRunE seeds flags.ProjectRef from it before
+// LoadConfig (this command never runs LoadProjectRef, so SUPABASE_PROJECT_ID env
+// alone would not trigger the [remotes.<ref>] merge).
+var pgdeltaCatalogProjectRef string
+
 // dbDeclarativeCatalogCmd is a hidden seam used by the native-TypeScript
 // declarative commands to provision a shadow-database platform baseline (and,
 // for migrations/declarative modes, apply migrations / declarative files) and
@@ -34,5 +41,6 @@ var dbDeclarativeCatalogCmd = &cobra.Command{
 
 func init() {
 	dbDeclarativeCatalogCmd.Flags().StringVar(&pgdeltaCatalogMode, "mode", "", "Catalog mode: baseline, migrations, or declarative.")
+	dbDeclarativeCatalogCmd.Flags().StringVar(&pgdeltaCatalogProjectRef, "project-ref", "", "Linked project ref, so the catalog merges the matching [remotes.<ref>] config override.")
 	dbDeclarativeCmd.AddCommand(dbDeclarativeCatalogCmd)
 }

apps/cli-go/internal/db/diff/diff.go

@@ -22,7 +22,6 @@ import (
 	"github.com/jackc/pgx/v4"
 	"github.com/spf13/afero"
 	"github.com/supabase/cli/internal/db/start"
-	"github.com/supabase/cli/internal/pgdelta"
 	"github.com/supabase/cli/internal/utils"
 	"github.com/supabase/cli/pkg/migration"
 	"github.com/supabase/cli/pkg/parser"
@@ -188,47 +187,14 @@ func MigrateShadowDatabase(ctx context.Context, container string, fsys afero.Fs,
 
 func DiffDatabase(ctx context.Context, schema []string, config pgconn.Config, w io.Writer, fsys afero.Fs, differ DiffFunc, usePgDelta bool, options ...func(*pgx.ConnConfig)) (DatabaseDiff, error) {
 	fmt.Fprintln(w, "Creating shadow database...")
-	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
+	shadowSource, err := PrepareShadowSource(ctx, schema, utils.IsLocalDatabase(config), usePgDelta, fsys, options...)
 	if err != nil {
 		return DatabaseDiff{}, err
 	}
-	defer utils.DockerRemove(shadow)
-	if err := start.WaitForHealthyService(ctx, utils.Config.Db.HealthTimeout, shadow); err != nil {
-		return DatabaseDiff{}, err
-	}
-	if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {
-		return DatabaseDiff{}, err
-	}
-	shadowConfig := pgconn.Config{
-		Host:     utils.Config.Hostname,
-		Port:     utils.Config.Db.ShadowPort,
-		User:     "postgres",
-		Password: utils.Config.Db.Password,
-		Database: "postgres",
-	}
-	if utils.IsLocalDatabase(config) {
-		if declared, err := loadDeclaredSchemas(fsys); len(declared) > 0 {
-			config = shadowConfig
-			config.Database = "contrib_regression"
-			if usePgDelta {
-				declDir := utils.GetDeclarativeDir()
-				if exists, _ := afero.DirExists(fsys, declDir); exists {
-					if err := pgdelta.ApplyDeclarative(ctx, config, fsys); err != nil {
-						return DatabaseDiff{}, err
-					}
-				} else {
-					if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil {
-						return DatabaseDiff{}, err
-					}
-				}
-			} else {
-				if err := migrateBaseDatabase(ctx, config, declared, fsys, options...); err != nil {
-					return DatabaseDiff{}, err
-				}
-			}
-		} else if err != nil {
-			return DatabaseDiff{}, err
-		}
+	defer utils.DockerRemove(shadowSource.Container)
+	shadowConfig := shadowSource.Source
+	if shadowSource.TargetOverride != nil {
+		config = *shadowSource.TargetOverride
 	}
 	// Load all user defined schemas
 	if len(schema) > 0 {

apps/cli-go/internal/db/diff/shadow.go

@@ -0,0 +1,116 @@
+package diff
+
+import (
+	"context"
+
+	"github.com/jackc/pgconn"
+	"github.com/jackc/pgx/v4"
+	"github.com/spf13/afero"
+	"github.com/supabase/cli/internal/db/start"
+	"github.com/supabase/cli/internal/pgdelta"
+	"github.com/supabase/cli/internal/utils"
+)
+
+// ShadowSource is a provisioned shadow database, left running for an external
+// caller (the native-TypeScript db diff/pull commands) to diff against and then
+// remove. It mirrors the shadow that DiffDatabase prepares as the diff "source".
+type ShadowSource struct {
+	// Container is the shadow database container id; the caller MUST remove it
+	// (e.g. `docker rm -f <id>`) when the diff completes.
+	Container string
+	// Source is the connection config for the diff source (the shadow with the
+	// platform baseline + local migrations applied).
+	Source pgconn.Config
+	// TargetOverride, when non-nil, replaces the diff target with a second shadow
+	// database (contrib_regression with declarative schemas applied). Mirrors
+	// DiffDatabase's local-target declarative branch, where the user's local
+	// database is not diffed at all.
+	TargetOverride *pgconn.Config
+}
+
+// PrepareShadowSource provisions the shadow database that DiffDatabase diffs
+// against, but returns it running instead of diffing + removing, so a native
+// caller can run the differ itself. targetLocal mirrors
+// utils.IsLocalDatabase(config) — the only target-derived input the shadow prep
+// needs. usePgDelta selects the declarative-apply engine for the local-declared
+// branch, matching DiffDatabase. On error the shadow container is removed.
+func PrepareShadowSource(ctx context.Context, schema []string, targetLocal bool, usePgDelta bool, fsys afero.Fs, options ...func(*pgx.ConnConfig)) (ShadowSource, error) {
+	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
+	if err != nil {
+		return ShadowSource{}, err
+	}
+	ok := false
+	defer func() {
+		if !ok {
+			utils.DockerRemove(shadow)
+		}
+	}()
+	if err := start.WaitForHealthyService(ctx, utils.Config.Db.HealthTimeout, shadow); err != nil {
+		return ShadowSource{}, err
+	}
+	if err := MigrateShadowDatabase(ctx, shadow, fsys, options...); err != nil {
+		return ShadowSource{}, err
+	}
+	shadowConfig := pgconn.Config{
+		Host:     utils.Config.Hostname,
+		Port:     utils.Config.Db.ShadowPort,
+		User:     "postgres",
+		Password: utils.Config.Db.Password,
+		Database: "postgres",
+	}
+	var targetOverride *pgconn.Config
+	if targetLocal {
+		declared, err := loadDeclaredSchemas(fsys)
+		if err != nil {
+			return ShadowSource{}, err
+		}
+		if len(declared) > 0 {
+			override := shadowConfig
+			override.Database = "contrib_regression"
+			if usePgDelta {
+				declDir := utils.GetDeclarativeDir()
+				if exists, _ := afero.DirExists(fsys, declDir); exists {
+					if err := pgdelta.ApplyDeclarative(ctx, override, fsys); err != nil {
+						return ShadowSource{}, err
+					}
+				} else {
+					if err := migrateBaseDatabase(ctx, override, declared, fsys, options...); err != nil {
+						return ShadowSource{}, err
+					}
+				}
+			} else {
+				if err := migrateBaseDatabase(ctx, override, declared, fsys, options...); err != nil {
+					return ShadowSource{}, err
+				}
+			}
+			targetOverride = &override
+		}
+	}
+	ok = true
+	return ShadowSource{Container: shadow, Source: shadowConfig, TargetOverride: targetOverride}, nil
+}
+
+// PrepareRawShadow provisions a bare shadow database (created + healthy, with no
+// platform baseline or migrations applied), left running for an external caller.
+// Mirrors the shadow that pull.pullDeclarativePgDelta uses as the empty
+// declarative-export source. On error the shadow container is removed.
+func PrepareRawShadow(ctx context.Context) (ShadowSource, error) {
+	shadow, err := CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
+	if err != nil {
+		return ShadowSource{}, err
+	}
+	if err := start.WaitForHealthyService(ctx, utils.Config.Db.HealthTimeout, shadow); err != nil {
+		utils.DockerRemove(shadow)
+		return ShadowSource{}, err
+	}
+	return ShadowSource{
+		Container: shadow,
+		Source: pgconn.Config{
+			Host:     utils.Config.Hostname,
+			Port:     utils.Config.Db.ShadowPort,
+			User:     "postgres",
+			Password: utils.Config.Db.Password,
+			Database: "postgres",
+		},
+	}, nil
+}

apps/cli-go/internal/db/pull/pull.go

@@ -20,7 +20,6 @@ import (
 	"github.com/supabase/cli/internal/db/declarative"
 	"github.com/supabase/cli/internal/db/diff"
 	"github.com/supabase/cli/internal/db/dump"
-	"github.com/supabase/cli/internal/db/start"
 	"github.com/supabase/cli/internal/migration/format"
 	"github.com/supabase/cli/internal/migration/list"
 	"github.com/supabase/cli/internal/migration/new"
@@ -86,21 +85,12 @@ func Run(ctx context.Context, schema []string, config pgconn.Config, name string
 // timestamped migration files.
 func pullDeclarativePgDelta(ctx context.Context, schema []string, config pgconn.Config, fsys afero.Fs, options ...func(*pgx.ConnConfig)) error {
 	fmt.Fprintln(os.Stderr, "Preparing declarative schema export using pg-delta...")
-	shadow, err := diff.CreateShadowDatabase(ctx, utils.Config.Db.ShadowPort)
+	shadowSource, err := diff.PrepareRawShadow(ctx)
 	if err != nil {
 		return err
 	}
-	defer utils.DockerRemove(shadow)
-	if err := start.WaitForHealthyService(ctx, utils.Config.Db.HealthTimeout, shadow); err != nil {
-		return err
-	}
-	shadowConfig := pgconn.Config{
-		Host:     utils.Config.Hostname,
-		Port:     utils.Config.Db.ShadowPort,
-		User:     "postgres",
-		Password: utils.Config.Db.Password,
-		Database: "postgres",
-	}
+	defer utils.DockerRemove(shadowSource.Container)
+	shadowConfig := shadowSource.Source
 	formatOptions := ""
 	if utils.Config.Experimental.PgDelta != nil {
 		formatOptions = strings.TrimSpace(utils.Config.Experimental.PgDelta.FormatOptions)

apps/cli-go/internal/utils/connect.go

@@ -23,6 +23,21 @@ import (
 )
 
 func ToPostgresURL(config pgconn.Config) string {
+	return toPostgresURL(config, url.UserPassword(config.User, config.Password))
+}
+
+// ToPostgresURLWithoutPassword renders the connection URL exactly like
+// ToPostgresURL but omits the password from the userinfo. Use it for callers that
+// print the URL to stdout (the hidden `db __shadow` seam): embedding the password
+// there is clear-text logging of a credential (CWE-312, flagged by CodeQL). The
+// password is never the seam's to share — the TS caller that consumes the seam
+// output re-injects the local Postgres password it already resolves from
+// config.toml (`utils.Config.Db.Password`).
+func ToPostgresURLWithoutPassword(config pgconn.Config) string {
+	return toPostgresURL(config, url.User(config.User))
+}
+
+func toPostgresURL(config pgconn.Config, userinfo *url.Userinfo) string {
 	timeoutSecond := int64(config.ConnectTimeout.Seconds())
 	if timeoutSecond == 0 {
 		timeoutSecond = 10
@@ -38,7 +53,7 @@ func ToPostgresURL(config pgconn.Config) string {
 	}
 	return fmt.Sprintf(
 		"postgresql://%s@%s:%d/%s?%s",
-		url.UserPassword(config.User, config.Password),
+		userinfo,
 		host,
 		config.Port,
 		url.PathEscape(config.Database),