Pin the tunnel's SSH host key so a reused connection name reconnects

.nextchanges/cli/ssh-connect-pins-host-key.md

@@ -0,0 +1 @@
+* `databricks ssh connect` and `ssh setup` now verify the tunnel's SSH host key against the key the workspace published for the connection, recorded in `~/.databricks/ssh-tunnel-known-hosts/<name>` instead of `~/.ssh/known_hosts`. Reconnecting with a name used before no longer fails with `Host key verification failed` when the compute behind that name changed, and no longer needs a manual `ssh-keygen -R`; host blocks written by an earlier `databricks ssh setup` pick this up once you re-run it. ([#6557](https://github.com/databricks/cli/pull/6557))

acceptance/ssh/connect-serverless-cpu/script

@@ -22,6 +22,6 @@ else
   CLI_RELEASES_DIR="$PWD/releases"
 
   title "CLI ssh connect (local sshd)\n"
-  errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR --user-known-hosts-file=known_hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
+  errcode $CLI ssh connect --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
   trace print_requests.py //api/2.2/jobs/runs/submit
 fi

acceptance/ssh/connect-serverless-cpu/test.toml

@@ -5,9 +5,11 @@ Cloud = false
 # Record requests so we can assert the bootstrap job the CLI submits.
 RecordRequests = true
 
-# Stand-in release archives staged by the local run; not golden output.
+# Stand-in release archives staged by the local run, plus the host key the CLI pins under
+# known-hosts; not golden output.
 Ignore = [
   "releases",
+  "known-hosts",
 ]
 
 # The local run drives a real sshd, reliable only on Linux.

acceptance/ssh/connect-serverless-gpu/script

@@ -24,6 +24,6 @@ else
     echo "SKIP_TEST sshd (openssh-server) not installed"
     exit 0
   fi
-  errcode $CLI ssh connect --name serverless-gpu-test --accelerator=GPU_1xA10 --releases-dir=$CLI_RELEASES_DIR --user-known-hosts-file=known_hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
+  errcode $CLI ssh connect --name serverless-gpu-test --accelerator=GPU_1xA10 --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
   trace print_requests.py //api/2.2/jobs/runs/submit
 fi

acceptance/ssh/connect-serverless-gpu/test.toml

@@ -5,9 +5,11 @@ Cloud = false
 # Assert the serverless GPU bootstrap job the CLI submits on a local run.
 RecordRequests = true
 
-# Local runs stage stand-in release archives here; not part of the golden output.
+# Local runs stage stand-in release archives here, and the CLI pins the server's host key
+# under known-hosts; neither is part of the golden output.
 Ignore = [
   "releases",
+  "known-hosts",
 ]
 
 # Linux-only: the local run drives a real sshd, reliable only there (and where

acceptance/ssh/connection/script

@@ -10,7 +10,7 @@ fi
 if [ -n "${CLOUD_ENV:-}" ]; then
   # On cloud the dedicated cluster runs the remote command over a full SSH
   # handshake; dump the run on failure.
-  errcode $CLI ssh connect --cluster=$TEST_DEFAULT_CLUSTER_ID --releases-dir=$CLI_RELEASES_DIR --user-known-hosts-file=known_hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
+  errcode $CLI ssh connect --cluster=$TEST_DEFAULT_CLUSTER_ID --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
   if ! grep -q "Connection successful" out.stdout.txt; then
     run_id=$(cat LOG.stderr | grep -o "Job submitted successfully with run ID: [0-9]*" | grep -o "[0-9]*$")
     trace $CLI jobs get-run "$run_id" > LOG.job
@@ -24,6 +24,6 @@ else
     echo "SKIP_TEST sshd (openssh-server) not installed"
     exit 0
   fi
-  errcode $CLI ssh connect --cluster=$TEST_DEFAULT_CLUSTER_ID --releases-dir=$CLI_RELEASES_DIR --user-known-hosts-file=known_hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
+  errcode $CLI ssh connect --cluster=$TEST_DEFAULT_CLUSTER_ID --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.stdout.txt 2>LOG.stderr
   trace print_requests.py //api/2.2/jobs/runs/submit
 fi

acceptance/ssh/connection/test.toml

@@ -6,9 +6,11 @@ Cloud = false
 # Assert the dedicated-cluster bootstrap job the CLI submits on a local run.
 RecordRequests = true
 
-# Local runs stage stand-in release archives here; not part of the golden output.
+# Local runs stage stand-in release archives here, and the CLI pins the server's host key
+# under known-hosts; neither is part of the golden output.
 Ignore = [
   "releases",
+  "known-hosts",
 ]
 
 # Linux-only: the local run drives a real sshd, reliable only there (and where

acceptance/ssh/reconnect-reused-name/out.connect-1.txt

@@ -0,0 +1 @@
+Connection successful

acceptance/ssh/reconnect-reused-name/out.connect-2.txt

@@ -0,0 +1 @@
+Connection successful

acceptance/ssh/reconnect-reused-name/output.txt

@@ -0,0 +1,10 @@
+
+=== CLI ssh connect over a stale host key for the connection name
+
+=== The stale key is replaced by the one the workspace published
+stale key replaced
+
+>>> grep -c ^reused-name ssh- known-hosts/reused-name
+1
+
+=== Reconnecting to the running server keeps working

acceptance/ssh/reconnect-reused-name/script

@@ -0,0 +1,42 @@
+# Reconnecting with a name used before must not fail on the host key
+# recorded for that name earlier. The tunnel's host key lives in the workspace, and the
+# CLI pins it for the connection name, so a key recorded by an earlier instance (or by
+# the same name in another workspace) is replaced rather than tripping strict checking.
+#
+# The test server backs the /ssh websocket with a real sshd whose host key comes from the
+# session's secret scope, so this asserts a full handshake against a pinned key. Skip when
+# sshd is absent (only task test-exp-ssh provisions it).
+if [ -z "$(command -v sshd || ls /usr/sbin/sshd /usr/local/sbin/sshd /sbin/sshd 2>/dev/null)" ]; then
+  echo "SKIP_TEST sshd (openssh-server) not installed"
+  exit 0
+fi
+
+# No release artifacts locally and they're never executed; stand in empty archives.
+mkdir -p releases
+: >releases/databricks_cli_linux_amd64.zip
+: >releases/databricks_cli_linux_arm64.zip
+CLI_RELEASES_DIR="$PWD/releases"
+
+# The state a user reconnecting with a reused name is in: an entry for this connection
+# name, pinned days earlier, whose key no longer belongs to anything. The CLI names the file
+# after the connection, so --known-hosts-dir is enough to keep the pin inside the test.
+STALE_KEY=AAAAC3NzaC1lZDI1NTE5AAAAICTzJweKcgiNoBUAyuvCY2Qu1Od8mKBON5aJeA03Nl+D
+mkdir -p known-hosts
+echo "reused-name ssh-ed25519 $STALE_KEY" >known-hosts/reused-name
+
+# The remote command's output is the assertion (see out.connect-*.txt); the tunnel's own
+# progress and teardown messages go to stderr, where the order of the last few lines
+# depends on which side closes the websocket first.
+title "CLI ssh connect over a stale host key for the connection name\n"
+errcode $CLI ssh connect --name reused-name --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.connect-1.txt 2>LOG.connect-1
+
+title "The stale key is replaced by the one the workspace published\n"
+if grep -q "$STALE_KEY" known-hosts/reused-name; then
+  echo "stale key still pinned"
+else
+  echo "stale key replaced"
+fi
+trace grep -c "^reused-name ssh-" known-hosts/reused-name
+
+title "Reconnecting to the running server keeps working\n"
+errcode $CLI ssh connect --name reused-name --releases-dir=$CLI_RELEASES_DIR --known-hosts-dir=known-hosts -- "echo 'Connection successful'" >out.connect-2.txt 2>LOG.connect-2

acceptance/ssh/reconnect-reused-name/test.toml

@@ -0,0 +1,18 @@
+Cloud = false
+
+# The connection name is explicit, so nothing here depends on the local host name.
+
+# Local runs stage stand-in release archives here; not part of the golden output.
+Ignore = [
+  "releases",
+  "known-hosts",
+]
+
+# Linux-only: the local run drives a real sshd, reliable only there (and where
+# task test-exp-ssh provisions it). Absent GOOS keys default to enabled, so the
+# other OSes must be disabled explicitly.
+GOOS.linux = true
+GOOS.darwin = false
+GOOS.windows = false
+
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

experimental/ssh/README.md

@@ -17,6 +17,20 @@ B. Spawn an ssh session directly:
 databricks ssh connect --cluster=id
 ```
 
+### Connection names and host keys
+
+`--name` is a stable handle for serverless compute, not a live session id: connecting with a
+name reuses the SSH server still running under it, and starts a new one on the same name once
+the previous server has shut down (`--shutdown-delay` after its last client disconnects).
+
+Both cases verify the server's host key. The server generates the key on first use, keeps it in
+the connection's secret scope in the workspace and reuses it for every `sshd` it launches, so
+the workspace is the authority on the key. The client reads it from there and pins it in
+`~/.databricks/ssh-tunnel-known-hosts/<name>` before each connection, and points ssh at that
+file with `StrictHostKeyChecking yes`. Tunnel host keys therefore never land in
+`~/.ssh/known_hosts`, where a name - unique only within one workspace - would collide with an
+entry left by other compute.
+
 ## Development
 ```shell
 ./task build snapshot-release

experimental/ssh/cmd/connect.go

@@ -53,7 +53,7 @@ Connect to a dedicated cluster:
 	var handoverTimeout time.Duration
 	var releasesDir string
 	var autoStartCluster bool
-	var userKnownHostsFile string
+	var knownHostsDir string
 	var liteswap string
 	var skipSettingsCheck bool
 	var environmentVersion int
@@ -82,8 +82,8 @@ Connect to a dedicated cluster:
 	cmd.Flags().StringVar(&releasesDir, "releases-dir", "", "Directory for local SSH tunnel development releases")
 	cmd.Flags().MarkHidden("releases-dir")
 
-	cmd.Flags().StringVar(&userKnownHostsFile, "user-known-hosts-file", "", "Path to user known hosts file for SSH client")
-	cmd.Flags().MarkHidden("user-known-hosts-file")
+	cmd.Flags().StringVar(&knownHostsDir, "known-hosts-dir", "", "Directory for the known hosts files the CLI maintains for tunnel connections")
+	cmd.Flags().MarkHidden("known-hosts-dir")
 
 	cmd.Flags().StringVar(&liteswap, "liteswap", "", "Liteswap header value for traffic routing (dev/test only)")
 	cmd.Flags().MarkHidden("liteswap")
@@ -145,7 +145,8 @@ Connect to a dedicated cluster:
 			AutoStartCluster:     autoStartCluster,
 			ClientPublicKeyName:  clientPublicKeyName,
 			ClientPrivateKeyName: clientPrivateKeyName,
-			UserKnownHostsFile:   userKnownHostsFile,
+			ServerPublicKeyName:  serverPublicKeyName,
+			KnownHostsDir:        knownHostsDir,
 			Liteswap:             liteswap,
 			SkipSettingsCheck:    skipSettingsCheck,
 			EnvironmentVersion:   environmentVersion,