diff --git a/.nextchanges/cli/setup-local-provision-conflict.md b/.nextchanges/cli/setup-local-provision-conflict.md new file mode 100644 index 0000000000..ab4a8b564c --- /dev/null +++ b/.nextchanges/cli/setup-local-provision-conflict.md @@ -0,0 +1 @@ +* `databricks environments setup-local` now reports a distinct `E_PROVISION_CONFLICT` error code in `--output json` when the project's dependencies conflict with the pins written for the target environment, making the requirements unsatisfiable (the same conflict surfaced as a `W_USER_CONSTRAINT_CONFLICT` warning); it is reported after the project files are written, without attempting the doomed provisioning, while other provisioning failures continue to report `E_PROVISION`. ([#6479](https://github.com/databricks/cli/pull/6479)) diff --git a/acceptance/localenv/provision-conflict/out.test.toml b/acceptance/localenv/provision-conflict/out.test.toml new file mode 100644 index 0000000000..0938e67898 --- /dev/null +++ b/acceptance/localenv/provision-conflict/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/localenv/provision-conflict/output.txt b/acceptance/localenv/provision-conflict/output.txt new file mode 100644 index 0000000000..6e640dde67 --- /dev/null +++ b/acceptance/localenv/provision-conflict/output.txt @@ -0,0 +1,62 @@ +{ + "schemaVersion": 1, + "command": "environments setup-local", + "ok": false, + "mode": "default", + "dryRun": false, + "compute": { + "source": "serverless", + "serverlessVersion": "v4", + "envKey": "serverless/serverless-v4" + }, + "resolved": { + "pythonVersion": "3.12", + "dbconnectVersion": "17.2.0", + "artifactSource": "network" + }, + "greenfield": false, + "phases": [ + { + "phase": "preflight", + "status": "ok" + }, + { + "phase": "resolve", + "status": "ok" + }, + { + "phase": "fetch", + "status": "ok" + }, + { + "phase": "merge", + "status": "ok" + }, + { + "phase": "provision", + "status": "error" + }, + { + "phase": "validate", + "status": "pending" + } + ], + "warnings": [ + { + "code": "W_DBCONNECT_PIN_OVERRIDDEN", + "message": "databricks-connect \"databricks-connect~=16.0.0\" is replaced by the environment's \"databricks-connect~=17.2.0\"" + }, + { + "code": "W_USER_CONSTRAINT_CONFLICT", + "message": "dependency \"pip==24.0\" conflicts with the environment constraint \"pip\u003c24\"" + } + ], + "error": { + "code": "E_PROVISION_CONFLICT", + "failurePhase": "provision", + "message": "dependency pins conflict with the environment constraints; relax the conflicting pins and re-run (see warnings)", + "diskMutated": true + }, + "backupPath": "[TEST_TMP_DIR]/pyproject.toml.bak", + "durationMs": [DURATION_MS] +} diff --git a/acceptance/localenv/provision-conflict/pyproject.toml b/acceptance/localenv/provision-conflict/pyproject.toml new file mode 100644 index 0000000000..269a5c16a1 --- /dev/null +++ b/acceptance/localenv/provision-conflict/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "demo" +requires-python = ">=3.12" +dependencies = ["pip==24.0"] + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] diff --git a/acceptance/localenv/provision-conflict/script b/acceptance/localenv/provision-conflict/script new file mode 100644 index 0000000000..5f192a56d2 --- /dev/null +++ b/acceptance/localenv/provision-conflict/script @@ -0,0 +1,6 @@ +# The user pins pip==24.0 while the remote environment constraints pin pip<24, a +# provably disjoint range. The merge phase detects that (W_USER_CONSTRAINT_CONFLICT) +# and setup-local reports E_PROVISION_CONFLICT right away — with the merged pins +# already written to disk (diskMutated=true), the contract the extension's recovery +# flow depends on — instead of spending a doomed Python install and uv sync. +musterr $CLI environments setup-local --serverless-version 4 --output json diff --git a/acceptance/localenv/provision-conflict/test.toml b/acceptance/localenv/provision-conflict/test.toml new file mode 100644 index 0000000000..f1d43d4602 --- /dev/null +++ b/acceptance/localenv/provision-conflict/test.toml @@ -0,0 +1,22 @@ +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# setup-local writes pyproject.toml through the merge phase (backing up the +# committed one) before it reports the conflict; the assertion is the JSON output, +# so ignore the mutated file and its backup. No .venv is created because the run +# stops before provisioning. +Ignore = ["pyproject.toml", "pyproject.toml.bak"] + +Env.DATABRICKS_LOCALENV_CONSTRAINT_SOURCE_URL_TEST_OVERRIDE = "$DATABRICKS_HOST" + +[[Server]] +Pattern = "GET /serverless/serverless-v4/pyproject.toml" +Response.Body = ''' +[project] +requires-python = ">=3.12" + +[dependency-groups] +dev = ["databricks-connect~=17.2.0"] + +[tool.uv] +constraint-dependencies = ["pip<24"] +''' diff --git a/cmd/environments/telemetry.go b/cmd/environments/telemetry.go index 05c8aeb0a6..302260d124 100644 --- a/cmd/environments/telemetry.go +++ b/cmd/environments/telemetry.go @@ -106,6 +106,8 @@ func errorCodeType(code libslocalenv.ErrorCode) protos.SetupLocalErrorCode { return protos.SetupLocalErrorCodePythonInstall case libslocalenv.ErrProvision: return protos.SetupLocalErrorCodeProvision + case libslocalenv.ErrProvisionConflict: + return protos.SetupLocalErrorCodeProvisionConflict case libslocalenv.ErrValidate: return protos.SetupLocalErrorCodeValidate case libslocalenv.ErrCanceled: diff --git a/cmd/environments/telemetry_test.go b/cmd/environments/telemetry_test.go index 867930f46d..130637255e 100644 --- a/cmd/environments/telemetry_test.go +++ b/cmd/environments/telemetry_test.go @@ -117,6 +117,7 @@ func TestErrorCodeCoversLocalenv(t *testing.T) { libslocalenv.ErrMerge, libslocalenv.ErrPythonInstall, libslocalenv.ErrProvision, + libslocalenv.ErrProvisionConflict, libslocalenv.ErrValidate, libslocalenv.ErrCanceled, } diff --git a/libs/localenv/pipeline.go b/libs/localenv/pipeline.go index 23eb71b298..cb069eedb8 100644 --- a/libs/localenv/pipeline.go +++ b/libs/localenv/pipeline.go @@ -256,8 +256,21 @@ func (p *Pipeline) run(ctx context.Context) error { } p.markOK(PhaseMerge, "") - // Phase: provision — ensure Python, run uv sync, seed pip. + // The merge proved the written pins conflict with the user's dependencies + // (W_USER_CONSTRAINT_CONFLICT): uv sync would deterministically fail to resolve + // them, so report the conflict now — with a distinct E_PROVISION_CONFLICT code — + // instead of spending a doomed Python install and sync. The constraints are + // already on disk (diskMutated=true), which the extension's recovery flow relies + // on, and the failure is attributed to the provision phase it stands in for. + // Gating on the CLI's own merge detection keeps the code precise (no stderr + // matching, and no false positive on an unrelated sync failure). p.report(ctx, PhaseProvision) + if p.hasConstraintConflictWarning() { + return p.fail(PhaseProvision, true, NewError(ErrProvisionConflict, nil, + "dependency pins conflict with the environment constraints; relax the conflicting pins and re-run (see warnings)")) + } + + // Phase: provision — ensure Python, run uv sync, seed pip. if err := p.provision(ctx, pyMinor); err != nil { return err } @@ -529,6 +542,19 @@ func (p *Pipeline) provision(ctx context.Context, pyMinor string) error { return nil } +// hasConstraintConflictWarning reports whether the merge phase recorded a provable +// user/environment version conflict (W_USER_CONSTRAINT_CONFLICT). It reads the +// warnings already accumulated on the Result, which the merge phase populates +// before provision runs, so it is only meaningful once merge has completed. +func (p *Pipeline) hasConstraintConflictWarning() bool { + for _, w := range p.res.Warnings { + if w.Code == WarnUserConstraintConflict { + return true + } + } + return false +} + // validate reads the Python and databricks-connect versions from the venv and // populates the venv path. dbcPin is the resolved databricks-connect pin; when it // is not managed (opts.SkipDBConnect) the databricks-connect assertion and version diff --git a/libs/localenv/pipeline_test.go b/libs/localenv/pipeline_test.go index 6e75c503a0..bc12cfb4cb 100644 --- a/libs/localenv/pipeline_test.go +++ b/libs/localenv/pipeline_test.go @@ -496,6 +496,103 @@ func TestPipelineProvisionsAndValidatesExisting(t *testing.T) { assert.FileExists(t, filepath.Join(dir, "pyproject.toml.bak")) } +func TestPipelineFailsFastOnConstraintConflict(t *testing.T) { + // The user pins pip==24.0 while the environment's constraint-dependencies pin + // pip<24 — a provably disjoint range — so the merge records + // W_USER_CONSTRAINT_CONFLICT. uv sync would deterministically fail to resolve + // that, so the run reports E_PROVISION_CONFLICT right after the merge, at the + // provision phase, with disk already mutated — and never spends a Python install + // or sync (recordingPM records neither call). + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte(`[project] +name = "demo" +requires-python = ">=3.12" +dependencies = ["pip==24.0"] + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] +`), 0o644)) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`[project] +requires-python = ">=3.12" + +[dependency-groups] +dev = ["databricks-connect~=17.2.0"] + +[tool.uv] +constraint-dependencies = ["pip<24"] +`)) + })) + defer srv.Close() + + pm := &recordingPM{fakePM: fakePM{py: "3.12", dbc: "17.2.0"}} + p := &Pipeline{ + Mode: ModeDefault, ProjectDir: dir, + ConstraintBaseURL: srv.URL, CacheDir: t.TempDir(), + Flags: ComputeFlags{Serverless: "v4"}, + Compute: stubCompute{}, PM: pm, + } + res, err := p.Run(t.Context()) + var pe *PipelineError + require.ErrorAs(t, err, &pe) + assert.Equal(t, ErrProvisionConflict, pe.Code) + assert.Equal(t, PhaseProvision, pe.FailurePhase) + assert.True(t, pe.DiskMutated, "the merge wrote the pins before the conflict was reported") + require.NotNil(t, res.Error) + assert.Equal(t, ErrProvisionConflict, res.Error.Code) + // The merge conflict warning that gates the code must be present. + assert.Contains(t, codes(res.Warnings), WarnUserConstraintConflict) + // Fail-fast: neither Python install nor sync was attempted. + assert.Empty(t, pm.minor, "EnsurePython must not run when the conflict is already proven") + assert.Empty(t, pm.provisionPython, "uv sync must not run when the conflict is already proven") +} + +func TestPipelineCheckReportsConflictAsWarningNotError(t *testing.T) { + // --dry-run computes a plan and never evaluates provisioning, so the same + // provably-disjoint pins surface only as the W_USER_CONSTRAINT_CONFLICT warning + // with ok=true and no error — the conflict becomes E_PROVISION_CONFLICT only on a + // real run, which is the phase that attempts (and here would fail) provisioning. + // This pins that intended divergence so it cannot regress silently. + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "pyproject.toml"), []byte(`[project] +name = "demo" +requires-python = ">=3.12" +dependencies = ["pip==24.0"] + +[dependency-groups] +dev = ["databricks-connect~=16.0.0"] +`), 0o644)) + before, _ := os.ReadFile(filepath.Join(dir, "pyproject.toml")) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`[project] +requires-python = ">=3.12" + +[dependency-groups] +dev = ["databricks-connect~=17.2.0"] + +[tool.uv] +constraint-dependencies = ["pip<24"] +`)) + })) + defer srv.Close() + + p := &Pipeline{ + Mode: ModeDefault, Check: true, ProjectDir: dir, + ConstraintBaseURL: srv.URL, CacheDir: t.TempDir(), + Flags: ComputeFlags{Serverless: "v4"}, + Compute: stubCompute{}, PM: fakePM{py: "3.12", dbc: "17.2.0"}, + } + res, err := p.Run(t.Context()) + require.NoError(t, err) + assert.True(t, res.OK) + assert.Nil(t, res.Error, "a dry run reports the conflict as a warning, not an error") + assert.Contains(t, codes(res.Warnings), WarnUserConstraintConflict) + // A dry run mutates nothing. + after, _ := os.ReadFile(filepath.Join(dir, "pyproject.toml")) + assert.Equal(t, string(before), string(after)) + assert.NoFileExists(t, filepath.Join(dir, "pyproject.toml.bak")) +} + func TestPipelineDryRunOmitsFabricatedDBConnectVersion(t *testing.T) { // A major-only pin like ~=17.0 (serverless, environments#15) is not a concrete // version. Under --dry-run validate never corrects the reported value, so it diff --git a/libs/localenv/result.go b/libs/localenv/result.go index f7189f0274..def05c7f59 100644 --- a/libs/localenv/result.go +++ b/libs/localenv/result.go @@ -94,7 +94,8 @@ const ( ErrWrite ErrorCode = "E_WRITE" // merge: greenfield write failed ErrMerge ErrorCode = "E_MERGE" // merge: existing-project merge failed ErrPythonInstall ErrorCode = "E_PYTHON_INSTALL" // provision: uv python install failed - ErrProvision ErrorCode = "E_PROVISION" // provision: uv sync failed + ErrProvision ErrorCode = "E_PROVISION" // provision: uv sync failed (generic) + ErrProvisionConflict ErrorCode = "E_PROVISION_CONFLICT" // provision: merge detected an unsatisfiable version conflict ErrValidate ErrorCode = "E_VALIDATE" // validate: post-provision version mismatch // ErrCanceled is not in the spec's error-code table: it reports a user/parent diff --git a/libs/telemetry/protos/setup_local.go b/libs/telemetry/protos/setup_local.go index a608b6182f..e6eaf60275 100644 --- a/libs/telemetry/protos/setup_local.go +++ b/libs/telemetry/protos/setup_local.go @@ -41,6 +41,7 @@ const ( SetupLocalErrorCodeMerge SetupLocalErrorCode = "E_MERGE" SetupLocalErrorCodePythonInstall SetupLocalErrorCode = "E_PYTHON_INSTALL" SetupLocalErrorCodeProvision SetupLocalErrorCode = "E_PROVISION" + SetupLocalErrorCodeProvisionConflict SetupLocalErrorCode = "E_PROVISION_CONFLICT" SetupLocalErrorCodeValidate SetupLocalErrorCode = "E_VALIDATE" SetupLocalErrorCodeCanceled SetupLocalErrorCode = "E_CANCELED" )