diff --git a/.nextchanges/bundles/7767.md b/.nextchanges/bundles/7767.md new file mode 100644 index 00000000000..8aa4a1a4c8c --- /dev/null +++ b/.nextchanges/bundles/7767.md @@ -0,0 +1 @@ +* Use the Lakebase role when its create request fails with a transient error but the role was created, instead of failing the deploy, in the direct deployment engine. ([#7767](https://github.com/databricks/cli/pull/7767)) diff --git a/acceptance/bundle/resources/postgres_roles/create_transient_500/databricks.yml b/acceptance/bundle/resources/postgres_roles/create_transient_500/databricks.yml new file mode 100644 index 00000000000..bca22928597 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/create_transient_500/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: test-bundle + +sync: + paths: [] + +resources: + postgres_projects: + my_project: + project_id: test-pg-proj + display_name: "Test Postgres Project" + pg_version: 16 + + postgres_branches: + main: + parent: ${resources.postgres_projects.my_project.id} + branch_id: main + no_expiry: true + + postgres_roles: + owner: + parent: ${resources.postgres_branches.main.id} + role_id: app-owner + postgres_role: app_owner diff --git a/acceptance/bundle/resources/postgres_roles/create_transient_500/out.test.toml b/acceptance/bundle/resources/postgres_roles/create_transient_500/out.test.toml new file mode 100644 index 00000000000..15c32f324ac --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/create_transient_500/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] +EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/postgres_roles/create_transient_500/output.txt b/acceptance/bundle/resources/postgres_roles/create_transient_500/output.txt new file mode 100644 index 00000000000..57b47fcc4ed --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/create_transient_500/output.txt @@ -0,0 +1,67 @@ + +=== The create request fails with 500 but the role was created: deploy uses the role + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_branches.main +Created postgres_projects.my_project +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles +POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner + +=== The create request fails with 500 and the role was not created: deploy repeats the create + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_branches.main +Created postgres_projects.my_project +Warn: deploying resources.postgres_roles.owner: retrying after 500 Internal Server Error from POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles +POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner +POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner/operations/[UUID][0] + +=== The role cannot be read after the create request fails: deploy fails with the create error + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_branches.main +Created postgres_projects.my_project +Warn: deploying resources.postgres_roles.owner: checking whether projects/test-pg-proj/branches/main/roles/app-owner was created: Fault injected by test. +Error: cannot create resources.postgres_roles.owner: Fault injected by test. (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj/branches/main/roles?role_id=app-owner +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Fault injected by test. + +Files: 0 uploaded, 0 deleted + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles +POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner + +=== With replace_existing, a failed create request is repeated without reading the role + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created postgres_branches.main +Created postgres_projects.my_project +Warn: deploying resources.postgres_roles.owner: retrying after 500 Internal Server Error from POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +Created postgres_roles.owner +Files: 0 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles +POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles +GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner/operations/[UUID][1] diff --git a/acceptance/bundle/resources/postgres_roles/create_transient_500/script b/acceptance/bundle/resources/postgres_roles/create_transient_500/script new file mode 100644 index 00000000000..c21d956254d --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/create_transient_500/script @@ -0,0 +1,29 @@ +title "The create request fails with 500 but the role was created: deploy uses the role" +fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "The create request fails with 500 and the role was not created: deploy repeats the create" +fault.py "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "The role cannot be read after the create request fails: deploy fails with the create error" +fault.py "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR +fault.py "GET /api/2.0/postgres/projects/*/branches/*/roles/app-owner" 403 0 1 PERMISSION_DENIED +trace musterr $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy +rm -f "$OUT_REQUESTS" + +title "With replace_existing, a failed create request is repeated without reading the role" +update_file.py databricks.yml "postgres_role: app_owner" "postgres_role: app_owner + replace_existing: true" +fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR +trace $CLI bundle deploy +trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path' +$CLI bundle destroy --auto-approve &> LOG.destroy diff --git a/acceptance/bundle/resources/postgres_roles/create_transient_500/test.toml b/acceptance/bundle/resources/postgres_roles/create_transient_500/test.toml new file mode 100644 index 00000000000..ab3f407bfa7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/create_transient_500/test.toml @@ -0,0 +1,6 @@ +Cloud = false +RecordRequests = true +EnvMatrix.READPLAN = [] +Env.DATABRICKS_BUNDLE_ENGINE = "direct" +Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100" +SuppressLines.warn_retry = "" diff --git a/bundle/direct/dresources/postgres_role.go b/bundle/direct/dresources/postgres_role.go index 8c658a2e4e9..3e630fa95d7 100644 --- a/bundle/direct/dresources/postgres_role.go +++ b/bundle/direct/dresources/postgres_role.go @@ -2,8 +2,12 @@ package dresources import ( "context" + "fmt" + "time" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/common/types/fieldmask" sdktime "github.com/databricks/databricks-sdk-go/common/types/time" @@ -123,7 +127,7 @@ func (r *ResourcePostgresRole) DoRead(ctx context.Context, id string) (*Postgres } func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRoleState) (string, *PostgresRoleRemote, error) { - waiter, err := r.client.Postgres.CreateRole(ctx, postgres.CreateRoleRequest{ + req := postgres.CreateRoleRequest{ RoleId: config.RoleId, Parent: config.Parent, Role: postgres.Role{ @@ -140,20 +144,62 @@ func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRol }, ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting, ForceSendFields: nil, - }) + } + start := time.Now() + waiter, err := r.client.Postgres.CreateRole(ctx, req) if err != nil { - return "", nil, err + return r.recoverFailedCreate(ctx, req, start, err) } + return r.waitCreated(ctx, waiter) +} +// waitCreated waits for the create operation of a role that a create request returned. +func (r *ResourcePostgresRole) waitCreated(ctx context.Context, waiter postgres.CreateRoleOperationInterface) (string, *PostgresRoleRemote, error) { result, err := waitCreateOperation(ctx, waiter.Wait) if err != nil { return "", nil, err } - remote := makePostgresRoleRemote(result) return remote.Name, remote, nil } +// recoverFailedCreate handles a create request that failed; see recoverPostgresCreate. Role +// creates on a just-created branch are answered with 500 INTERNAL_ERROR now and then, and the +// request may have created the role anyway, so look for it before repeating the request. +func (r *ResourcePostgresRole) recoverFailedCreate(ctx context.Context, req postgres.CreateRoleRequest, start time.Time, createErr error) (string, *PostgresRoleRemote, error) { + if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) || req.RoleId == "" { + return "", nil, createErr + } + // A role found after a replace may predate it, and repeating a replace is safe. + if req.ReplaceExisting { + return "", nil, retrySafe(createErr) + } + + name := req.Parent + "/roles/" + req.RoleId + return recoverPostgresCreate(ctx, postgresCreateRecovery[postgres.Role, *PostgresRoleRemote]{ + name: name, + get: func(ctx context.Context) (*postgres.Role, error) { + return r.client.Postgres.GetRole(ctx, postgres.GetRoleRequest{Name: name}) + }, + create: func(ctx context.Context) (func() (string, *PostgresRoleRemote, error), error) { + waiter, err := r.client.Postgres.CreateRole(ctx, req) + if err != nil { + return nil, err + } + return func() (string, *PostgresRoleRemote, error) { return r.waitCreated(ctx, waiter) }, nil + }, + use: func(ctx context.Context, role *postgres.Role) (string, *PostgresRoleRemote, error) { + if !createdByRequestAt(role.CreateTime, start) { + return "", nil, fmt.Errorf("%w; role %s exists but was not created by this deployment", createErr, role.Name) + } + // Info, not Warn: the deploy got the role it asked for, so there is nothing to act on. + log.Infof(ctx, "using role %s, created despite the failed create request: %s", role.Name, createErr) + remote := makePostgresRoleRemote(role) + return remote.Name, remote, nil + }, + }, createErr) +} + func (r *ResourcePostgresRole) DoUpdate(ctx context.Context, id string, config *PostgresRoleState, entry *PlanEntry) (*PostgresRoleRemote, error) { // Build update mask from fields that have action="update" in the changes map. // Prefix with "spec." because the API expects paths relative to the Role diff --git a/bundle/direct/dresources/postgres_role_test.go b/bundle/direct/dresources/postgres_role_test.go new file mode 100644 index 00000000000..47b1b04dde0 --- /dev/null +++ b/bundle/direct/dresources/postgres_role_test.go @@ -0,0 +1,183 @@ +package dresources + +import ( + "encoding/json" + "errors" + "testing" + "time" + + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" + "github.com/databricks/cli/libs/testserver" + "github.com/databricks/databricks-sdk-go" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testRoleParent = "projects/p/branches/main" + testRoleName = testRoleParent + "/roles/owner" +) + +// roleCreateScript plays the given responses, in order, to the role create and get requests. +// A response is a postgres.Role (for a create: the finished operation that returns it) or a +// testserver.Response for an error. +type roleCreateScript struct { + t *testing.T + creates []any + gets []any + + createCount int + getCount int + operation *postgres.Operation +} + +func (s *roleCreateScript) client() *databricks.WorkspaceClient { + server := testserver.New(s.t) + server.Handle("POST", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles", func(req testserver.Request) any { + s.createCount++ + require.LessOrEqual(s.t, s.createCount, len(s.creates), "unexpected create request") + resp := s.creates[s.createCount-1] + if role, ok := resp.(postgres.Role); ok { + body, err := json.Marshal(role) + require.NoError(s.t, err) + s.operation = &postgres.Operation{Name: testRoleName + "/operations/op", Done: true, Response: body} + return s.operation + } + return resp + }) + // The SDK polls the operation even when the create response says it is done. + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles/{role_id}/operations/{operation_id}", func(req testserver.Request) any { + require.NotNil(s.t, s.operation, "unexpected operation request") + return s.operation + }) + server.Handle("GET", "/api/2.0/postgres/projects/{project_id}/branches/{branch_id}/roles/{role_id}", func(req testserver.Request) any { + s.getCount++ + require.LessOrEqual(s.t, s.getCount, len(s.gets), "unexpected get request") + return s.gets[s.getCount-1] + }) + testserver.AddDefaultHandlers(server) + + client, err := databricks.NewWorkspaceClient(&databricks.Config{Host: server.URL, Token: "testtoken"}) + require.NoError(s.t, err) + return client +} + +var ( + // What the cloud answers to a role create now and then, right after the branch is created. + roleInternalError = branchError(500, "INTERNAL_ERROR", "The request could not be processed. Please try again later and contact Databricks if the problem persists.") + roleNotFound = branchError(404, "NOT_FOUND", "role not found") + // The real API answers a duplicate role with 400, not 409 ALREADY_EXISTS. + roleAlreadyExists = branchError(400, "BAD_REQUEST", "role with that name already exists") +) + +func testRole(created time.Time) postgres.Role { + return postgres.Role{Name: testRoleName, RoleId: "owner", Parent: testRoleParent, CreateTime: sdktime.New(created)} +} + +// createTestRole runs DoCreate with fast retries. +func createTestRole(t *testing.T, s *roleCreateScript, replaceExisting bool) (string, *PostgresRoleRemote, error) { + t.Helper() + ctx := env.Set(t.Context(), retry.IntervalMsVariable, "1") + r := (&ResourcePostgresRole{}).New(s.client()) + return r.DoCreate(ctx, &PostgresRoleState{RoleId: "owner", Parent: testRoleParent, ReplaceExisting: &replaceExisting}) +} + +func TestPostgresRoleCreateUsesRoleCreatedDespiteError(t *testing.T) { + s := &roleCreateScript{t: t, creates: []any{roleInternalError}, gets: []any{testRole(time.Now())}} + + id, remote, err := createTestRole(t, s, false) + + require.NoError(t, err) + assert.Equal(t, testRoleName, id) + assert.Equal(t, "owner", remote.RoleId) + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresRoleCreateRepeatsCreateWhileRoleIsMissing(t *testing.T) { + s := &roleCreateScript{t: t, creates: []any{roleInternalError, testRole(time.Now())}, gets: []any{roleNotFound}} + + id, _, err := createTestRole(t, s, false) + + require.NoError(t, err) + assert.Equal(t, testRoleName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresRoleCreateUsesRoleWhenRepeatedCreateFindsIt(t *testing.T) { + // The first request was still completing when the role was read, and has landed since. + s := &roleCreateScript{ + t: t, + creates: []any{roleInternalError, roleAlreadyExists}, + gets: []any{roleNotFound, testRole(time.Now())}, + } + + id, _, err := createTestRole(t, s, false) + + require.NoError(t, err) + assert.Equal(t, testRoleName, id) + assert.Equal(t, 2, s.createCount) + assert.Equal(t, 2, s.getCount) +} + +func TestPostgresRoleCreateGivesUpWhenRoleStaysMissing(t *testing.T) { + creates := []any{roleInternalError} + var gets []any + for range retry.MaxRetries { + creates = append(creates, roleInternalError) + gets = append(gets, roleNotFound) + } + gets = append(gets, roleNotFound) + s := &roleCreateScript{t: t, creates: creates, gets: gets} + + _, _, err := createTestRole(t, s, false) + + require.ErrorContains(t, err, "The request could not be processed") + assert.Equal(t, retry.MaxRetries+1, s.createCount) + assert.Equal(t, retry.MaxRetries+1, s.getCount) +} + +func TestPostgresRoleCreateDoesNotTakeOverOlderRole(t *testing.T) { + s := &roleCreateScript{t: t, creates: []any{roleInternalError}, gets: []any{testRole(time.Now().Add(-time.Hour))}} + + _, _, err := createTestRole(t, s, false) + + require.ErrorContains(t, err, "The request could not be processed") + require.ErrorContains(t, err, "role "+testRoleName+" exists but was not created by this deployment") + assert.Equal(t, 1, s.createCount) +} + +func TestPostgresRoleCreateFailsWhenRoleCannotBeRead(t *testing.T) { + s := &roleCreateScript{t: t, creates: []any{roleInternalError}, gets: []any{branchError(403, "PERMISSION_DENIED", "no access")}} + + _, _, err := createTestRole(t, s, false) + + require.ErrorContains(t, err, "The request could not be processed") + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 1, s.getCount) +} + +func TestPostgresRoleCreateLeavesReplaceToCallerRetry(t *testing.T) { + s := &roleCreateScript{t: t, creates: []any{roleInternalError}} + + _, _, err := createTestRole(t, s, true) + + // Marked safe to retry for DeploymentUnit.Create, without reading the role. + _, ok := errors.AsType[*RetrySafeError](err) + assert.True(t, ok, "want a RetrySafeError, got %v", err) + assert.Equal(t, 0, s.getCount) +} + +func TestPostgresRoleCreateDoesNotRecoverFromNonTransientError(t *testing.T) { + s := &roleCreateScript{t: t, creates: []any{branchError(400, "INVALID_PARAMETER_VALUE", "bad spec")}} + + _, _, err := createTestRole(t, s, false) + + require.ErrorContains(t, err, "bad spec") + assert.Equal(t, 1, s.createCount) + assert.Equal(t, 0, s.getCount) +}