Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .nextchanges/bundles/7767.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
* Use the Lakebase role when its create request fails with a transient error but the role was created, instead of failing the deploy, in the direct deployment engine. ([#7767](https://github.com/databricks/cli/pull/7767))
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
bundle:
name: test-bundle

sync:
paths: []

resources:
postgres_projects:
my_project:
project_id: test-pg-proj
display_name: "Test Postgres Project"
pg_version: 16

postgres_branches:
main:
parent: ${resources.postgres_projects.my_project.id}
branch_id: main
no_expiry: true

postgres_roles:
owner:
parent: ${resources.postgres_branches.main.id}
role_id: app-owner
postgres_role: app_owner
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Cloud = false
EnvMatrix.DMS = ["", "true"]
EnvMatrix.READPLAN = []
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@

=== The create request fails with 500 but the role was created: deploy uses the role

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_branches.main
Created postgres_projects.my_project
Created postgres_roles.owner
Files: 0 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles
POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner

=== The create request fails with 500 and the role was not created: deploy repeats the create

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_branches.main
Created postgres_projects.my_project
Warn: deploying resources.postgres_roles.owner: retrying after 500 Internal Server Error from POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
Created postgres_roles.owner
Files: 0 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles
POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner
POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner/operations/[UUID][0]

=== The role cannot be read after the create request fails: deploy fails with the create error

>>> musterr [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_branches.main
Created postgres_projects.my_project
Warn: deploying resources.postgres_roles.owner: checking whether projects/test-pg-proj/branches/main/roles/app-owner was created: Fault injected by test.
Error: cannot create resources.postgres_roles.owner: Fault injected by test. (500 INTERNAL_ERROR)

Endpoint: POST [DATABRICKS_URL]/api/2.0/postgres/projects/test-pg-proj/branches/main/roles?role_id=app-owner
HTTP Status: 500 Internal Server Error
API error_code: INTERNAL_ERROR
API message: Fault injected by test.

Files: 0 uploaded, 0 deleted

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles
POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner

=== With replace_existing, a failed create request is repeated without reading the role

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
Created postgres_branches.main
Created postgres_projects.my_project
Warn: deploying resources.postgres_roles.owner: retrying after 500 Internal Server Error from POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
Created postgres_roles.owner
Files: 0 uploaded, 0 deleted
Resources: 3 created, 0 changed, 0 deleted, 0 unchanged

>>> print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles
POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
POST /api/2.0/postgres/projects/test-pg-proj/branches/main/roles
GET /api/2.0/postgres/projects/test-pg-proj/branches/main/roles/app-owner/operations/[UUID][1]
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
title "The create request fails with 500 but the role was created: deploy uses the role"
fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "The create request fails with 500 and the role was not created: deploy repeats the create"
fault.py "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "The role cannot be read after the create request fails: deploy fails with the create error"
fault.py "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR
fault.py "GET /api/2.0/postgres/projects/*/branches/*/roles/app-owner" 403 0 1 PERMISSION_DENIED
trace musterr $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
rm -f "$OUT_REQUESTS"

title "With replace_existing, a failed create request is repeated without reading the role"
update_file.py databricks.yml "postgres_role: app_owner" "postgres_role: app_owner
replace_existing: true"
fault.py --after-handler "POST /api/2.0/postgres/projects/*/branches/*/roles" 500 0 1 INTERNAL_ERROR
trace $CLI bundle deploy
trace print_requests.py --get //postgres/projects/test-pg-proj/branches/main/roles | jq -r '.method + " " + .path'
$CLI bundle destroy --auto-approve &> LOG.destroy
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Cloud = false
RecordRequests = true
EnvMatrix.READPLAN = []
Env.DATABRICKS_BUNDLE_ENGINE = "direct"
Env.DATABRICKS_BUNDLE_RETRY_INTERVAL_MS = "100"
SuppressLines.warn_retry = ""
54 changes: 50 additions & 4 deletions bundle/direct/dresources/postgres_role.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@ package dresources

Comment thread
denik marked this conversation as resolved.
Resolved
import (
"context"
"fmt"
"time"

"github.com/databricks/cli/bundle/config/resources"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/retry"
"github.com/databricks/databricks-sdk-go"
"github.com/databricks/databricks-sdk-go/common/types/fieldmask"
sdktime "github.com/databricks/databricks-sdk-go/common/types/time"
Expand Down Expand Up @@ -123,7 +127,7 @@ func (r *ResourcePostgresRole) DoRead(ctx context.Context, id string) (*Postgres
}

func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRoleState) (string, *PostgresRoleRemote, error) {
waiter, err := r.client.Postgres.CreateRole(ctx, postgres.CreateRoleRequest{
req := postgres.CreateRoleRequest{
RoleId: config.RoleId,
Parent: config.Parent,
Role: postgres.Role{
Expand All @@ -140,20 +144,62 @@ func (r *ResourcePostgresRole) DoCreate(ctx context.Context, config *PostgresRol
},
ReplaceExisting: config.ReplaceExisting != nil && *config.ReplaceExisting,
ForceSendFields: nil,
})
}
start := time.Now()
waiter, err := r.client.Postgres.CreateRole(ctx, req)
if err != nil {
return "", nil, err
return r.recoverFailedCreate(ctx, req, start, err)
}
return r.waitCreated(ctx, waiter)
}

// waitCreated waits for the create operation of a role that a create request returned.
func (r *ResourcePostgresRole) waitCreated(ctx context.Context, waiter postgres.CreateRoleOperationInterface) (string, *PostgresRoleRemote, error) {
result, err := waitCreateOperation(ctx, waiter.Wait)
if err != nil {
return "", nil, err
}

remote := makePostgresRoleRemote(result)
return remote.Name, remote, nil
}

// recoverFailedCreate handles a create request that failed; see recoverPostgresCreate. Role
// creates on a just-created branch are answered with 500 INTERNAL_ERROR now and then, and the
// request may have created the role anyway, so look for it before repeating the request.
func (r *ResourcePostgresRole) recoverFailedCreate(ctx context.Context, req postgres.CreateRoleRequest, start time.Time, createErr error) (string, *PostgresRoleRemote, error) {
if ctx.Err() != nil || !retry.IsTransient(ctx, createErr) || req.RoleId == "" {
return "", nil, createErr
}
// A role found after a replace may predate it, and repeating a replace is safe.
if req.ReplaceExisting {
return "", nil, retrySafe(createErr)
}

name := req.Parent + "/roles/" + req.RoleId
return recoverPostgresCreate(ctx, postgresCreateRecovery[postgres.Role, *PostgresRoleRemote]{
name: name,
get: func(ctx context.Context) (*postgres.Role, error) {
return r.client.Postgres.GetRole(ctx, postgres.GetRoleRequest{Name: name})
},
create: func(ctx context.Context) (func() (string, *PostgresRoleRemote, error), error) {
waiter, err := r.client.Postgres.CreateRole(ctx, req)
if err != nil {
return nil, err
}
return func() (string, *PostgresRoleRemote, error) { return r.waitCreated(ctx, waiter) }, nil
},
use: func(ctx context.Context, role *postgres.Role) (string, *PostgresRoleRemote, error) {
if !createdByRequestAt(role.CreateTime, start) {
Comment thread
denik marked this conversation as resolved.
Resolved
return "", nil, fmt.Errorf("%w; role %s exists but was not created by this deployment", createErr, role.Name)
}
// Info, not Warn: the deploy got the role it asked for, so there is nothing to act on.
log.Infof(ctx, "using role %s, created despite the failed create request: %s", role.Name, createErr)
remote := makePostgresRoleRemote(role)
return remote.Name, remote, nil
},
}, createErr)
}

func (r *ResourcePostgresRole) DoUpdate(ctx context.Context, id string, config *PostgresRoleState, entry *PlanEntry) (*PostgresRoleRemote, error) {
// Build update mask from fields that have action="update" in the changes map.
// Prefix with "spec." because the API expects paths relative to the Role
Expand Down
Loading
Loading