From bcdf2ddd195d29051543d5b6ee174fe96242060a Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 15 Nov 2026 05:30:24 +0000 Subject: [PATCH 01/11] air: add a hidden `air deploy` command that registers and serves a model `databricks air deploy -f deploy.yaml` registers a Unity Catalog model version through the registerModelPackage API (a checkpoint in a volume, a model from code, a model directory saved by MLflow) and serves it on a Model Serving endpoint, or serves an existing version. Registration is idempotent per key; the CLI repeats the same request until the version is ready, and prints the command that continues after Ctrl-C or --timeout. The command is hidden until the API is on everywhere. The run YAML loader, --override, -h config., code_source file selection and root path resolution are shared with `air run`. Co-authored-by: Isaac --- cmd/air/air.go | 2 +- cmd/air/air_test.go | 2 +- cmd/air/deploy.go | 382 ++++++++++++++++++++++++++++ cmd/air/deploy_test.go | 366 +++++++++++++++++++++++++++ cmd/air/deploycode.go | 155 ++++++++++++ cmd/air/deploycode_test.go | 183 ++++++++++++++ cmd/air/deployconfig.go | 387 +++++++++++++++++++++++++++++ cmd/air/deployconfig_test.go | 306 +++++++++++++++++++++++ cmd/air/deployendpoint.go | 186 ++++++++++++++ cmd/air/deployendpoint_test.go | 280 +++++++++++++++++++++ cmd/air/deployregister.go | 172 +++++++++++++ cmd/air/deployregister_test.go | 165 ++++++++++++ cmd/air/run.go | 4 +- cmd/air/runconfig.go | 47 ++-- cmd/air/runconfig_load.go | 33 ++- cmd/air/runconfig_override.go | 7 +- cmd/air/runconfig_override_test.go | 2 +- cmd/air/runconfig_test.go | 6 +- cmd/air/runsubmit.go | 8 +- cmd/air/runsubmit_test.go | 6 +- 20 files changed, 2658 insertions(+), 41 deletions(-) create mode 100644 cmd/air/deploy.go create mode 100644 cmd/air/deploy_test.go create mode 100644 cmd/air/deploycode.go create mode 100644 cmd/air/deploycode_test.go create mode 100644 cmd/air/deployconfig.go create mode 100644 cmd/air/deployconfig_test.go create mode 100644 cmd/air/deployendpoint.go create mode 100644 cmd/air/deployendpoint_test.go create mode 100644 cmd/air/deployregister.go create mode 100644 cmd/air/deployregister_test.go diff --git a/cmd/air/air.go b/cmd/air/air.go index 4154bab891c..584f4165c58 100644 --- a/cmd/air/air.go +++ b/cmd/air/air.go @@ -47,7 +47,7 @@ active runs with "databricks air list".`, wrapRunErrorWithDebugTip(runCommand) addToGroup(cmd, runGroupID, runCommand) addToGroup(cmd, manageGroupID, newGetCommand(), newListCommand(), newLogsCommand(), newCancelCommand()) - addToGroup(cmd, packageGroupID, newConvertToDabsCommand(), newImagesCommand()) + addToGroup(cmd, packageGroupID, newDeployCommand(), newConvertToDabsCommand(), newImagesCommand()) addToGroup(cmd, capacityGroupID, newPoolsCommand()) // Subcommands inherit this help func, so the footer is printed only for `air` itself. diff --git a/cmd/air/air_test.go b/cmd/air/air_test.go index 11e4546d21d..1c877160270 100644 --- a/cmd/air/air_test.go +++ b/cmd/air/air_test.go @@ -18,7 +18,7 @@ func TestNewRegistersAllSubcommands(t *testing.T) { registered[c.Name()] = true } - want := []string{"run", "get", "list", "pools", "logs", "cancel", "convert-to-dabs", "images"} + want := []string{"run", "get", "list", "pools", "logs", "cancel", "deploy", "convert-to-dabs", "images"} for _, name := range want { assert.True(t, registered[name], "subcommand %q is not registered", name) } diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go new file mode 100644 index 00000000000..1b8f0932e92 --- /dev/null +++ b/cmd/air/deploy.go @@ -0,0 +1,382 @@ +package aircmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/spf13/cobra" +) + +// defaultDeployTimeout bounds the whole wait: staging a checkpoint's weights and +// starting the endpoint. +const defaultDeployTimeout = time.Hour + +// Errors air deploy wraps when it stops waiting. The server keeps working, and +// the error says how to continue. +var ( + errDeployInterrupted = errors.New("air deploy was interrupted") + errDeployTimedOut = errors.New("air deploy timed out") +) + +// deployResult is the JSON payload for `air deploy`. +type deployResult struct { + Status string `json:"status"` + DryRun bool `json:"dry_run,omitempty"` + // CodeFiles lists, on a dry run of a model from code, the files it would send. + CodeFiles []string `json:"code_files,omitempty"` + RegisteredModel string `json:"registered_model,omitempty"` + ModelVersion string `json:"model_version,omitempty"` + ModelID string `json:"model_id,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` + // EnvironmentStagingFailed is set when the version's serving environment + // couldn't be prepared ahead of time, so the endpoint can be slower to start. + EnvironmentStagingFailed bool `json:"environment_staging_failed,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + EndpointURL string `json:"endpoint_url,omitempty"` +} + +// deployOptions are the `air deploy` flags. +type deployOptions struct { + file string + overrides []string + dryRun bool + idempotencyKey string + retryFailedStaging bool + timeout time.Duration +} + +func newDeployCommand() *cobra.Command { + return newDeployCommandWithPollInterval(registrationPollInterval) +} + +func newDeployCommandWithPollInterval(pollInterval time.Duration) *cobra.Command { + var opts deployOptions + + cmd := &cobra.Command{ + Use: "deploy", + Args: root.NoArgs, + Short: "Register a model in Unity Catalog and serve it on an endpoint", + Long: `Register a model in Unity Catalog and serve it on a Model Serving endpoint. + +The deployment is described by a YAML file (see --file). The model is a +checkpoint (weights in a Unity Catalog volume and the command that serves +them), a model from code, a model directory saved by MLflow, or an existing +model version. air deploy registers a new version, waits until it is ready, +then creates or updates the endpoint to serve it and waits until the endpoint +is ready. + +Registration is idempotent: run air deploy again with the idempotency key it +prints to continue an interrupted deployment. To look up a YAML field, pass +its path to -h as a separate argument: + + databricks air deploy -h config all fields + databricks air deploy -h config.model one section + databricks air deploy -h config.model.type one field`, + Example: ` # Register a checkpoint and serve it + databricks air deploy -f deploy.yaml + + # Validate the config, and list the code it would send, without deploying + databricks air deploy -f deploy.yaml --dry-run + + # Continue a deployment that was interrupted + databricks air deploy -f deploy.yaml --idempotency-key 3f6c0b52-4c7e-4d0b-9b8e-0f7ad6a4c2d1`, + // Hidden until the registration API is available in every workspace. + Hidden: true, + } + + // cobra passes -h's positional args to the help func before Args/required-flag + // validation, so a config path documents a field without needing -f. + cmd.SetHelpFunc(func(c *cobra.Command, args []string) { + fields := c.Flags().Args() + if len(fields) == 0 { + // Parent() is nil for a detached command (unit tests). + if parent := c.Parent(); parent != nil { + parent.HelpFunc()(c, args) + return + } + _ = c.Usage() + return + } + field, err := resolveSchemaField(deploySchema(), fields[0]) + if err != nil { + c.PrintErrln("Error:", err) + return + } + renderConfigField(c.OutOrStdout(), deployCommandPath, field, nil) + }) + + // List flags in declaration order (most used first) rather than alphabetically. + cmd.Flags().SortFlags = false + cmd.Flags().StringVarP(&opts.file, "file", "f", "", "Path to the deployment YAML config") + cmd.Flags().StringArrayVar(&opts.overrides, "override", nil, "Override a YAML field, e.g. compute.workload_size=Medium (repeatable)") + cmd.Flags().BoolVar(&opts.dryRun, "dry-run", false, "Validate the config, and list the code it would send, without deploying") + cmd.Flags().StringVar(&opts.idempotencyKey, "idempotency-key", "", "Continue the registration that used this key, instead of registering another version") + cmd.Flags().BoolVar(&opts.retryFailedStaging, "retry-failed-staging", false, "Copy a checkpoint's weights again after copying them failed") + cmd.Flags().DurationVar(&opts.timeout, "timeout", defaultDeployTimeout, "How long to wait for the version and the endpoint to be ready") + _ = cmd.MarkFlagRequired("file") + + // --dry-run never authenticates, so it reports config mistakes without + // credentials. + cmd.PreRunE = func(cmd *cobra.Command, args []string) error { + if opts.dryRun { + return nil + } + return root.MustWorkspaceClient(cmd, args) + } + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + err := runDeploy(cmd, opts, pollInterval) + if err == nil { + return nil + } + code, retryable := deployErrorCode(err) + kind := "PERMANENT" + if retryable { + kind = "TRANSIENT" + } + return renderError(cmd.Context(), cmd, code, kind, retryable, err) + } + return cmd +} + +func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duration) error { + ctx := cmd.Context() + cfg, err := loadDeployConfig(ctx, opts.file, opts.overrides) + if err != nil { + return err + } + if err := validateDeployFlags(cfg, opts); err != nil { + return err + } + + var code *modelPackageCode + if cfg.Model.Type == modelTypeCode { + code, err = collectModelCode(ctx, cfg, opts.file) + if err != nil { + return err + } + } + + jsonOut := root.OutputType(cmd) == flags.OutputJSON + if opts.dryRun { + return renderDeployDryRun(ctx, cfg, code, jsonOut) + } + + w := cmdctx.WorkspaceClient(ctx) + ctx, cancel := context.WithTimeout(ctx, opts.timeout) + defer cancel() + ctx, stop := notifyInterrupt(ctx) + defer stop() + + result := deployResult{RegisteredModel: cfg.RegisteredModel, Endpoint: cfg.Endpoint} + if cfg.registers() { + // Fail on an endpoint air deploy won't change before copying any weights. + if err := checkEndpoint(ctx, w, cfg.Endpoint, cfg.RegisteredModel); err != nil { + return err + } + result.IdempotencyKey = effectiveIdempotencyToken(opts.idempotencyKey, cfg.IdempotencyToken) + resp, err := registerDeployment(ctx, w, cfg, opts, code, result.IdempotencyKey, pollInterval, !jsonOut) + if err != nil { + return err + } + result.ModelVersion, result.ModelID = resp.ModelVersion, resp.ModelID + result.EnvironmentStagingFailed = resp.environmentState() == stagingStateFailed + if !jsonOut { + cmdio.LogString(ctx, fmt.Sprintf("Version %s of %s is ready.", resp.ModelVersion, cfg.RegisteredModel)) + if result.EnvironmentStagingFailed { + cmdio.LogString(ctx, "Warning: the version's serving environment couldn't be prepared ahead of time, so the endpoint can be slower to start, or fail to start.") + } + } + } else { + result.ModelVersion = *cfg.Model.Version + } + + entity := cfg.servedEntity(result.ModelVersion) + endpoint, err := serveVersion(ctx, w, cfg.Endpoint, entity, !jsonOut) + if err != nil { + resume := deployResumeCommand(w.Config.Profile, opts, result.IdempotencyKey) + if stopped := deployStopped(ctx, err); stopped != nil { + return fmt.Errorf("%w while waiting for endpoint %s, which keeps updating. To continue, run:\n %s", stopped, cfg.Endpoint, resume) + } + return fmt.Errorf("serving version %s of %s on endpoint %s failed: %w\n\nTo try again without registering another version, run:\n %s", result.ModelVersion, cfg.RegisteredModel, cfg.Endpoint, err, resume) + } + result.Status = "READY" + result.EndpointURL = endpointInvocationsURL(w.Config.Host, endpoint.Name) + if jsonOut { + return renderEnvelope(ctx, result) + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "Endpoint %s serves version %s of %s.\n", endpoint.Name, result.ModelVersion, cfg.RegisteredModel) + fmt.Fprintln(out, "Query it at: "+result.EndpointURL) + return nil +} + +// renderDeployDryRun reports a valid config and, for a model from code, the +// files a deployment would send. +func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPackageCode, jsonOut bool) error { + var files []string + if code != nil { + files = codeFilePaths(code) + } + if jsonOut { + return renderEnvelope(ctx, deployResult{Status: "DRY_RUN_OK", DryRun: true, CodeFiles: files}) + } + if code != nil { + cmdio.LogString(ctx, fmt.Sprintf("Dry run: the model from code would include %d file(s):", len(files))) + for _, f := range files { + cmdio.LogString(ctx, " "+f) + } + } + cmdio.LogString(ctx, fmt.Sprintf("Dry run: configuration for endpoint %q is valid; not deploying.", cfg.Endpoint)) + return nil +} + +// registerDeployment registers the version and waits until it is ready, showing +// progress on a spinner when show is true. Its errors say how to continue. +func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, code *modelPackageCode, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { + if show { + cmdio.LogString(ctx, fmt.Sprintf("Registering a version of %s with idempotency key %s", cfg.RegisteredModel, key)) + } + req := cfg.registrationRequest(key, code) + req.RetryFailedStaging = opts.retryFailedStaging + + progress := func(*registerModelPackageResponse) {} + if show { + sp := cmdio.NewSpinner(ctx) + defer sp.Close() + progress = func(resp *registerModelPackageResponse) { + sp.Update(registrationProgress(cfg, resp)) + } + } + resp, err := registerUntilReady(ctx, w, req, pollInterval, progress) + if err == nil { + return resp, nil + } + resume := deployResumeCommand(w.Config.Profile, opts, key) + if errors.Is(err, errStagingFailed) { + source := "the checkpoint's weights" + if cfg.Model.Path != nil { + source = "the weights in " + *cfg.Model.Path + } + return nil, fmt.Errorf("copying %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", source, resp.ModelVersion, cfg.RegisteredModel, resume) + } + if stopped := deployStopped(ctx, err); stopped != nil { + return nil, fmt.Errorf("%w while registering a version of %s. Copying a checkpoint's weights continues, and the version is finalized when you continue. To continue, run:\n %s", stopped, cfg.RegisteredModel, resume) + } + return nil, fmt.Errorf("registering a version of %s with idempotency key %s failed: %w", cfg.RegisteredModel, key, err) +} + +// deployStopped returns why air deploy stopped waiting, or nil if it didn't. The +// SDK's endpoint poller reports a canceled context as its own timeout, so the +// context decides between Ctrl-C and --timeout. +func deployStopped(ctx context.Context, err error) error { + switch { + case errors.Is(ctx.Err(), context.Canceled): + return errDeployInterrupted + case errors.Is(ctx.Err(), context.DeadlineExceeded): + return errDeployTimedOut + } + if _, ok := errors.AsType[*retries.ErrTimedOut](err); ok { + return errDeployTimedOut + } + return nil +} + +// deployErrorCode classifies a failed deployment for the JSON error envelope. +func deployErrorCode(err error) (code string, retryable bool) { + switch { + case errors.Is(err, errDeployInterrupted): + return "INTERRUPTED", true + case errors.Is(err, errDeployTimedOut): + return "TIMEOUT", true + } + if apiErr, ok := errors.AsType[*apierr.APIError](err); ok && apiErr.ErrorCode != "" { + return apiErr.ErrorCode, apiErr.StatusCode == http.StatusTooManyRequests || apiErr.StatusCode >= http.StatusInternalServerError + } + return "INVALID_ARGS", false +} + +// validateDeployFlags rejects flags the config's model source can't use. +func validateDeployFlags(cfg *deployConfig, opts deployOptions) error { + if opts.timeout <= 0 { + return fmt.Errorf("--timeout must be positive, got %s", opts.timeout) + } + if opts.idempotencyKey != "" { + if !cfg.registers() { + return errors.New("--idempotency-key applies only when air deploy registers a version; model.type is registered") + } + if err := validateDeployIdempotencyKey(opts.idempotencyKey, "--idempotency-key"); err != nil { + return err + } + } + if opts.retryFailedStaging && cfg.Model.Type != modelTypeCheckpoint { + return fmt.Errorf("--retry-failed-staging applies only to a checkpoint; model.type is %s", cfg.Model.Type) + } + return nil +} + +// registrationProgress describes a registration response on the spinner. +func registrationProgress(cfg *deployConfig, resp *registerModelPackageResponse) string { + state := resp.weightsState() + if state == "" { + return fmt.Sprintf("Registering version %s of %s…", resp.ModelVersion, cfg.RegisteredModel) + } + return fmt.Sprintf("Copying the weights into version %s of %s (%s)…", resp.ModelVersion, cfg.RegisteredModel, state) +} + +// serveVersion serves entity on the endpoint, showing the endpoint's state on a +// spinner when show is true. +func serveVersion(ctx context.Context, w *databricks.WorkspaceClient, name string, entity serving.ServedEntityInput, show bool) (*serving.ServingEndpointDetailed, error) { + if !show { + return serveOnEndpoint(ctx, w, name, entity, nil) + } + cmdio.LogString(ctx, fmt.Sprintf("Serving version %s of %s on endpoint %s", entity.EntityVersion, entity.EntityName, name)) + sp := cmdio.NewSpinner(ctx) + defer sp.Close() + return serveOnEndpoint(ctx, w, name, entity, func(endpoint *serving.ServingEndpointDetailed) { + if endpoint.State != nil { + sp.Update(fmt.Sprintf("Waiting for endpoint %s (%s, %s)…", name, endpoint.State.Ready, endpoint.State.ConfigUpdate)) + } + }) +} + +// deployResumeCommand is the command that continues this deployment: the same +// config (by absolute path, so it runs from any directory), overrides and +// profile, with the registration's key when it has one. The overrides are part +// of the request the key was first used with, so they must be repeated. +func deployResumeCommand(profile string, opts deployOptions, key string) string { + file := opts.file + if abs, err := filepath.Abs(file); err == nil { + file = abs + } + args := []string{deployCommandPath, "-f", shellquote.BashArg(filepath.ToSlash(file))} + for _, override := range opts.overrides { + args = append(args, "--override", shellquote.BashArg(override)) + } + if key != "" { + args = append(args, "--idempotency-key", shellquote.BashArg(key)) + } + if profile != "" { + args = append(args, "-p", shellquote.BashArg(profile)) + } + return strings.Join(args, " ") +} + +// endpointInvocationsURL is the REST URL that queries the endpoint. +func endpointInvocationsURL(host, name string) string { + return strings.TrimSuffix(host, "/") + "/serving-endpoints/" + name + "/invocations" +} diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go new file mode 100644 index 00000000000..ce232b46ff4 --- /dev/null +++ b/cmd/air/deploy_test.go @@ -0,0 +1,366 @@ +package aircmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/shellquote" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/retries" + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// withDeadline returns the test's context with a deadline, which air deploy +// always sets from --timeout. +func withDeadline(t *testing.T) context.Context { + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + t.Cleanup(cancel) + return ctx +} + +// deployServer serves registerModelPackage with responses and the serving +// endpoints API from endpoints. +func deployServer(t *testing.T, endpoints *fakeEndpoints, responses ...string) (*httptest.Server, *fakeModelServing) { + t.Helper() + fake := newFakeModelServing(t, responses...) + fake.endpoints = endpoints.ServeHTTP + return fake.start(), fake +} + +// runDeployCmd runs `air deploy` on config with the given flags. An empty srvURL +// runs it without a workspace client, as --dry-run does. +func runDeployCmd(t *testing.T, out flags.Output, buf *bytes.Buffer, srvURL, config string, flagValues ...string) error { + t.Helper() + cmd := withOutput(newDeployCommandWithPollInterval(time.Millisecond), out) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", config))) + for i := 0; i < len(flagValues); i += 2 { + require.NoError(t, cmd.Flags().Set(flagValues[i], flagValues[i+1])) + } + + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), out, nil, buf, buf, "", "")) + if srvURL != "" { + ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL)) + } + cmd.SetContext(ctx) + cmd.SetOut(buf) + return cmd.RunE(cmd, nil) +} + +func TestDeployCheckpointJSON(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServer(t, endpoints, stagingInProgress, stagingReady) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + require.NoError(t, err) + + var got struct { + Data deployResult `json:"data"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, deployResult{ + Status: "READY", + RegisteredModel: "main.models.qwen", + ModelVersion: "3", + ModelID: "m-1", + IdempotencyKey: "key-1", + Endpoint: "qwen-chat", + EndpointURL: srv.URL + "/serving-endpoints/qwen-chat/invocations", + }, got.Data) + + bodies := fake.requests() + require.Len(t, bodies, 2) + assert.Equal(t, "key-1", bodies[0].IdempotencyKey) + assert.Equal(t, bodies[0], bodies[1]) + assert.Equal(t, []string{"create"}, endpoints.writes) + served := endpoints.endpoint.Config.ServedEntities[0] + assert.Equal(t, "3", served.EntityVersion) + assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, served.EnvironmentVars) +} + +func TestDeployCheckpointText(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), stagingReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig)) + out := buf.String() + assert.Contains(t, out, "Registering a version of main.models.qwen with idempotency key ") + assert.Contains(t, out, "Version 3 of main.models.qwen is ready.") + assert.Contains(t, out, "Serving version 3 of main.models.qwen on endpoint qwen-chat") + assert.Contains(t, out, "Endpoint qwen-chat serves version 3 of main.models.qwen.\n") + assert.Contains(t, out, "Query it at: "+srv.URL+"/serving-endpoints/qwen-chat/invocations\n") + assert.NotContains(t, out, "Warning") +} + +func TestDeployWarnsWhenTheEnvironmentFailedToStage(t *testing.T) { + environmentFailed := strings.Replace(stagingReady, `"environment_artifact": {"state": "SUCCEEDED"}`, `"environment_artifact": {"state": "FAILED"}`, 1) + srv, _ := deployServer(t, newFakeEndpoints(t, nil), environmentFailed) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig)) + assert.Contains(t, buf.String(), "Warning: the version's serving environment couldn't be prepared ahead of time") +} + +func TestDeployUsesTheConfigTokenThenANewKey(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig+"idempotency_token: from-config\n")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig+"idempotency_token: from-config\n", "idempotency-key", "from-flag")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig)) + + bodies := fake.requests() + require.Len(t, bodies, 3) + assert.Equal(t, "from-config", bodies[0].IdempotencyKey) + assert.Equal(t, "from-flag", bodies[1].IdempotencyKey) + assert.Regexp(t, deployIdempotencyKeyRe(), bodies[2].IdempotencyKey) + assert.NotContains(t, []string{"from-config", "from-flag"}, bodies[2].IdempotencyKey) +} + +func TestDeployRetriesFailedStagingOnce(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress, stagingReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "retry-failed-staging", "true")) + bodies := fake.requests() + require.Len(t, bodies, 2) + assert.True(t, bodies[0].RetryFailedStaging) + assert.False(t, bodies[1].RetryFailedStaging) +} + +func TestDeployFailedStagingSaysHowToRetry(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), stagingFailed) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "copying the weights in /Volumes/main/models/weights/qwen3_8b into version 3 of main.models.qwen failed") + assert.Contains(t, err.Error(), " --idempotency-key key-1 --retry-failed-staging") +} + +func TestDeployReportsRegistrationErrors(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), "") + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + assert.ErrorContains(t, err, "registering a version of main.models.qwen with idempotency key key-1 failed: The idempotency key can't be used for this registration") +} + +func TestDeployReportsErrorsAsJSON(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), "") + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "RESOURCE_CONFLICT", got.Error.Code) + assert.Equal(t, "PERMANENT", got.Error.Kind) + assert.False(t, got.Error.Retryable) + + buf.Reset() + err = runDeployCmd(t, flags.OutputJSON, &buf, "", "registered_model: x\n", "dry-run", "true") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "INVALID_ARGS", got.Error.Code) + assert.Contains(t, got.Error.Message, "registered_model must be a Unity Catalog model name") +} + +func TestDeployChecksTheEndpointBeforeRegistering(t *testing.T) { + other := testEntity("5") + other.EntityName = "main.models.llama" + srv, fake := deployServer(t, newFakeEndpoints(t, endpointServing(other, readyState)), stagingReady) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig) + assert.ErrorContains(t, err, "endpoint qwen-chat serves version 5 of main.models.llama") + assert.Empty(t, fake.requests()) +} + +func TestDeployEndpointFailureSaysHowToRetry(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + endpoints.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed} + srv, _ := deployServer(t, endpoints, stagingReady) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "serving version 3 of main.models.qwen on endpoint qwen-chat failed: ") + assert.Contains(t, err.Error(), "To try again without registering another version, run:\n databricks air deploy -f ") + assert.Contains(t, err.Error(), " --idempotency-key key-1") +} + +func TestDeployRegisteredVersionSkipsRegistration(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServer(t, endpoints) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, `registered_model: main.models.qwen +model: {type: registered, version: 7} +endpoint: qwen-chat +compute: {workload_type: GPU_MEDIUM, workload_size: Small} +`) + require.NoError(t, err) + assert.Empty(t, fake.requests()) + assert.Equal(t, "7", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) + assert.NotContains(t, buf.String(), "idempotency_key") +} + +func TestDeployCodeSendsTheFiles(t *testing.T) { + cfg, _ := writeCodeTree(t, map[string]string{"agent.py": "import tools\n", "tools/__init__.py": "x = 1\n"}) + codeReady := `{"model_id": "m-2", "registered_model": "main.agents.support", "model_version": "1", "ready": true}` + srv, fake := deployServer(t, newFakeEndpoints(t, nil), codeReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, codeConfigAt(cfg.CodeSource.RootPath))) + bodies := fake.requests() + require.Len(t, bodies, 1) + assert.Equal(t, &modelPackageCode{ + EntryFile: "agent.py", + EntrySource: "import tools\n", + Files: []modelPackageFile{{Path: "tools/__init__.py", Content: "x = 1\n"}}, + Task: "agent/v1/responses", + Environment: modelPackageEnvironment{Runtime: "client.5", Dependencies: []string{"mlflow==3.5.0"}}, + }, bodies[0].Code) + assert.Nil(t, bodies[0].Checkpoint) +} + +func TestDeployDryRunNeedsNoWorkspace(t *testing.T) { + cmd := newDeployCommand() + require.NoError(t, cmd.Flags().Set("dry-run", "true")) + assert.NoError(t, cmd.PreRunE(cmd, nil)) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", checkpointDeployConfig, "dry-run", "true")) + assert.Contains(t, buf.String(), `Dry run: configuration for endpoint "qwen-chat" is valid; not deploying.`) + + buf.Reset() + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", checkpointDeployConfig, "dry-run", "true")) + var got struct { + Data deployResult `json:"data"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, deployResult{Status: "DRY_RUN_OK", DryRun: true}, got.Data) +} + +func TestDeployDryRunListsCodeFiles(t *testing.T) { + cfg, _ := writeCodeTree(t, map[string]string{"agent.py": "x\n", "tools/a.py": "a\n"}) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", codeConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) + assert.Contains(t, buf.String(), "Dry run: the model from code would include 2 file(s):\n agent.py\n tools/a.py\n") + + buf.Reset() + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", codeConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) + var got struct { + Data deployResult `json:"data"` + } + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, []string{"agent.py", "tools/a.py"}, got.Data.CodeFiles) +} + +// codeConfigAt is codeDeployConfig with its code at root. +func codeConfigAt(root string) string { + return strings.Replace(codeDeployConfig, "root_path: .", "root_path: "+strconv.Quote(filepath.ToSlash(root)), 1) +} + +func TestDeployFlagValidation(t *testing.T) { + registered := "registered_model: a.b.c\nmodel: {type: registered, version: 1}\nendpoint: e\ncompute: {workload_type: CPU, workload_size: Small}\n" + tests := []struct { + name string + config string + flagValues []string + wantErr string + }{ + {"key for a registered version", registered, []string{"idempotency-key", "k"}, "--idempotency-key applies only when air deploy registers a version"}, + {"bad key", checkpointDeployConfig, []string{"idempotency-key", "a/b"}, "--idempotency-key must be 1 to 128 letters"}, + {"retry for a registered version", registered, []string{"retry-failed-staging", "true"}, "--retry-failed-staging applies only to a checkpoint"}, + {"zero timeout", checkpointDeployConfig, []string{"timeout", "0s"}, "--timeout must be positive"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, "", tt.config, append(tt.flagValues, "dry-run", "true")...) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployResumeCommand(t *testing.T) { + abs, err := filepath.Abs("my deploy.yaml") + require.NoError(t, err) + opts := deployOptions{file: "my deploy.yaml", overrides: []string{"compute.workload_size=Medium", "task=llm/v1/chat"}} + assert.Equal(t, + "databricks air deploy -f "+shellquote.BashArg(filepath.ToSlash(abs))+" --override 'compute.workload_size=Medium' --override 'task=llm/v1/chat' --idempotency-key k-1 -p dev", + deployResumeCommand("dev", opts, "k-1")) + + abs, err = filepath.Abs("d.yaml") + require.NoError(t, err) + assert.Equal(t, "databricks air deploy -f "+shellquote.BashArg(filepath.ToSlash(abs)), deployResumeCommand("", deployOptions{file: "d.yaml"}, "")) +} + +func TestDeployStopped(t *testing.T) { + assert.NoError(t, deployStopped(t.Context(), errors.New("boom"))) + + canceled, cancel := context.WithCancel(t.Context()) + cancel() + // The SDK's endpoint poller reports Ctrl-C as its own timeout. + assert.ErrorIs(t, deployStopped(canceled, &retries.ErrTimedOut{}), errDeployInterrupted) + assert.ErrorIs(t, deployStopped(canceled, context.Canceled), errDeployInterrupted) + + expired, cancelExpired := context.WithTimeout(t.Context(), -time.Second) + defer cancelExpired() + assert.ErrorIs(t, deployStopped(expired, context.DeadlineExceeded), errDeployTimedOut) + assert.ErrorIs(t, deployStopped(t.Context(), &retries.ErrTimedOut{}), errDeployTimedOut) +} + +func TestDeployErrorCode(t *testing.T) { + for _, tt := range []struct { + err error + code string + retryable bool + }{ + {fmt.Errorf("x: %w", errDeployInterrupted), "INTERRUPTED", true}, + {fmt.Errorf("x: %w", errDeployTimedOut), "TIMEOUT", true}, + {fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "TEMPORARILY_UNAVAILABLE", StatusCode: 503}), "TEMPORARILY_UNAVAILABLE", true}, + {fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "INVALID_PARAMETER_VALUE", StatusCode: 400}), "INVALID_PARAMETER_VALUE", false}, + {errors.New("model: section is required"), "INVALID_ARGS", false}, + } { + code, retryable := deployErrorCode(tt.err) + assert.Equal(t, tt.code, code, tt.err.Error()) + assert.Equal(t, tt.retryable, retryable, tt.err.Error()) + } +} + +func TestDeployHelpDocumentsAField(t *testing.T) { + var out strings.Builder + cmd := newDeployCommand() + cmd.SetOut(&out) + cmd.SetArgs([]string{"-h", "config.model.type"}) + require.NoError(t, cmd.Execute()) + assert.Contains(t, out.String(), "config.model.type\n One of checkpoint") + assert.Contains(t, out.String(), "Required: yes") + + out.Reset() + cmd = newDeployCommand() + cmd.SetOut(&out) + cmd.SetArgs([]string{"-h", "config"}) + require.NoError(t, cmd.Execute()) + // The coding agent skill writes run YAML, so the deploy help leaves it out. + assert.NotContains(t, out.String(), "databricks-ai-runtime") +} diff --git a/cmd/air/deploycode.go b/cmd/air/deploycode.go new file mode 100644 index 00000000000..eb2c1ae1ee0 --- /dev/null +++ b/cmd/air/deploycode.go @@ -0,0 +1,155 @@ +package aircmd + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "unicode/utf8" +) + +// Limits of a model from code, as the server enforces them. +const ( + // maxCodePackageBytes caps the code files plus the dependency lines. The server + // also counts the small environment file it writes, so this check rejects only + // packages the server would reject too. + maxCodePackageBytes = 4 * 1024 * 1024 + // maxCodeFiles caps the files besides the entry file. + maxCodeFiles = 1000 + maxCodePathDepth = 16 +) + +// pythonCacheDir holds bytecode Python writes next to imported sources. +const pythonCacheDir = "__pycache__" + +// codePathSegmentRe matches a segment of a code file path the server can place. +var codePathSegmentRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*$`) }) + +// collectModelCode reads a model from code from code_source: model.entry and the +// other files a code_source snapshot would include, as UTF-8 text. Files git +// ignores aren't sent, and nor are hidden files and directories or __pycache__ +// directories, whose names the server can't place or which hold no source. +// configPath locates a relative code_source.root_path. +// +// The files are stored in the model version, where anyone who can read the model +// can read them. Outside a git repository no .gitignore filters them, so the user +// must list them in code_source.include_paths. +func collectModelCode(ctx context.Context, cfg *deployConfig, configPath string) (*modelPackageCode, error) { + root, err := resolveRootPath(ctx, cfg.CodeSource.RootPath, filepath.Dir(configPath)) + if err != nil { + return nil, err + } + isGitRepo := newGitRepo(root).isRepository(ctx) + if !isGitRepo && len(cfg.CodeSource.IncludePaths) == 0 { + return nil, fmt.Errorf("code_source.include_paths is required because %s isn't in a git repository: air deploy stores the files in the model version, so list the ones to send, e.g. include_paths: [%s]", filepath.ToSlash(root), *cfg.Model.Entry) + } + files, err := snapshotFiles(ctx, root, cfg.CodeSource.IncludePaths, isGitRepo) + if err != nil { + return nil, err + } + files = slices.DeleteFunc(files, skippedCodeFile) + + code := &modelPackageCode{EntryFile: *cfg.Model.Entry, Environment: cfg.servingEnvironment()} + if cfg.Task != nil { + code.Task = *cfg.Task + } + // Check the listing before reading anything, so a root_path that holds large + // data fails fast instead of being read into memory. + if err := checkCodeFiles(files, code.Environment.Dependencies); err != nil { + return nil, err + } + + foundEntry := false + for _, f := range files { + rel := filepath.ToSlash(f.rel) + content, err := os.ReadFile(filepath.Join(root, f.rel)) + if err != nil { + return nil, fmt.Errorf("failed to read code_source file %s: %w", rel, err) + } + if !utf8.Valid(content) { + return nil, fmt.Errorf("code_source file %s isn't UTF-8 text; air deploy sends source files only, so leave it out with code_source.include_paths", rel) + } + if rel == code.EntryFile { + code.EntrySource = string(content) + foundEntry = true + continue + } + code.Files = append(code.Files, modelPackageFile{Path: rel, Content: string(content)}) + } + if !foundEntry { + return nil, fmt.Errorf("model.entry %s isn't among the files code_source sends from %s; check that it exists, isn't ignored by git and is under code_source.include_paths", code.EntryFile, filepath.ToSlash(root)) + } + return code, nil +} + +// checkCodeFiles checks the files to send against the server's limits, from +// their listing alone. +func checkCodeFiles(files []snapshotFile, dependencies []string) error { + // One of the files is the entry file, which the limit doesn't count. + if len(files) > maxCodeFiles+1 { + return fmt.Errorf("code_source sends %d files, but a model from code can include at most %d besides model.entry; narrow it with code_source.include_paths", len(files), maxCodeFiles) + } + var size int64 + for _, dependency := range dependencies { + size += int64(len(dependency)) + 1 + } + for _, f := range files { + rel := filepath.ToSlash(f.rel) + if !os.FileMode(f.mode).IsRegular() { + return fmt.Errorf("code_source path %s isn't a regular file; air deploy sends the contents of files only, so replace it with a file or leave it out with code_source.include_paths", rel) + } + if err := validateCodePath(rel); err != nil { + return err + } + size += f.size + if size > maxCodePackageBytes { + return fmt.Errorf("code_source files and environment.dependencies total more than %d bytes, the most a model from code can hold; narrow code_source.include_paths", maxCodePackageBytes) + } + } + return nil +} + +// codeFilePaths lists the files of code, entry file first, as dry runs show them. +func codeFilePaths(code *modelPackageCode) []string { + paths := []string{code.EntryFile} + for _, f := range code.Files { + paths = append(paths, f.Path) + } + return paths +} + +// sentCodePath reports whether a code_source file at rel (slash-separated) is +// sent: hidden files and directories and __pycache__ directories are skipped. +func sentCodePath(rel string) bool { + segments := strings.Split(rel, "/") + directories := segments[:len(segments)-1] + return !slices.ContainsFunc(segments, isHiddenName) && !slices.Contains(directories, pythonCacheDir) +} + +// skippedCodeFile reports whether a code_source file isn't sent. +func skippedCodeFile(f snapshotFile) bool { + return !sentCodePath(filepath.ToSlash(f.rel)) +} + +// isHiddenName reports whether a file or directory name is hidden by convention. +func isHiddenName(name string) bool { + return strings.HasPrefix(name, ".") +} + +// validateCodePath checks that the server can place a code file at rel. +func validateCodePath(rel string) error { + segments := strings.Split(rel, "/") + if len(segments) <= maxCodePathDepth && !slices.ContainsFunc(segments, invalidCodePathSegment) { + return nil + } + return fmt.Errorf("code_source file %s can't be sent: a path must have at most %d parts of letters, digits, '_', '-' and '.', each starting with a letter, digit or '_'; rename it or leave it out with code_source.include_paths", rel, maxCodePathDepth) +} + +// invalidCodePathSegment reports whether the server can't place a path segment. +func invalidCodePathSegment(segment string) bool { + return !codePathSegmentRe().MatchString(segment) +} diff --git a/cmd/air/deploycode_test.go b/cmd/air/deploycode_test.go new file mode 100644 index 00000000000..c5d50fb88ce --- /dev/null +++ b/cmd/air/deploycode_test.go @@ -0,0 +1,183 @@ +package aircmd + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeCodeTree writes files (slash-separated path to content) into a new git +// repository and returns a deployment config whose code_source is its root. +func writeCodeTree(t *testing.T, files map[string]string) (*deployConfig, string) { + t.Helper() + return codeTreeConfig(t, newTestRepo(t), files) +} + +// codeTreeConfig writes files under dir and returns a deployment config whose +// code_source is dir. +func codeTreeConfig(t *testing.T, dir string, files map[string]string) (*deployConfig, string) { + t.Helper() + for rel, content := range files { + writeRepoFile(t, dir, filepath.FromSlash(rel), content) + } + entry, task := "agent.py", "agent/v1/responses" + cfg := &deployConfig{ + Model: &deployModelConfig{Type: modelTypeCode, Entry: &entry}, + Task: &task, + CodeSource: &deployCodeSourceConfig{RootPath: dir}, + Environment: &deployEnvironmentConfig{Dependencies: []string{"mlflow==3.5.0"}}, + Compute: &deployComputeConfig{WorkloadType: "CPU", WorkloadSize: "Small"}, + } + return cfg, filepath.Join(dir, "deploy.yaml") +} + +func TestCollectModelCode(t *testing.T) { + cfg, configPath := writeCodeTree(t, map[string]string{ + "agent.py": "import tools.search\n", + "tools/__init__.py": "", + "tools/search.py": "def search(): ...\n", + "tools/__pycache__/search.pyc": "\x00\xff", + ".env": "TOKEN=x\n", + ".venv/lib/site.py": "x = 1\n", + "notes/.draft.py": "y = 2\n", + "deploy.yaml": "endpoint: e\n", + "configs/prompts/system.md": "You are helpful.\n", + "configs/prompts/__init__.py.md": "not hidden\n", + }) + + code, err := collectModelCode(t.Context(), cfg, configPath) + require.NoError(t, err) + assert.Equal(t, "agent.py", code.EntryFile) + assert.Equal(t, "import tools.search\n", code.EntrySource) + assert.Equal(t, "agent/v1/responses", code.Task) + assert.Equal(t, modelPackageEnvironment{Runtime: "client.6", Dependencies: []string{"mlflow==3.5.0"}}, code.Environment) + + var paths []string + for _, f := range code.Files { + paths = append(paths, f.Path) + } + assert.ElementsMatch(t, []string{ + "configs/prompts/__init__.py.md", + "configs/prompts/system.md", + "deploy.yaml", + "tools/__init__.py", + "tools/search.py", + }, paths) +} + +func TestCollectModelCodeIsDeterministic(t *testing.T) { + cfg, configPath := writeCodeTree(t, map[string]string{ + "agent.py": "x = 1\n", "b.py": "b\n", "a/c.py": "c\n", "a.py": "a\n", + }) + first, err := collectModelCode(t.Context(), cfg, configPath) + require.NoError(t, err) + second, err := collectModelCode(t.Context(), cfg, configPath) + require.NoError(t, err) + assert.Equal(t, first, second) +} + +func TestCollectModelCodeIncludePaths(t *testing.T) { + cfg, configPath := writeCodeTree(t, map[string]string{ + "agent.py": "x = 1\n", "tools/a.py": "a\n", "data/big.csv": "1,2\n", + }) + cfg.CodeSource.IncludePaths = []string{"agent.py", "tools"} + code, err := collectModelCode(t.Context(), cfg, configPath) + require.NoError(t, err) + require.Len(t, code.Files, 1) + assert.Equal(t, "tools/a.py", code.Files[0].Path) +} + +func TestCollectModelCodeErrors(t *testing.T) { + tests := []struct { + name string + files map[string]string + include []string + wantErr string + }{ + {"missing entry", map[string]string{"main.py": "x\n"}, nil, "model.entry agent.py isn't among the files code_source sends"}, + {"entry left out", map[string]string{"agent.py": "x\n", "tools/a.py": "a\n"}, []string{"tools"}, "model.entry agent.py isn't among the files code_source sends"}, + {"binary file", map[string]string{"agent.py": "x\n", "model.bin": "\xff\xfe\x00"}, nil, "code_source file model.bin isn't UTF-8 text"}, + {"unplaceable name", map[string]string{"agent.py": "x\n", "my tools/a.py": "a\n"}, nil, "code_source file my tools/a.py can't be sent"}, + {"too deep", map[string]string{"agent.py": "x\n", strings.Repeat("d/", maxCodePathDepth) + "a.py": "a\n"}, nil, "can't be sent: a path must have at most 16 parts"}, + {"too large", map[string]string{"agent.py": "x\n", "blob.txt": strings.Repeat("a", maxCodePackageBytes)}, nil, "total more than 4194304 bytes"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg, configPath := writeCodeTree(t, tt.files) + cfg.CodeSource.IncludePaths = tt.include + _, err := collectModelCode(t.Context(), cfg, configPath) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestCollectModelCodeLeavesOutIgnoredFiles(t *testing.T) { + cfg, configPath := writeCodeTree(t, map[string]string{ + "agent.py": "x\n", "credentials.json": "{}\n", "tools/a.py": "a\n", + }) + require.NoError(t, os.WriteFile(filepath.Join(cfg.CodeSource.RootPath, ".gitignore"), []byte("credentials.json\n"), 0o600)) + code, err := collectModelCode(t.Context(), cfg, configPath) + require.NoError(t, err) + assert.Equal(t, []string{"agent.py", "tools/a.py"}, codeFilePaths(code)) +} + +func TestCollectModelCodeOutsideGitNeedsIncludePaths(t *testing.T) { + cfg, configPath := codeTreeConfig(t, t.TempDir(), map[string]string{"agent.py": "x\n", "credentials.json": "{}\n"}) + _, err := collectModelCode(t.Context(), cfg, configPath) + assert.ErrorContains(t, err, "code_source.include_paths is required because") + assert.ErrorContains(t, err, "isn't in a git repository") + + cfg.CodeSource.IncludePaths = []string{"agent.py"} + code, err := collectModelCode(t.Context(), cfg, configPath) + require.NoError(t, err) + assert.Equal(t, []string{"agent.py"}, codeFilePaths(code)) +} + +func TestCollectModelCodeChecksSizesBeforeReading(t *testing.T) { + // Read in full, this file would fail as binary; the size check must reject it + // from its listing first. + cfg, configPath := writeCodeTree(t, map[string]string{ + "agent.py": "x\n", "data.bin": strings.Repeat("\xff", maxCodePackageBytes+1), + }) + _, err := collectModelCode(t.Context(), cfg, configPath) + assert.ErrorContains(t, err, "total more than 4194304 bytes") +} + +func TestCollectModelCodeRejectsSymlink(t *testing.T) { + cfg, configPath := writeCodeTree(t, map[string]string{"agent.py": "x\n", "real.py": "r\n"}) + dir := filepath.Dir(configPath) + if err := os.Symlink("real.py", filepath.Join(dir, "link.py")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + _, err := collectModelCode(t.Context(), cfg, configPath) + assert.ErrorContains(t, err, "code_source path link.py isn't a regular file") +} + +func TestCollectModelCodeTooManyFiles(t *testing.T) { + files := map[string]string{"agent.py": "x\n"} + for i := range maxCodeFiles + 1 { + files[fmt.Sprintf("pkg/m%d.py", i)] = "" + } + cfg, configPath := writeCodeTree(t, files) + _, err := collectModelCode(t.Context(), cfg, configPath) + assert.ErrorContains(t, err, "code_source sends 1002 files, but a model from code can include at most 1000 besides model.entry") +} + +func TestSentCodePath(t *testing.T) { + for rel, want := range map[string]bool{ + "agent.py": true, + "tools/search.py": true, + ".env": false, + "tools/.cache/x.py": false, + "__pycache__/a.pyc": false, + "tools/__pycache__/a.p": false, + "__pycache__": true, + } { + assert.Equal(t, want, sentCodePath(rel), rel) + } +} diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go new file mode 100644 index 00000000000..4b035cd616e --- /dev/null +++ b/cmd/air/deployconfig.go @@ -0,0 +1,387 @@ +package aircmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "maps" + "reflect" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "unicode/utf8" + + "github.com/databricks/databricks-sdk-go/service/serving" + "go.yaml.in/yaml/v3" +) + +// This file defines the `air deploy` YAML schema and its structural validation. +// Fields shared with `air run` (environment, code_source, env_variables, secrets, +// idempotency_token) keep their meaning; the server validates what needs the +// workspace, such as task names, runtimes and dependencies. + +// Model sources, as model.type. +const ( + modelTypeCheckpoint = "checkpoint" + modelTypeCode = "code" + modelTypeSavedModelDir = "saved_model_dir" + modelTypeRegistered = "registered" +) + +// modelTypes lists model.type values in the order the help and errors name them. +var modelTypes = []string{modelTypeCheckpoint, modelTypeCode, modelTypeSavedModelDir, modelTypeRegistered} + +// checkpointTasks are the tasks a checkpoint can serve, as the registration API +// documents them; models from code serve agent tasks too. +var checkpointTasks = []string{"llm/v1/chat", "llm/v1/completions", "llm/v1/embeddings"} + +// deployCommandPath is the command whose -h documents deployConfig. +const deployCommandPath = "databricks air deploy" + +// defaultServingEnvironmentVersion is the serverless environment version a model +// is served on when environment.version is unset, as for `air run`. +const defaultServingEnvironmentVersion = 6 + +// Limits the server enforces on a registration, checked here to fail early. +const ( + maxServingCommandChars = 32 * 1024 + maxVolumePathChars = 1024 + maxDependencies = 1000 +) + +// cpuWorkloadTypePrefix starts the name of every CPU serving workload type (CPU, +// CPU_MEDIUM, CPU_LARGE); the others are GPU workload types. +const cpuWorkloadTypePrefix = "CPU" + +// registeredModelRe matches .. with the characters the +// server accepts in each part. +var registeredModelRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile("^[^\\s./`]+\\.[^\\s./`]+\\.[^\\s./`]+$") }) + +// deployIdempotencyKeyRe matches the registration keys the server records. +var deployIdempotencyKeyRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9._-]{1,128}$`) }) + +// modelVersionRe matches a Unity Catalog model version number. +var modelVersionRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[1-9][0-9]*$`) }) + +// entryFileRe matches the name of a model from code's entry file, which the server +// places at the model's root. +var entryFileRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$`) }) + +// deployConfig is the `air deploy` YAML schema: a model to register in Unity +// Catalog and the serving endpoint that serves it. +type deployConfig struct { + RegisteredModel string `yaml:"registered_model" help:"Unity Catalog model, as ... air deploy registers a new version of it, creating the model if it doesn't exist, unless model.type is registered." required:"yes" group:"Model"` + Model *deployModelConfig `yaml:"model" help:"What to serve: a checkpoint, a model from code, a model directory saved by MLflow, or an existing version." required:"yes" group:"Model"` + Task *string `yaml:"task" help:"The model's task as MLflow names it: llm/v1/chat, llm/v1/completions or llm/v1/embeddings for a checkpoint, or e.g. agent/v1/responses for a model from code. A model from code without a task has no signature." required:"when model.type is checkpoint" group:"Model"` + Command *string `yaml:"command" help:"Shell command that starts the model server of a checkpoint, at most 32,768 characters. It runs from the directory where model.path appears under its own name." required:"when model.type is checkpoint" group:"Model"` + Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment version to serve on and the packages to install, for a checkpoint or a model from code." group:"Model"` + CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of a model from code, sent with the registration and stored in the model version, where anyone who can read the model can read it. Hidden files and directories (names starting with '.') and __pycache__ directories aren't sent; run with --dry-run to list the files that are." required:"when model.type is code" group:"Model"` + + Endpoint string `yaml:"endpoint" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` + Compute *deployComputeConfig `yaml:"compute" help:"Serving compute and how the endpoint scales." required:"yes" group:"Serving"` + EnvVariables map[string]string `yaml:"env_variables" help:"Plain environment variables of the served model, as NAME: value. A name here cannot also appear in secrets." group:"Serving"` + Secrets map[string]string `yaml:"secrets" help:"Environment variables of the served model sourced from secrets, as NAME: scope/key." group:"Serving"` + + IdempotencyToken *string `yaml:"idempotency_token" help:"Key of the registration: 1 to 128 letters, digits, '.', '_' or '-'. Running air deploy again with the same key and unchanged inputs continues that registration instead of registering another version. Defaults to a new key, which air deploy prints." group:"Registration"` + MLflowExperimentPath *string `yaml:"mlflow_experiment_path" help:"Workspace path of the MLflow experiment that records the registered model (default: /Users//air-deployments)." group:"Registration"` +} + +// deployModelConfig is the `model` block: where the served model comes from. +type deployModelConfig struct { + Type string `yaml:"type" help:"One of checkpoint (weights in a volume, started by command), code (a Python file from code_source), saved_model_dir (a model directory saved by MLflow in a volume) or registered (an existing version of registered_model)." required:"yes"` + Path *string `yaml:"path" help:"Unity Catalog volume directory, as /Volumes////, holding a checkpoint's weights or a model saved by MLflow. Model Serving copies it into the version." required:"when type is checkpoint or saved_model_dir"` + Entry *string `yaml:"entry" help:"Python file directly under code_source.root_path that builds the model and calls mlflow.models.set_model, e.g. agent.py." required:"when type is code"` + Version *string `yaml:"version" help:"Version of registered_model to serve, e.g. 3." required:"when type is registered"` +} + +// deployEnvironmentConfig is the `environment` block of a deployment. +type deployEnvironmentConfig struct { + Version *int `yaml:"version" help:"Serverless environment version to serve on, e.g. 6 (default: 6). The model uses its Python version."` + Dependencies []string `yaml:"dependencies" help:"pip requirement lines to install, e.g. einops==0.8.0, at most 1,000. A model from code must list mlflow or mlflow-skinny."` +} + +// deployCodeSourceConfig is the `code_source` block of a model from code. +type deployCodeSourceConfig struct { + RootPath string `yaml:"root_path" help:"Directory holding the model's code. A relative path resolves against the YAML file's directory." required:"yes"` + IncludePaths []string `yaml:"include_paths" help:"Send only these paths, relative to root_path and without \"..\". Omit to send every file that git doesn't ignore; required when root_path isn't in a git repository."` +} + +// deployComputeConfig is the `compute` block: the served entity's workload. +type deployComputeConfig struct { + WorkloadType string `yaml:"workload_type" help:"Serving compute, e.g. CPU, GPU_SMALL, GPU_MEDIUM or GPU_LARGE. A checkpoint is served on a GPU workload type." required:"yes"` + WorkloadSize string `yaml:"workload_size" help:"How far the endpoint scales: Small, Medium or Large." required:"yes"` + ScaleToZero bool `yaml:"scale_to_zero" help:"Whether the endpoint scales to zero when idle (default: false)."` +} + +// sourceFields lists, per model.type, the optional fields that apply to it, in +// addition to the required ones in requiredSourceFields. Every other source field +// is rejected, so a field the source can't use is never silently dropped. +var sourceFields = map[string][]string{ + modelTypeCheckpoint: {"environment", "idempotency_token", "mlflow_experiment_path"}, + modelTypeCode: {"task", "environment", "idempotency_token", "mlflow_experiment_path"}, + modelTypeSavedModelDir: {"idempotency_token", "mlflow_experiment_path"}, + modelTypeRegistered: {}, +} + +// requiredSourceFields lists the fields each model.type requires. +var requiredSourceFields = map[string][]string{ + modelTypeCheckpoint: {"model.path", "task", "command"}, + modelTypeCode: {"model.entry", "code_source"}, + modelTypeSavedModelDir: {"model.path"}, + modelTypeRegistered: {"model.version"}, +} + +// validate runs structural validation over the whole config, returning the +// first failure. Fields are checked in declaration order to keep errors stable. +func (c *deployConfig) validate() error { + if !registeredModelRe().MatchString(c.RegisteredModel) { + return fmt.Errorf("registered_model must be a Unity Catalog model name as .., got %q", c.RegisteredModel) + } + if c.Model == nil { + return errors.New("model: section is required") + } + if err := c.validateSource(); err != nil { + return err + } + if strings.TrimSpace(c.Endpoint) == "" { + return errors.New("endpoint is required") + } + if c.Compute == nil { + return errors.New("compute: section is required") + } + if workloadTypes := servingWorkloadTypes(); !slices.Contains(workloadTypes, c.Compute.WorkloadType) { + return fmt.Errorf("compute.workload_type must be one of %s, got %q", strings.Join(workloadTypes, ", "), c.Compute.WorkloadType) + } + if strings.TrimSpace(c.Compute.WorkloadSize) == "" { + return errors.New("compute.workload_size is required") + } + if c.Model.Type == modelTypeCheckpoint && !c.servesOnGPU() { + return fmt.Errorf("a checkpoint is served on GPU; set compute.workload_type to a GPU workload type such as GPU_MEDIUM, got %q", c.Compute.WorkloadType) + } + if err := validateEnvironmentVariables(c.EnvVariables, c.Secrets); err != nil { + return err + } + if c.IdempotencyToken != nil { + if err := validateDeployIdempotencyKey(*c.IdempotencyToken, "idempotency_token"); err != nil { + return err + } + } + if c.MLflowExperimentPath != nil && !strings.HasPrefix(*c.MLflowExperimentPath, "/") { + return fmt.Errorf("mlflow_experiment_path must be an absolute workspace path such as /Users//air-deployments, got %q", *c.MLflowExperimentPath) + } + return nil +} + +// validateSource checks model.type and the fields that describe the source. +func (c *deployConfig) validateSource() error { + m := c.Model + if !slices.Contains(modelTypes, m.Type) { + return fmt.Errorf("model.type must be one of %s, got %q", strings.Join(modelTypes, ", "), m.Type) + } + + set := map[string]bool{ + "model.path": m.Path != nil, + "model.entry": m.Entry != nil, + "model.version": m.Version != nil, + "task": c.Task != nil, + "command": c.Command != nil, + "environment": c.Environment != nil, + "code_source": c.CodeSource != nil, + "idempotency_token": c.IdempotencyToken != nil, + "mlflow_experiment_path": c.MLflowExperimentPath != nil, + } + required := requiredSourceFields[m.Type] + for _, name := range required { + if !set[name] { + return fmt.Errorf("%s is required when model.type is %s", name, m.Type) + } + } + for _, name := range slices.Sorted(maps.Keys(set)) { + if set[name] && !slices.Contains(required, name) && !slices.Contains(sourceFields[m.Type], name) { + return fmt.Errorf("%s can't be set when model.type is %s", name, m.Type) + } + } + + if m.Path != nil { + v := *m.Path + if !strings.HasPrefix(v, "/Volumes/") || utf8.RuneCountInString(v) > maxVolumePathChars { + return fmt.Errorf("model.path must be a Unity Catalog volume directory as /Volumes////, of at most %d characters, got %q", maxVolumePathChars, v) + } + } + if m.Entry != nil && !entryFileRe().MatchString(*m.Entry) { + return fmt.Errorf("model.entry must name a .py file directly under code_source.root_path, e.g. agent.py, got %q", *m.Entry) + } + if m.Version != nil && !modelVersionRe().MatchString(*m.Version) { + return fmt.Errorf("model.version must be a model version number such as 3, got %q", *m.Version) + } + if c.Task != nil && strings.TrimSpace(*c.Task) == "" { + return errors.New("task cannot be empty") + } + if m.Type == modelTypeCheckpoint && !slices.Contains(checkpointTasks, *c.Task) { + return fmt.Errorf("task of a checkpoint must be one of %s, got %q", strings.Join(checkpointTasks, ", "), *c.Task) + } + if c.Command != nil { + if err := validateCommand(*c.Command); err != nil { + return err + } + if n := utf8.RuneCountInString(*c.Command); n > maxServingCommandChars { + return fmt.Errorf("command must be at most %d characters, got %d", maxServingCommandChars, n) + } + } + if c.Environment != nil { + if err := c.Environment.validate(); err != nil { + return err + } + } + if c.CodeSource != nil { + snapshot := snapshotSourceConfig{RootPath: c.CodeSource.RootPath, IncludePaths: c.CodeSource.IncludePaths} + if err := snapshot.validate("code_source"); err != nil { + return err + } + } + return nil +} + +// validate checks the environment version and dependency lines. +func (e *deployEnvironmentConfig) validate() error { + if e.Version != nil && *e.Version < 1 { + return fmt.Errorf("environment.version must be a serverless environment version such as 6, got %d", *e.Version) + } + if len(e.Dependencies) > maxDependencies { + return fmt.Errorf("environment.dependencies can list at most %d packages, got %d", maxDependencies, len(e.Dependencies)) + } + for i, dependency := range e.Dependencies { + if strings.TrimSpace(dependency) == "" { + return fmt.Errorf("environment.dependencies[%d] cannot be empty", i) + } + } + return nil +} + +// validateDeployIdempotencyKey checks a registration key from source (the YAML +// field or the flag) against the server's rule. +func validateDeployIdempotencyKey(key, source string) error { + if !deployIdempotencyKeyRe().MatchString(key) { + return fmt.Errorf("%s must be 1 to 128 letters, digits, '.', '_' or '-', got %q", source, key) + } + return nil +} + +// servingWorkloadTypes lists the served entity workload types, sorted. +func servingWorkloadTypes() []string { + var workloadType serving.ServingModelWorkloadType + var types []string + for _, t := range workloadType.Values() { + types = append(types, string(t)) + } + slices.Sort(types) + return types +} + +// registers reports whether air deploy registers a new version, rather than +// serving an existing one. +func (c *deployConfig) registers() bool { + return c.Model.Type != modelTypeRegistered +} + +// servesOnGPU reports whether compute.workload_type is a GPU workload type. +func (c *deployConfig) servesOnGPU() bool { + return !strings.HasPrefix(c.Compute.WorkloadType, cpuWorkloadTypePrefix) +} + +// servingRuntime is the serverless runtime the model is served on, as the server +// names it: client., with a -gpu suffix on GPU compute. +func (c *deployConfig) servingRuntime() string { + version := defaultServingEnvironmentVersion + if c.Environment != nil && c.Environment.Version != nil { + version = *c.Environment.Version + } + runtime := "client." + strconv.Itoa(version) + if c.servesOnGPU() { + runtime += "-gpu" + } + return runtime +} + +// servingEnvironment is the registration's runtime and dependencies. +func (c *deployConfig) servingEnvironment() modelPackageEnvironment { + environment := modelPackageEnvironment{Runtime: c.servingRuntime()} + if c.Environment != nil { + environment.Dependencies = c.Environment.Dependencies + } + return environment +} + +// servedEnvironmentVars merges env_variables and secrets into the served +// entity's environment variables, with each secret as {{secrets/scope/key}}. +func (c *deployConfig) servedEnvironmentVars() map[string]string { + if len(c.EnvVariables) == 0 && len(c.Secrets) == 0 { + return nil + } + vars := maps.Clone(c.EnvVariables) + if vars == nil { + vars = map[string]string{} + } + for name, ref := range c.Secrets { + vars[name] = "{{secrets/" + ref + "}}" + } + return vars +} + +// deploySchema is the reflection walk over deployConfig, shared by +// `air deploy -h config.` and --override path validation. +func deploySchema() configField { + return configField{ + path: configHelpRoot, + help: `Deployment YAML schema. For details on a field, pass its path to -h, for example: + + databricks air deploy -h config.model.type + + Minimal example, a checkpoint served by vLLM: + registered_model: main.models.qwen + model: + type: checkpoint + path: /Volumes/main/models/weights/qwen3_8b + task: llm/v1/chat + command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --served-model-name base --host 0.0.0.0 --port 8080 + endpoint: qwen-chat + compute: + workload_type: GPU_MEDIUM + workload_size: Small`, + children: describeStruct(reflect.TypeFor[deployConfig](), configHelpRoot), + } +} + +// loadDeployConfig reads a deployment YAML, applies --override entries, decodes +// it with unknown keys rejected and validates it. ctx is used only to log applied +// overrides. +func loadDeployConfig(ctx context.Context, path string, overrides []string) (*deployConfig, error) { + var ( + document *yaml.Node + validationYAML []byte + err error + ) + if len(overrides) == 0 { + document, validationYAML, err = readRunConfigDocument(path) + } else { + document, validationYAML, err = readConfigWithOverrides(ctx, path, overrides, deploySchema()) + } + if err != nil { + return nil, err + } + if len(document.Content) == 0 { + return nil, fmt.Errorf("config %s is empty", path) + } + + cfg, err := decodeConfigReader[deployConfig](bytes.NewReader(validationYAML), path) + if err != nil { + return nil, err + } + if err := cfg.validate(); err != nil { + return nil, err + } + return cfg, nil +} diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go new file mode 100644 index 00000000000..80f64e76012 --- /dev/null +++ b/cmd/air/deployconfig_test.go @@ -0,0 +1,306 @@ +package aircmd + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const checkpointDeployConfig = `registered_model: main.models.qwen +model: + type: checkpoint + path: /Volumes/main/models/weights/qwen3_8b +task: llm/v1/chat +command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080 +environment: + dependencies: [einops==0.8.0] +endpoint: qwen-chat +compute: + workload_type: GPU_MEDIUM + workload_size: Small +secrets: + HF_TOKEN: llm/hf_token +env_variables: + LOG_LEVEL: info +` + +const codeDeployConfig = `registered_model: main.agents.support +model: + type: code + entry: agent.py +task: agent/v1/responses +code_source: + root_path: . +environment: + version: 5 + dependencies: [mlflow==3.5.0] +endpoint: support-agent +compute: + workload_type: CPU + workload_size: Small + scale_to_zero: true +` + +func loadDeployConfigText(t *testing.T, text string, overrides ...string) (*deployConfig, error) { + t.Helper() + return loadDeployConfig(t.Context(), writeConfigFile(t, "deploy.yaml", text), overrides) +} + +func TestLoadDeployConfigCheckpoint(t *testing.T) { + cfg, err := loadDeployConfigText(t, checkpointDeployConfig) + require.NoError(t, err) + assert.True(t, cfg.registers()) + assert.Equal(t, "client.6-gpu", cfg.servingRuntime()) + assert.Equal(t, modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}}, cfg.servingEnvironment()) + assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, cfg.servedEnvironmentVars()) + + req := cfg.registrationRequest("key-1", nil) + assert.Equal(t, registerModelPackageRequest{ + RegisteredModel: "main.models.qwen", + IdempotencyKey: "key-1", + Checkpoint: &modelPackageCheckpoint{ + VolumePath: "/Volumes/main/models/weights/qwen3_8b", + ServingCommand: "python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", + Task: "llm/v1/chat", + Environment: modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}}, + }, + }, req) +} + +func TestLoadDeployConfigCodeOnCPU(t *testing.T) { + cfg, err := loadDeployConfigText(t, codeDeployConfig) + require.NoError(t, err) + assert.Equal(t, "client.5", cfg.servingRuntime()) + assert.Nil(t, cfg.servedEnvironmentVars()) + + entity := cfg.servedEntity("4") + assert.Equal(t, "main.agents.support", entity.EntityName) + assert.Equal(t, "4", entity.EntityVersion) + assert.True(t, entity.ScaleToZeroEnabled) + assert.Contains(t, entity.ForceSendFields, "ScaleToZeroEnabled") +} + +func TestLoadDeployConfigRegisteredVersion(t *testing.T) { + cfg, err := loadDeployConfigText(t, `registered_model: main.ml.churn +model: {type: registered, version: 7} +endpoint: churn +compute: {workload_type: CPU, workload_size: Small} +`) + require.NoError(t, err) + assert.False(t, cfg.registers()) + assert.Equal(t, "7", *cfg.Model.Version) +} + +func TestLoadDeployConfigSavedModelDir(t *testing.T) { + cfg, err := loadDeployConfigText(t, `registered_model: main.ml.churn +model: {type: saved_model_dir, path: /Volumes/main/ml/runs/model} +mlflow_experiment_path: /Shared/churn +endpoint: churn +compute: {workload_type: CPU, workload_size: Small} +`) + require.NoError(t, err) + assert.Equal(t, registerModelPackageRequest{ + RegisteredModel: "main.ml.churn", + IdempotencyKey: "k", + SavedModel: &modelPackageSavedModel{VolumePath: "/Volumes/main/ml/runs/model"}, + ExperimentPath: "/Shared/churn", + }, cfg.registrationRequest("k", nil)) +} + +func TestLoadDeployConfigOverride(t *testing.T) { + cfg, err := loadDeployConfigText(t, checkpointDeployConfig, "compute.workload_size=Medium", "environment.version=5") + require.NoError(t, err) + assert.Equal(t, "Medium", cfg.Compute.WorkloadSize) + assert.Equal(t, "client.5-gpu", cfg.servingRuntime()) +} + +func TestLoadDeployConfigRejectsUnknownOverride(t *testing.T) { + _, err := loadDeployConfigText(t, checkpointDeployConfig, "compute.accelerator_type=GPU_1xH100") + assert.ErrorContains(t, err, `invalid --override "compute.accelerator_type": "accelerator_type" is not a known field; available fields are: scale_to_zero, workload_size, workload_type`) +} + +func TestLoadDeployConfigRejectsUnknownField(t *testing.T) { + _, err := loadDeployConfigText(t, checkpointDeployConfig+"experiment_name: x\n") + assert.ErrorContains(t, err, "field experiment_name not found") +} + +func TestLoadDeployConfigRejectsEmptyFile(t *testing.T) { + _, err := loadDeployConfigText(t, "") + assert.ErrorContains(t, err, "is empty") +} + +func TestDeployConfigValidation(t *testing.T) { + tests := []struct { + name string + edit func(string) string + wantErr string + }{ + {"bad model name", replaceFirst("main.models.qwen", "qwen"), "registered_model must be a Unity Catalog model name"}, + {"backticked model name", replaceFirst("main.models.qwen", "main.models.`qwen`"), "registered_model must be a Unity Catalog model name"}, + {"unknown type", replaceFirst("type: checkpoint", "type: pickle"), `model.type must be one of checkpoint, code, saved_model_dir, registered, got "pickle"`}, + {"checkpoint without task", replaceFirst("task: llm/v1/chat\n", ""), "task is required when model.type is checkpoint"}, + {"checkpoint without command", dropLine("command:"), "command is required when model.type is checkpoint"}, + {"checkpoint with entry", replaceFirst(" type: checkpoint\n", " type: checkpoint\n entry: a.py\n"), "model.entry can't be set when model.type is checkpoint"}, + {"checkpoint with code_source", appendText("code_source: {root_path: .}\n"), "code_source can't be set when model.type is checkpoint"}, + {"path outside volumes", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/dbfs/weights"), "model.path must be a Unity Catalog volume directory"}, + {"path too long", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("a", 1100)), "model.path must be a Unity Catalog volume directory"}, + {"empty command", replaceFirst("command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", `command: " "`), "command cannot be empty"}, + {"command too long", replaceFirst("command: python", "command: python "+strings.Repeat("x", maxServingCommandChars)), "command must be at most 32768 characters"}, + {"path too long in characters", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1020)), "model.path must be a Unity Catalog volume directory"}, + {"checkpoint on CPU", replaceFirst("GPU_MEDIUM", "CPU"), "a checkpoint is served on GPU"}, + {"unknown workload type", replaceFirst("GPU_MEDIUM", "GPU_MEDUIM"), `compute.workload_type must be one of CPU, CPU_LARGE, CPU_MEDIUM, GPU_LARGE, GPU_MEDIUM, GPU_SMALL, GPU_XLARGE, GPU_XLARGE_8, MULTIGPU_MEDIUM, got "GPU_MEDUIM"`}, + {"lowercase workload type", replaceFirst("GPU_MEDIUM", "gpu_medium"), "compute.workload_type must be one of"}, + {"agent task on a checkpoint", replaceFirst("task: llm/v1/chat", "task: agent/v1/responses"), `task of a checkpoint must be one of llm/v1/chat, llm/v1/completions, llm/v1/embeddings, got "agent/v1/responses"`}, + {"zero environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: 0"), "environment.version must be a serverless environment version"}, + {"empty dependency", replaceFirst("[einops==0.8.0]", `[""]`), "environment.dependencies[0] cannot be empty"}, + {"no endpoint", dropLine("endpoint:"), "endpoint is required"}, + {"no compute", dropBlock("compute:"), "compute: section is required"}, + {"no workload size", dropLine(" workload_size:"), "compute.workload_size is required"}, + {"secret without key", replaceFirst("llm/hf_token", "llm"), "expected format 'scope/key'"}, + {"env var and secret", replaceFirst("LOG_LEVEL: info", "HF_TOKEN: x"), `"HF_TOKEN" is set in both env_variables and secrets`}, + {"bad idempotency token", appendText("idempotency_token: a b\n"), "idempotency_token must be 1 to 128 letters"}, + {"long idempotency token", appendText("idempotency_token: " + strings.Repeat("k", 129) + "\n"), "idempotency_token must be 1 to 128 letters"}, + {"relative experiment", appendText("mlflow_experiment_path: Users/me/x\n"), "mlflow_experiment_path must be an absolute workspace path"}, + {"no model", dropBlock("model:"), "model: section is required"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, tt.edit(checkpointDeployConfig)) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployConfigSourceFields(t *testing.T) { + base := "endpoint: e\ncompute: {workload_type: CPU, workload_size: Small}\nregistered_model: a.b.c\n" + tests := []struct { + name string + source string + wantErr string + }{ + {"code without entry", "model: {type: code}\ncode_source: {root_path: .}\n", "model.entry is required when model.type is code"}, + {"code without code_source", "model: {type: code, entry: agent.py}\n", "code_source is required when model.type is code"}, + {"code with command", "model: {type: code, entry: agent.py}\ncode_source: {root_path: .}\ncommand: x\n", "command can't be set when model.type is code"}, + {"code entry in a directory", "model: {type: code, entry: src/agent.py}\ncode_source: {root_path: .}\n", "model.entry must name a .py file directly under code_source.root_path"}, + {"code entry not python", "model: {type: code, entry: agent.ipynb}\ncode_source: {root_path: .}\n", "model.entry must name a .py file"}, + {"code include traversal", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [../x]}\n", "code_source.include_paths cannot contain '..' traversal"}, + {"saved model with task", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\ntask: llm/v1/chat\n", "task can't be set when model.type is saved_model_dir"}, + {"saved model with environment", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\nenvironment: {version: 6}\n", "environment can't be set when model.type is saved_model_dir"}, + {"registered without version", "model: {type: registered}\n", "model.version is required when model.type is registered"}, + {"registered with path", "model: {type: registered, version: 3, path: /Volumes/a/b/c/m}\n", "model.path can't be set when model.type is registered"}, + {"registered with token", "model: {type: registered, version: 3}\nidempotency_token: k\n", "idempotency_token can't be set when model.type is registered"}, + {"registered bad version", "model: {type: registered, version: v3}\n", "model.version must be a model version number"}, + {"registered version zero", "model: {type: registered, version: 0}\n", "model.version must be a model version number"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, base+tt.source) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestDeployConfigCountsCharactersNotBytes(t *testing.T) { + // 1,000 two-byte characters fit the 1,024-character cap on a volume path. + _, err := loadDeployConfigText(t, replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1000))(checkpointDeployConfig)) + assert.NoError(t, err) +} + +func TestDeployConfigServingRuntimeByWorkload(t *testing.T) { + for workloadType, want := range map[string]string{ + "CPU": "client.6", + "CPU_LARGE": "client.6", + "GPU_SMALL": "client.6-gpu", + "MULTIGPU_MEDIUM": "client.6-gpu", + } { + cfg := deployConfig{Compute: &deployComputeConfig{WorkloadType: workloadType}} + assert.Equal(t, want, cfg.servingRuntime(), workloadType) + } +} + +func TestDeploySchemaHelp(t *testing.T) { + root, err := resolveSchemaField(deploySchema(), "config") + require.NoError(t, err) + var out strings.Builder + renderConfigField(&out, deployCommandPath, root, nil) + help := out.String() + assert.Contains(t, help, " databricks air deploy -h config.model.type\n") + assert.Contains(t, help, " Model:\n") + assert.Contains(t, help, " Serving:\n") + assert.Contains(t, help, " Registration:\n") + assert.Contains(t, help, `For details on a field, run "databricks air deploy -h config.".`) + + field, err := resolveSchemaField(deploySchema(), "model.type") + require.NoError(t, err) + assert.Equal(t, "yes", field.required) + + _, err = resolveSchemaField(deploySchema(), "model.typ") + assert.ErrorContains(t, err, `did you mean "type"?`) + + _, err = resolveSchemaField(deploySchema(), "secrets.HF_TOKEN") + assert.ErrorContains(t, err, "holds user-defined keys") +} + +// Every source field named in sourceFields and requiredSourceFields must be a +// schema path, so the per-source checks can't drift from the YAML. +func TestDeploySourceFieldsAreSchemaPaths(t *testing.T) { + for _, table := range []map[string][]string{sourceFields, requiredSourceFields} { + for _, fields := range table { + for _, f := range fields { + _, err := resolveSchemaField(deploySchema(), f) + assert.NoError(t, err, f) + } + } + } + for _, modelType := range modelTypes { + assert.Contains(t, requiredSourceFields, modelType) + assert.Contains(t, sourceFields, modelType) + } +} + +func replaceFirst(old, with string) func(string) string { + return func(s string) string { + if !strings.Contains(s, old) { + panic("fixture does not contain " + old) + } + return strings.Replace(s, old, with, 1) + } +} + +func appendText(extra string) func(string) string { + return func(s string) string { return s + extra } +} + +// dropLine removes the first line starting with prefix. +func dropLine(prefix string) func(string) string { + return func(s string) string { + lines := strings.SplitAfter(s, "\n") + for i, line := range lines { + if strings.HasPrefix(line, prefix) { + return strings.Join(append(lines[:i:i], lines[i+1:]...), "") + } + } + panic("fixture has no line starting with " + prefix) + } +} + +// dropBlock removes the top-level key starting with prefix and its indented lines. +func dropBlock(prefix string) func(string) string { + return func(s string) string { + var kept []string + dropping := false + for _, line := range strings.SplitAfter(s, "\n") { + switch { + case strings.HasPrefix(line, prefix): + dropping = true + case dropping && strings.HasPrefix(line, " "): + default: + dropping = false + kept = append(kept, line) + } + } + return strings.Join(kept, "") + } +} diff --git a/cmd/air/deployendpoint.go b/cmd/air/deployendpoint.go new file mode 100644 index 00000000000..7a5d4d5f90a --- /dev/null +++ b/cmd/air/deployendpoint.go @@ -0,0 +1,186 @@ +package aircmd + +import ( + "context" + "errors" + "fmt" + "maps" + "strings" + "time" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/serving" +) + +// servedEntity is the one served entity air deploy puts on the endpoint. +func (c *deployConfig) servedEntity(version string) serving.ServedEntityInput { + return serving.ServedEntityInput{ + EntityName: c.RegisteredModel, + EntityVersion: version, + WorkloadType: serving.ServingModelWorkloadType(c.Compute.WorkloadType), + WorkloadSize: c.Compute.WorkloadSize, + ScaleToZeroEnabled: c.Compute.ScaleToZero, + EnvironmentVars: c.servedEnvironmentVars(), + // scale_to_zero: false is the user's choice, so send it rather than leave it + // to the server's default. + ForceSendFields: []string{"ScaleToZeroEnabled"}, + } +} + +// getEndpoint returns the endpoint named name, or nil if it doesn't exist. +func getEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string) (*serving.ServingEndpointDetailed, error) { + endpoint, err := w.ServingEndpoints.GetByName(ctx, name) + if errors.Is(err, apierr.ErrResourceDoesNotExist) { + return nil, nil + } + return endpoint, err +} + +// checkEndpoint fails when the endpoint named name, if it exists, serves +// something air deploy won't replace with a version of model. air deploy checks +// it before registering, so a wrong endpoint name fails before any weights are +// copied. +func checkEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name, model string) error { + endpoint, err := getEndpoint(ctx, w, name) + if err != nil || endpoint == nil { + return err + } + return checkReplaceable(endpoint, model) +} + +// checkReplaceable fails unless the endpoint serves at most one entity, a version +// of model, in its current and pending configs. Replacing anything else would +// remove another model or a traffic split the user didn't ask to change. +func checkReplaceable(endpoint *serving.ServingEndpointDetailed, model string) error { + for _, served := range [][]serving.ServedEntityOutput{currentEntities(endpoint), pendingEntities(endpoint)} { + if len(served) > 1 || (len(served) == 1 && !strings.EqualFold(served[0].EntityName, model)) { + return fmt.Errorf("endpoint %s serves %s, and air deploy replaces only a single version of %s; set endpoint to another name, or change this endpoint with databricks serving-endpoints update-config", endpoint.Name, describeEntities(served), model) + } + } + return nil +} + +// serveOnEndpoint makes the endpoint named name serve only entity and waits until +// it is ready, calling progress on every poll. It creates the endpoint if it +// doesn't exist. An endpoint that already serves entity isn't updated, and an +// update to entity that an interrupted air deploy started is waited for. +func serveOnEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string, entity serving.ServedEntityInput, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { + endpoint, err := getEndpoint(ctx, w, name) + if err != nil { + return nil, err + } + if endpoint == nil { + _, err := w.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ + Name: name, + Config: &serving.EndpointCoreConfigInput{ServedEntities: []serving.ServedEntityInput{entity}}, + }) + if err != nil { + return nil, err + } + return waitServes(ctx, w, name, entity, progress) + } + if updating(endpoint) { + if servesOnly(pendingEntities(endpoint), entity) { + return waitServes(ctx, w, name, entity, progress) + } + // The endpoint takes no other update until this one finishes. + endpoint, err = w.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, name, untilDeadline(ctx), progress) + if err != nil { + return nil, err + } + } + if err := checkReplaceable(endpoint, entity.EntityName); err != nil { + return nil, err + } + if servesOnly(currentEntities(endpoint), entity) { + return checkServes(endpoint, entity) + } + _, err = w.ServingEndpoints.UpdateConfig(ctx, serving.EndpointCoreConfigInput{ + Name: name, + ServedEntities: []serving.ServedEntityInput{entity}, + }) + if err != nil { + return nil, err + } + return waitServes(ctx, w, name, entity, progress) +} + +// waitServes waits until the endpoint's update finishes, then checks that it +// serves entity. +func waitServes(ctx context.Context, w *databricks.WorkspaceClient, name string, entity serving.ServedEntityInput, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { + endpoint, err := w.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, name, untilDeadline(ctx), progress) + if err != nil { + return nil, err + } + return checkServes(endpoint, entity) +} + +// checkServes returns endpoint if it serves only entity and is ready to serve. +func checkServes(endpoint *serving.ServingEndpointDetailed, entity serving.ServedEntityInput) (*serving.ServingEndpointDetailed, error) { + if !servesOnly(currentEntities(endpoint), entity) { + return nil, fmt.Errorf("endpoint %s was changed to serve %s while air deploy waited for it", endpoint.Name, describeEntities(currentEntities(endpoint))) + } + if endpoint.State == nil || endpoint.State.Ready != serving.EndpointStateReadyReady { + return nil, fmt.Errorf("endpoint %s serves version %s of %s but isn't ready; check its events in the Serving UI", endpoint.Name, entity.EntityVersion, entity.EntityName) + } + return endpoint, nil +} + +func updating(endpoint *serving.ServingEndpointDetailed) bool { + return endpoint.State != nil && endpoint.State.ConfigUpdate == serving.EndpointStateConfigUpdateInProgress +} + +func currentEntities(endpoint *serving.ServingEndpointDetailed) []serving.ServedEntityOutput { + if endpoint.Config == nil { + return nil + } + return endpoint.Config.ServedEntities +} + +func pendingEntities(endpoint *serving.ServingEndpointDetailed) []serving.ServedEntityOutput { + if endpoint.PendingConfig == nil { + return nil + } + return endpoint.PendingConfig.ServedEntities +} + +// describeEntities names served entities for an error, e.g. "version 2 of a.b.c". +func describeEntities(served []serving.ServedEntityOutput) string { + if len(served) == 0 { + return "nothing" + } + names := make([]string, 0, len(served)) + for _, s := range served { + if s.EntityVersion == "" { + names = append(names, s.EntityName) + continue + } + names = append(names, fmt.Sprintf("version %s of %s", s.EntityVersion, s.EntityName)) + } + return strings.Join(names, " and ") +} + +// servesOnly reports whether served is exactly entity, with the same workload and +// environment variables. +func servesOnly(served []serving.ServedEntityOutput, entity serving.ServedEntityInput) bool { + if len(served) != 1 { + return false + } + s := served[0] + // Unity Catalog names are case-insensitive, and the endpoint reports them as + // Unity Catalog stores them. + return strings.EqualFold(s.EntityName, entity.EntityName) && + s.EntityVersion == entity.EntityVersion && + s.WorkloadType == entity.WorkloadType && + s.WorkloadSize == entity.WorkloadSize && + s.ScaleToZeroEnabled == entity.ScaleToZeroEnabled && + maps.Equal(s.EnvironmentVars, entity.EnvironmentVars) +} + +// untilDeadline is the time left before ctx's deadline, which air deploy always +// sets from --timeout. +func untilDeadline(ctx context.Context) time.Duration { + deadline, _ := ctx.Deadline() + return time.Until(deadline) +} diff --git a/cmd/air/deployendpoint_test.go b/cmd/air/deployendpoint_test.go new file mode 100644 index 00000000000..3733a6a4ae9 --- /dev/null +++ b/cmd/air/deployendpoint_test.go @@ -0,0 +1,280 @@ +package aircmd + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const servingEndpointsPath = "/api/2.0/serving-endpoints" + +// fakeEndpoints is a stateful fake of the serving endpoints API for one endpoint. +// A create or config update is applied at once, and the endpoint then reports +// finalState (READY and NOT_UPDATING unless a test sets otherwise). +type fakeEndpoints struct { + t *testing.T + mu sync.Mutex + endpoint *serving.ServingEndpointDetailed + finalState serving.EndpointState + writes []string + // rawWrites holds the body of every create and config update, as sent. + rawWrites []string +} + +func newFakeEndpoints(t *testing.T, existing *serving.ServingEndpointDetailed) *fakeEndpoints { + return &fakeEndpoints{ + t: t, + endpoint: existing, + finalState: serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating}, + } +} + +func (f *fakeEndpoints) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + switch { + case r.Method == http.MethodPost && r.URL.Path == servingEndpointsPath: + var req serving.CreateServingEndpoint + if !f.decode(w, r, &req) { + return + } + f.writes = append(f.writes, "create") + f.endpoint = &serving.ServingEndpointDetailed{Name: req.Name, Config: f.output(req.Config.ServedEntities)} + f.endpoint.State = &f.finalState + case r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, "/config"): + var req serving.EndpointCoreConfigInput + if !f.decode(w, r, &req) { + return + } + if updating(f.endpoint) { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_CONFLICT", "message": "The endpoint is being updated."}`)) + return + } + f.writes = append(f.writes, "update") + f.endpoint.Config = f.output(req.ServedEntities) + f.endpoint.PendingConfig = nil + f.endpoint.State = &f.finalState + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, servingEndpointsPath+"/"): + if f.endpoint == nil { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "Endpoint not found"}`)) + return + } + default: + _, _ = w.Write([]byte(`{}`)) + return + } + assert.NoError(f.t, json.NewEncoder(w).Encode(f.endpoint)) +} + +// decode reads a request body into v, failing the request (not the handler +// goroutine) when it can't. +func (f *fakeEndpoints) decode(w http.ResponseWriter, r *http.Request, v any) bool { + raw, err := io.ReadAll(r.Body) + if !assert.NoError(f.t, err) || !assert.NoError(f.t, json.Unmarshal(raw, v)) { + w.WriteHeader(http.StatusInternalServerError) + return false + } + f.rawWrites = append(f.rawWrites, string(raw)) + return true +} + +func (f *fakeEndpoints) output(entities []serving.ServedEntityInput) *serving.EndpointCoreConfigOutput { + out := &serving.EndpointCoreConfigOutput{ConfigVersion: 1} + for _, e := range entities { + out.ServedEntities = append(out.ServedEntities, servedOutput(e)) + } + return out +} + +func servedOutput(e serving.ServedEntityInput) serving.ServedEntityOutput { + return serving.ServedEntityOutput{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + WorkloadType: e.WorkloadType, + WorkloadSize: e.WorkloadSize, + ScaleToZeroEnabled: e.ScaleToZeroEnabled, + EnvironmentVars: e.EnvironmentVars, + } +} + +func (f *fakeEndpoints) start() *httptest.Server { + srv := httptest.NewServer(f) + f.t.Cleanup(srv.Close) + return srv +} + +func testEntity(version string) serving.ServedEntityInput { + return serving.ServedEntityInput{ + EntityName: "main.models.qwen", + EntityVersion: version, + WorkloadType: serving.ServingModelWorkloadTypeGpuMedium, + WorkloadSize: "Small", + EnvironmentVars: map[string]string{"HF_TOKEN": "{{secrets/llm/hf_token}}"}, + } +} + +func endpointServing(entity serving.ServedEntityInput, state serving.EndpointState) *serving.ServingEndpointDetailed { + return &serving.ServingEndpointDetailed{ + Name: "qwen-chat", + Config: &serving.EndpointCoreConfigOutput{ServedEntities: []serving.ServedEntityOutput{servedOutput(entity)}}, + State: &state, + } +} + +var readyState = serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating} + +func TestServeOnEndpointCreatesAMissingEndpoint(t *testing.T) { + fake := newFakeEndpoints(t, nil) + srv := fake.start() + + endpoint, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"create"}, fake.writes) + assert.Equal(t, "3", endpoint.Config.ServedEntities[0].EntityVersion) + // scale_to_zero: false is sent, not left to the server's default. + require.Len(t, fake.rawWrites, 1) + assert.Contains(t, fake.rawWrites[0], `"scale_to_zero_enabled":false`) +} + +func TestServeOnEndpointUpdatesAnotherVersion(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("2"), readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"update"}, fake.writes) +} + +func TestServeOnEndpointUpdatesChangedCompute(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("3"), readyState)) + srv := fake.start() + entity := testEntity("3") + entity.WorkloadSize = "Medium" + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", entity, nil) + require.NoError(t, err) + assert.Equal(t, []string{"update"}, fake.writes) +} + +func TestServeOnEndpointLeavesAServedVersion(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("3"), readyState)) + srv := fake.start() + // Unity Catalog reports names lowercased. + entity := testEntity("3") + entity.EntityName = "Main.Models.Qwen" + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", entity, nil) + require.NoError(t, err) + assert.Empty(t, fake.writes) +} + +func TestServeOnEndpointWaitsForItsOwnPendingUpdate(t *testing.T) { + existing := endpointServing(testEntity("2"), serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress}) + existing.PendingConfig = &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{servedOutput(testEntity("3"))}} + fake := newFakeEndpoints(t, existing) + srv := fake.start() + + polls := 0 + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(*serving.ServingEndpointDetailed) { + // The pending update finishes after the first poll. + polls++ + fake.mu.Lock() + defer fake.mu.Unlock() + fake.endpoint = endpointServing(testEntity("3"), readyState) + }) + require.NoError(t, err) + assert.Empty(t, fake.writes) + assert.Positive(t, polls) +} + +func TestServeOnEndpointWaitsForAnotherUpdateBeforeUpdating(t *testing.T) { + existing := endpointServing(testEntity("1"), serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress}) + existing.PendingConfig = &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{servedOutput(testEntity("2"))}} + fake := newFakeEndpoints(t, existing) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(endpoint *serving.ServingEndpointDetailed) { + // The other update finishes after the first poll. + if updating(endpoint) { + fake.mu.Lock() + defer fake.mu.Unlock() + fake.endpoint = endpointServing(testEntity("2"), readyState) + } + }) + require.NoError(t, err) + assert.Equal(t, []string{"update"}, fake.writes) + assert.Equal(t, "3", fake.endpoint.Config.ServedEntities[0].EntityVersion) +} + +func TestServeOnEndpointRefusesToReplaceOtherEntities(t *testing.T) { + other := testEntity("5") + other.EntityName = "main.models.llama" + split := endpointServing(testEntity("2"), readyState) + split.Config.ServedEntities = append(split.Config.ServedEntities, servedOutput(testEntity("1"))) + tests := []struct { + name string + existing *serving.ServingEndpointDetailed + wantErr string + }{ + {"another model", endpointServing(other, readyState), "endpoint qwen-chat serves version 5 of main.models.llama, and air deploy replaces only a single version of main.models.qwen"}, + {"a traffic split", split, "endpoint qwen-chat serves version 2 of main.models.qwen and version 1 of main.models.qwen"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := newFakeEndpoints(t, tt.existing) + w := newTestWorkspaceClient(t, fake.start().URL) + + assert.ErrorContains(t, checkEndpoint(withDeadline(t), w, "qwen-chat", "main.models.qwen"), tt.wantErr) + _, err := serveOnEndpoint(withDeadline(t), w, "qwen-chat", testEntity("3"), nil) + assert.ErrorContains(t, err, tt.wantErr) + assert.Empty(t, fake.writes) + }) + } +} + +func TestCheckEndpointAllowsAMissingEndpoint(t *testing.T) { + srv := newFakeEndpoints(t, nil).start() + assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen")) +} + +func TestServeOnEndpointReportsAFailedUpdate(t *testing.T) { + fake := newFakeEndpoints(t, nil) + fake.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed} + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + assert.ErrorContains(t, err, "UPDATE_FAILED") +} + +func TestServeOnEndpointReportsAnEndpointThatIsNotReady(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testEntity("3"), serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating})) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + assert.ErrorContains(t, err, "endpoint qwen-chat serves version 3 of main.models.qwen but isn't ready") +} + +func TestServesOnly(t *testing.T) { + entity := testEntity("3") + assert.True(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity)}, entity)) + assert.False(t, servesOnly(nil, entity)) + assert.False(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity), servedOutput(testEntity("2"))}, entity)) + + changed := entity + changed.EnvironmentVars = map[string]string{"HF_TOKEN": "{{secrets/llm/other}}"} + assert.False(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity)}, changed)) + + scaled := entity + scaled.ScaleToZeroEnabled = true + assert.False(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity)}, scaled)) +} diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go new file mode 100644 index 00000000000..f09867aa04f --- /dev/null +++ b/cmd/air/deployregister.go @@ -0,0 +1,172 @@ +package aircmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" +) + +// registerModelPackagePath registers a model whose files Model Serving writes. It +// runs as the caller and is idempotent per key: a repeat request with the same key +// and inputs continues the registration, and finalizes a checkpoint's version once +// its weights are staged. +const registerModelPackagePath = "/api/2.0/serving-endpoints:registerModelPackage" + +// registrationPollInterval is how often air deploy repeats the registration while +// a checkpoint's weights are staged. +const registrationPollInterval = 15 * time.Second + +// stagingStateFailed is the ModelDeploymentStagingStatus state of an artifact that +// failed to stage and won't be retried automatically. +const stagingStateFailed = "FAILED" + +// registerModelPackageRequest is the RegisterModelPackageRequest body. Exactly one +// of Checkpoint, Code and SavedModel is set. +type registerModelPackageRequest struct { + RegisteredModel string `json:"registered_model"` + IdempotencyKey string `json:"idempotency_key"` + Checkpoint *modelPackageCheckpoint `json:"checkpoint,omitempty"` + Code *modelPackageCode `json:"code,omitempty"` + SavedModel *modelPackageSavedModel `json:"saved_model,omitempty"` + ExperimentPath string `json:"experiment_path,omitempty"` + RetryFailedStaging bool `json:"retry_failed_staging,omitempty"` +} + +type modelPackageCheckpoint struct { + VolumePath string `json:"volume_path"` + ServingCommand string `json:"serving_command"` + Task string `json:"task"` + Environment modelPackageEnvironment `json:"environment"` +} + +type modelPackageCode struct { + EntryFile string `json:"entry_file"` + EntrySource string `json:"entry_source"` + Files []modelPackageFile `json:"files,omitempty"` + Task string `json:"task,omitempty"` + Environment modelPackageEnvironment `json:"environment"` +} + +type modelPackageFile struct { + Path string `json:"path"` + Content string `json:"content"` +} + +type modelPackageSavedModel struct { + VolumePath string `json:"volume_path"` +} + +type modelPackageEnvironment struct { + Runtime string `json:"runtime"` + Dependencies []string `json:"dependencies,omitempty"` +} + +// registerModelPackageResponse is the RegisterModelPackageResponse body. +type registerModelPackageResponse struct { + ModelID string `json:"model_id"` + RegisteredModel string `json:"registered_model"` + ModelVersion string `json:"model_version"` + Ready bool `json:"ready"` + StagingStatus *modelStagingStatus `json:"staging_status,omitempty"` +} + +type modelStagingStatus struct { + State string `json:"state,omitempty"` + ModelArtifact *artifactStagingStatus `json:"model_artifact,omitempty"` + EnvironmentArtifact *artifactStagingStatus `json:"environment_artifact,omitempty"` +} + +type artifactStagingStatus struct { + State string `json:"state,omitempty"` +} + +// weightsState is the staging state of a checkpoint's weights, or "" when the +// response reports none. +func (r *registerModelPackageResponse) weightsState() string { + if r.StagingStatus == nil || r.StagingStatus.ModelArtifact == nil { + return "" + } + return r.StagingStatus.ModelArtifact.State +} + +// environmentState is the staging state of the serving environment, or "" when the +// response reports none. +func (r *registerModelPackageResponse) environmentState() string { + if r.StagingStatus == nil || r.StagingStatus.EnvironmentArtifact == nil { + return "" + } + return r.StagingStatus.EnvironmentArtifact.State +} + +// errStagingFailed reports that Model Serving couldn't copy a checkpoint's weights +// into the version. Rerunning with --retry-failed-staging stages them again. +var errStagingFailed = errors.New("staging failed") + +// registrationRequest builds the request that registers cfg's model under key. +// code is the collected model from code, and is nil for the other sources. +func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) registerModelPackageRequest { + req := registerModelPackageRequest{ + RegisteredModel: c.RegisteredModel, + IdempotencyKey: key, + } + if c.MLflowExperimentPath != nil { + req.ExperimentPath = *c.MLflowExperimentPath + } + switch c.Model.Type { + case modelTypeCheckpoint: + req.Checkpoint = &modelPackageCheckpoint{ + VolumePath: *c.Model.Path, + ServingCommand: *c.Command, + Task: *c.Task, + Environment: c.servingEnvironment(), + } + case modelTypeCode: + req.Code = code + case modelTypeSavedModelDir: + req.SavedModel = &modelPackageSavedModel{VolumePath: *c.Model.Path} + } + return req +} + +// registerUntilReady sends req until the version it registers is ready to serve, +// calling progress with every response. A checkpoint's version becomes ready once +// its weights are staged; the other sources are ready after the first request. +// It returns errStagingFailed, with the last response, when staging the weights +// failed. +func registerUntilReady(ctx context.Context, w *databricks.WorkspaceClient, req registerModelPackageRequest, interval time.Duration, progress func(*registerModelPackageResponse)) (*registerModelPackageResponse, error) { + apiClient, err := client.New(w.Config) + if err != nil { + return nil, fmt.Errorf("failed to create API client: %w", err) + } + headers := auth.WorkspaceIDHeaders(w.Config) + for { + var resp registerModelPackageResponse + if err := apiClient.Do(ctx, http.MethodPost, registerModelPackagePath, headers, nil, req, &resp); err != nil { + return nil, err + } + progress(&resp) + if resp.Ready { + return &resp, nil + } + if resp.weightsState() == stagingStateFailed { + return &resp, errStagingFailed + } + // Only the first request restages a failed checkpoint: repeating it would + // restart a staging that keeps failing instead of reporting the failure. + req.RetryFailedStaging = false + + timer := time.NewTimer(interval) + select { + case <-ctx.Done(): + timer.Stop() + return &resp, ctx.Err() + case <-timer.C: + } + } +} diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go new file mode 100644 index 00000000000..90732e14df5 --- /dev/null +++ b/cmd/air/deployregister_test.go @@ -0,0 +1,165 @@ +package aircmd + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeModelServing serves registerModelPackage with the given responses in turn, +// repeating the last, and records every registration body. Other paths are served +// by endpoints when set, else get an empty object (the SDK's config probe). +type fakeModelServing struct { + t *testing.T + mu sync.Mutex + responses []string + bodies []registerModelPackageRequest + endpoints http.HandlerFunc +} + +func newFakeModelServing(t *testing.T, responses ...string) *fakeModelServing { + return &fakeModelServing{t: t, responses: responses} +} + +func (f *fakeModelServing) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != registerModelPackagePath { + if f.endpoints != nil { + f.endpoints(w, r) + return + } + _, _ = w.Write([]byte(`{}`)) + return + } + raw, err := io.ReadAll(r.Body) + var body registerModelPackageRequest + if !assert.NoError(f.t, err) || !assert.NoError(f.t, json.Unmarshal(raw, &body)) { + w.WriteHeader(http.StatusInternalServerError) + return + } + + f.mu.Lock() + defer f.mu.Unlock() + f.bodies = append(f.bodies, body) + i := min(len(f.bodies), len(f.responses)) - 1 + if f.responses[i] == "" { + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_CONFLICT", "message": "The idempotency key can't be used for this registration. Use a new key."}`)) + return + } + _, _ = w.Write([]byte(f.responses[i])) +} + +func (f *fakeModelServing) start() *httptest.Server { + srv := httptest.NewServer(f) + f.t.Cleanup(srv.Close) + return srv +} + +func (f *fakeModelServing) requests() []registerModelPackageRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]registerModelPackageRequest(nil), f.bodies...) +} + +const ( + stagingInProgress = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, + "staging_status": {"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + stagingReady = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": true, + "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + stagingFailed = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, + "staging_status": {"state": "FAILED", "model_artifact": {"state": "FAILED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` +) + +func checkpointRequest() registerModelPackageRequest { + return registerModelPackageRequest{ + RegisteredModel: "main.models.qwen", + IdempotencyKey: "key-1", + Checkpoint: &modelPackageCheckpoint{ + VolumePath: "/Volumes/main/models/weights/qwen3_8b", + ServingCommand: "python -m vllm.entrypoints.openai.api_server --model qwen3_8b", + Task: "llm/v1/chat", + Environment: modelPackageEnvironment{Runtime: "client.6-gpu"}, + }, + } +} + +func TestRegisterUntilReadyRepeatsTheSameRequest(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress, stagingInProgress, stagingReady) + srv := fake.start() + + var states []string + req := checkpointRequest() + req.RetryFailedStaging = true + resp, err := registerUntilReady(t.Context(), newTestWorkspaceClient(t, srv.URL), req, time.Millisecond, func(r *registerModelPackageResponse) { + states = append(states, r.weightsState()) + }) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Equal(t, "3", resp.ModelVersion) + assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED"}, states) + + bodies := fake.requests() + require.Len(t, bodies, 3) + assert.Equal(t, req, bodies[0]) + // Only the first request restages; the repeats continue the same registration. + req.RetryFailedStaging = false + assert.Equal(t, req, bodies[1]) + assert.Equal(t, req, bodies[2]) +} + +func TestRegisterUntilReadyReportsFailedStaging(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress, stagingFailed) + srv := fake.start() + + resp, err := registerUntilReady(t.Context(), newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Millisecond, func(*registerModelPackageResponse) {}) + require.ErrorIs(t, err, errStagingFailed) + assert.Equal(t, "3", resp.ModelVersion) + assert.Len(t, fake.requests(), 2) +} + +func TestRegisterUntilReadyReturnsServerErrors(t *testing.T) { + srv := newFakeModelServing(t, "").start() + + _, err := registerUntilReady(t.Context(), newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Millisecond, func(*registerModelPackageResponse) {}) + assert.ErrorContains(t, err, "The idempotency key can't be used for this registration") +} + +func TestRegisterUntilReadyStopsWhenCanceled(t *testing.T) { + srv := newFakeModelServing(t, stagingInProgress).start() + ctx, cancel := context.WithCancel(t.Context()) + + _, err := registerUntilReady(ctx, newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Hour, func(*registerModelPackageResponse) { cancel() }) + assert.ErrorIs(t, err, context.Canceled) +} + +func TestRegisterModelPackageRequestJSON(t *testing.T) { + raw, err := json.Marshal(registerModelPackageRequest{ + RegisteredModel: "a.b.c", + IdempotencyKey: "k", + Code: &modelPackageCode{ + EntryFile: "agent.py", + EntrySource: "x = 1\n", + Files: []modelPackageFile{{Path: "tools/a.py", Content: ""}}, + Environment: modelPackageEnvironment{Runtime: "client.6", Dependencies: []string{"mlflow"}}, + }, + }) + require.NoError(t, err) + assert.JSONEq(t, `{ + "registered_model": "a.b.c", + "idempotency_key": "k", + "code": { + "entry_file": "agent.py", + "entry_source": "x = 1\n", + "files": [{"path": "tools/a.py", "content": ""}], + "environment": {"runtime": "client.6", "dependencies": ["mlflow"]} + } + }`, string(raw)) +} diff --git a/cmd/air/run.go b/cmd/air/run.go index 20a7683461f..6492764d3c0 100644 --- a/cmd/air/run.go +++ b/cmd/air/run.go @@ -131,7 +131,7 @@ field, pass its path to -h as a separate argument: if field.path == "config.compute" || field.path == "config.compute.accelerator_type" { computeOptions = loadComputeOptionsForHelp(c, args) } - renderConfigField(c.OutOrStdout(), field, computeOptions) + renderConfigField(c.OutOrStdout(), runCommandPath, field, computeOptions) }) // List flags in declaration order (most used first) rather than alphabetically. @@ -170,7 +170,7 @@ field, pass its path to -h as a separate argument: } snapshotFileCount = &count } - idempotencyToken := effectiveIdempotencyToken(idempotencyKey, cfg) + idempotencyToken := effectiveIdempotencyToken(idempotencyKey, cfg.IdempotencyToken) ctx, validationErr := validateDryRunInWorkspace(ctx, cmd, args, cfg, idempotencyToken, validationTimeout) if err := validateIdempotencyToken(idempotencyToken); err != nil { return err diff --git a/cmd/air/runconfig.go b/cmd/air/runconfig.go index 78554b02183..7306c1ab3c6 100644 --- a/cmd/air/runconfig.go +++ b/cmd/air/runconfig.go @@ -119,18 +119,10 @@ func (c *runConfig) validate() error { } } - if err := validateSecretRefs(c.Secrets); err != nil { + if err := validateEnvironmentVariables(c.EnvVariables, c.Secrets); err != nil { return err } - // A name can't be both a plain env var and a secret: the precedence would be - // ambiguous and could leak the secret. Sorted for a stable error. - for _, name := range slices.Sorted(maps.Keys(c.EnvVariables)) { - if _, ok := c.Secrets[name]; ok { - return fmt.Errorf("%q is set in both env_variables and secrets; remove it from one", name) - } - } - if c.CodeSource != nil { if err := c.CodeSource.validate(); err != nil { return err @@ -335,6 +327,23 @@ func validateCommand(v string) error { return nil } +// validateEnvironmentVariables checks the env_variables and secrets of a config: +// secret references use the "scope/key" format, and no name is in both. +func validateEnvironmentVariables(envVariables, secrets map[string]string) error { + if err := validateSecretRefs(secrets); err != nil { + return err + } + + // A name can't be both a plain env var and a secret: the precedence would be + // ambiguous and could leak the secret. Sorted for a stable error. + for _, name := range slices.Sorted(maps.Keys(envVariables)) { + if _, ok := secrets[name]; ok { + return fmt.Errorf("%q is set in both env_variables and secrets; remove it from one", name) + } + } + return nil +} + // validateSecretRefs checks that secret references use the "scope/key" format. func validateSecretRefs(secrets map[string]string) error { for varName, ref := range secrets { @@ -608,13 +617,16 @@ func (p *permission) validate() error { // or bare `compute`). const configHelpRoot = "config" +// runCommandPath is the command whose -h documents runConfig. +const runCommandPath = "databricks air run" + // writeConfigFieldHelp resolves a dotted config path and writes its docs. func writeConfigFieldHelp(w io.Writer, path string) error { field, err := resolveConfigField(path) if err != nil { return err } - renderConfigField(w, field, nil) + renderConfigField(w, runCommandPath, field, nil) return nil } @@ -675,8 +687,13 @@ func configSchema() configField { // resolveConfigField resolves a dotted YAML path against the run config schema. // The leading "config." is optional; an empty path describes the whole schema. func resolveConfigField(path string) (configField, error) { + return resolveSchemaField(configSchema(), path) +} + +// resolveSchemaField resolves a dotted YAML path against a config schema rooted +// at root, as resolveConfigField does for the run config. +func resolveSchemaField(root configField, path string) (configField, error) { trimmed := strings.TrimPrefix(strings.TrimPrefix(path, configHelpRoot), ".") - root := configSchema() if trimmed == "" { return root, nil } @@ -867,7 +884,8 @@ func underlyingConfigStruct(t reflect.Type) reflect.Type { // renderConfigField writes a resolved field's documentation. An object lists its // immediate children; a leaf gets its type, required-ness, and description. -func renderConfigField(w io.Writer, f configField, computeOptions []computeOption) { +// command is the command whose -h documents the schema. +func renderConfigField(w io.Writer, command string, f configField, computeOptions []computeOption) { fmt.Fprintf(w, "%s\n", f.path) if f.help != "" { fmt.Fprintf(w, " %s\n", f.help) @@ -916,8 +934,9 @@ func renderConfigField(w io.Writer, f configField, computeOptions []computeOptio fmt.Fprintf(w, " %-*s %s\n", width, configLeafName(c.path), configFieldSummary(c)) } } - fmt.Fprintf(w, "\nFor details on a field, run \"databricks air run -h %s.\".\n", f.path) - if f.path == configHelpRoot { + fmt.Fprintf(w, "\nFor details on a field, run \"%s -h %s.\".\n", command, f.path) + // The skill writes run YAML, so only `air run -h config` points at it. + if f.path == configHelpRoot && command == runCommandPath { fmt.Fprint(w, agentSkillHelp) } } diff --git a/cmd/air/runconfig_load.go b/cmd/air/runconfig_load.go index 2e05e1b50e6..e272bc8f096 100644 --- a/cmd/air/runconfig_load.go +++ b/cmd/air/runconfig_load.go @@ -18,10 +18,16 @@ import ( // decodeRunConfigReader decodes and unknown-key-checks a run YAML from r. path is // used only for error messages. func decodeRunConfigReader(r io.Reader, path string) (*runConfig, error) { + return decodeConfigReader[runConfig](r, path) +} + +// decodeConfigReader decodes and unknown-key-checks a YAML config from r into a +// T. path is used only for error messages. +func decodeConfigReader[T any](r io.Reader, path string) (*T, error) { dec := yaml.NewDecoder(r) dec.KnownFields(true) - var cfg runConfig + var cfg T if err := dec.Decode(&cfg); err != nil { if errors.Is(err, io.EOF) { return nil, fmt.Errorf("config %s is empty", path) @@ -59,30 +65,41 @@ func loadRunConfigWithOverrides(ctx context.Context, path string, overrides []st if len(overrides) == 0 { return loadRunConfig(path) } + document, validationYAML, err := readConfigWithOverrides(ctx, path, overrides, configSchema()) + if err != nil { + return nil, err + } + return finishRunConfigLoad(path, document, validationYAML) +} +// readConfigWithOverrides reads a YAML config and applies --override KEY=VALUE +// entries, checked against schema, to its ordered YAML tree. It returns the tree +// and its serialization, which the caller decodes with unknown keys rejected. +// ctx is used only to log applied overrides. +func readConfigWithOverrides(ctx context.Context, path string, overrides []string, schema configField) (*yaml.Node, []byte, error) { entries, err := parseOverrides(overrides) if err != nil { - return nil, err + return nil, nil, err } - if err := validateOverridePaths(entries); err != nil { - return nil, err + if err := validateOverridePaths(entries, schema); err != nil { + return nil, nil, err } document, _, err := readRunConfigDocument(path) if err != nil { - return nil, err + return nil, nil, err } if len(document.Content) == 0 { document.Kind = yaml.DocumentNode document.Content = []*yaml.Node{newMappingNode()} } if err := applyOverrides(ctx, document, entries); err != nil { - return nil, err + return nil, nil, err } validationYAML, err := yaml.Marshal(document) if err != nil { - return nil, fmt.Errorf("failed to serialize config %s: %w", path, err) + return nil, nil, fmt.Errorf("failed to serialize config %s: %w", path, err) } - return finishRunConfigLoad(path, document, validationYAML) + return document, validationYAML, nil } func readRunConfigDocument(path string) (*yaml.Node, []byte, error) { diff --git a/cmd/air/runconfig_override.go b/cmd/air/runconfig_override.go index 3fbccd076cb..d7585c8214a 100644 --- a/cmd/air/runconfig_override.go +++ b/cmd/air/runconfig_override.go @@ -45,11 +45,10 @@ type overrideEntry struct { raw string } -// validateOverridePaths checks every dotted path against the runConfig schema +// validateOverridePaths checks every dotted path against a config schema // before mutation, so an error names the exact --override key rather than the // re-decode's Go-type language. -func validateOverridePaths(entries []overrideEntry) error { - schema := configSchema() +func validateOverridePaths(entries []overrideEntry, schema configField) error { for _, e := range entries { if err := checkOverridePath(strings.Split(e.path, "."), schema, e.path); err != nil { return err @@ -100,7 +99,7 @@ func applyOverrides(ctx context.Context, document *yaml.Node, entries []override root = document.Content[0] } if root.Kind != yaml.MappingNode { - return errors.New("run config must be a YAML mapping") + return errors.New("config must be a YAML mapping") } for _, e := range entries { diff --git a/cmd/air/runconfig_override_test.go b/cmd/air/runconfig_override_test.go index c0dc4937138..982f9ad74a0 100644 --- a/cmd/air/runconfig_override_test.go +++ b/cmd/air/runconfig_override_test.go @@ -90,7 +90,7 @@ func TestValidateOverridePaths(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := validateOverridePaths([]overrideEntry{{path: tt.path, raw: "x"}}) + err := validateOverridePaths([]overrideEntry{{path: tt.path, raw: "x"}}, configSchema()) if tt.wantErr != "" { require.ErrorContains(t, err, tt.wantErr) return diff --git a/cmd/air/runconfig_test.go b/cmd/air/runconfig_test.go index 74b4c187628..64413f5d789 100644 --- a/cmd/air/runconfig_test.go +++ b/cmd/air/runconfig_test.go @@ -1057,14 +1057,14 @@ func TestConfigSchemaSharedByHelpAndOverride(t *testing.T) { for _, p := range []string{"code_source.type", "code_source.snapshot.root_path"} { t.Run(p+" compatibility override", func(t *testing.T) { - require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}})) + require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}}, configSchema())) _, err := resolveConfigField(p) require.Error(t, err) }) } for _, p := range paths { t.Run(p, func(t *testing.T) { - require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}})) + require.NoError(t, validateOverridePaths([]overrideEntry{{path: p, raw: "x"}}, configSchema())) // The help path accepts the same field (free-form sub-paths resolve to // the map itself, which is the schema's most specific node). _, err := resolveConfigField(p) @@ -1075,7 +1075,7 @@ func TestConfigSchemaSharedByHelpAndOverride(t *testing.T) { } // A field unknown to one is unknown to the other. - require.Error(t, validateOverridePaths([]overrideEntry{{path: "compute.bogus", raw: "x"}})) + require.Error(t, validateOverridePaths([]overrideEntry{{path: "compute.bogus", raw: "x"}}, configSchema())) _, err := resolveConfigField("compute.bogus") require.Error(t, err) } diff --git a/cmd/air/runsubmit.go b/cmd/air/runsubmit.go index c06165dd2b4..8ee0beca466 100644 --- a/cmd/air/runsubmit.go +++ b/cmd/air/runsubmit.go @@ -270,10 +270,10 @@ func aiRuntimeTaskFromSubmitBody(body map[string]any) (map[string]any, error) { // effectiveIdempotencyToken resolves the token sent to validation and submit: // the --idempotency-key flag wins, then the config's token, else a generated one. -func effectiveIdempotencyToken(flag string, cfg *runConfig) string { +func effectiveIdempotencyToken(flag string, configured *string) string { token := flag - if token == "" && cfg.IdempotencyToken != nil { - token = *cfg.IdempotencyToken + if token == "" && configured != nil { + token = *configured } if token == "" { token = uuid.New().String() @@ -345,7 +345,7 @@ func submitWorkload(ctx context.Context, w *databricks.WorkspaceClient, cfg *run return 0, "", err } containers := submittedContainers(cfg, funcDir) - token := effectiveIdempotencyToken(idempotencyKey, cfg) + token := effectiveIdempotencyToken(idempotencyKey, cfg.IdempotencyToken) deps, _ := cfg.inlineDependencies() diff --git a/cmd/air/runsubmit_test.go b/cmd/air/runsubmit_test.go index e101bbc28af..daa87788e8b 100644 --- a/cmd/air/runsubmit_test.go +++ b/cmd/air/runsubmit_test.go @@ -33,9 +33,9 @@ func stubValidateConfig(server *testserver.Server) { func TestEffectiveIdempotencyToken(t *testing.T) { cfg := &runConfig{IdempotencyToken: new("from-config")} - assert.Equal(t, "from-flag", effectiveIdempotencyToken("from-flag", cfg)) - assert.Equal(t, "from-config", effectiveIdempotencyToken("", cfg)) - assert.NotEmpty(t, effectiveIdempotencyToken("", &runConfig{})) + assert.Equal(t, "from-flag", effectiveIdempotencyToken("from-flag", cfg.IdempotencyToken)) + assert.Equal(t, "from-config", effectiveIdempotencyToken("", cfg.IdempotencyToken)) + assert.NotEmpty(t, effectiveIdempotencyToken("", nil)) } func TestDlRuntimeImage(t *testing.T) { From 11000234dab303d74666b05149897198b4f06287 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 15 Nov 2026 06:27:26 +0000 Subject: [PATCH 02/11] air deploy: poll staging status, guard endpoints and code, type JSON errors - Register once, poll the read-only staging status API while a checkpoint's weights copy, then register again to finalize. - Retry transient failures of the polled requests via libs/retry. - Check the endpoint before registering: replace only a single version of the same model, and wait out another update first. - Require code_source.include_paths, refuse pathspec magic and files git tracks although .gitignore matches them, and list the files on --dry-run. Fix the shared non-git listing, which was relative to the working directory instead of root_path. - Print the idempotency key before the first request in every mode; reject an empty --idempotency-key; give JSON errors typed codes. - Validate endpoint names, workload types and sizes, checkpoint tasks, and count length limits as the server does. Co-authored-by: Isaac --- cmd/air/deploy.go | 112 ++++++++++++++++++----- cmd/air/deploy_test.go | 103 ++++++++++++++++++--- cmd/air/deploycode.go | 56 +++++++++--- cmd/air/deploycode_test.go | 19 ++-- cmd/air/deployconfig.go | 49 +++++++--- cmd/air/deployconfig_test.go | 9 +- cmd/air/deployendpoint.go | 39 ++++++-- cmd/air/deployendpoint_test.go | 23 ++++- cmd/air/deployregister.go | 101 ++++++++++++++++----- cmd/air/deployregister_test.go | 148 ++++++++++++++++++++++++------- cmd/air/snapshot_package.go | 4 +- cmd/air/snapshot_package_test.go | 18 ++++ 12 files changed, 550 insertions(+), 131 deletions(-) diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go index 1b8f0932e92..8ac4dd903ba 100644 --- a/cmd/air/deploy.go +++ b/cmd/air/deploy.go @@ -6,7 +6,9 @@ import ( "fmt" "net/http" "path/filepath" + "regexp" "strings" + "sync" "time" "github.com/databricks/cli/cmd/root" @@ -32,6 +34,38 @@ var ( errDeployTimedOut = errors.New("air deploy timed out") ) +// JSON error codes of air deploy's own failures; API errors keep the server's. +const ( + codeInvalidArgs = "INVALID_ARGS" + codeInterrupted = "INTERRUPTED" + codeTimeout = "TIMEOUT" + codeStagingFailed = "STAGING_FAILED" + // codeEndpointNotReady: the endpoint serves the version but isn't ready, which + // running air deploy again doesn't change. + codeEndpointNotReady = "ENDPOINT_NOT_READY" + codeInternalError = "INTERNAL_ERROR" +) + +// apiErrorCodeRe matches an error code the server sent. The SDK fills ErrorCode +// with "" or "" when the response body has none. +var apiErrorCodeRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Z][A-Z_]*$`) }) + +// deployError carries the JSON error code and retryability of a failure whose +// message is err's. +type deployError struct { + code string + retryable bool + err error +} + +func (e *deployError) Error() string { return e.err.Error() } + +func (e *deployError) Unwrap() error { return e.err } + +func withCode(code string, retryable bool, err error) error { + return &deployError{code: code, retryable: retryable, err: err} +} + // deployResult is the JSON payload for `air deploy`. type deployResult struct { Status string `json:"status"` @@ -51,9 +85,12 @@ type deployResult struct { // deployOptions are the `air deploy` flags. type deployOptions struct { - file string - overrides []string - dryRun bool + file string + overrides []string + dryRun bool + // idempotencyKeySet records an explicit --idempotency-key, so an empty value + // (say, from an unset shell variable) fails instead of registering anew. + idempotencyKeySet bool idempotencyKey string retryFailedStaging bool timeout time.Duration @@ -139,6 +176,7 @@ its path to -h as a separate argument: } cmd.RunE = func(cmd *cobra.Command, args []string) error { + opts.idempotencyKeySet = cmd.Flags().Changed("idempotency-key") err := runDeploy(cmd, opts, pollInterval) if err == nil { return nil @@ -157,17 +195,17 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio ctx := cmd.Context() cfg, err := loadDeployConfig(ctx, opts.file, opts.overrides) if err != nil { - return err + return withCode(codeInvalidArgs, false, err) } if err := validateDeployFlags(cfg, opts); err != nil { - return err + return withCode(codeInvalidArgs, false, err) } var code *modelPackageCode if cfg.Model.Type == modelTypeCode { code, err = collectModelCode(ctx, cfg, opts.file) if err != nil { - return err + return withCode(codeInvalidArgs, false, err) } } @@ -186,6 +224,9 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio if cfg.registers() { // Fail on an endpoint air deploy won't change before copying any weights. if err := checkEndpoint(ctx, w, cfg.Endpoint, cfg.RegisteredModel); err != nil { + if stopped := deployStopped(ctx, err); stopped != nil { + return fmt.Errorf("%w while checking endpoint %s, before registering anything", stopped, cfg.Endpoint) + } return err } result.IdempotencyKey = effectiveIdempotencyToken(opts.idempotencyKey, cfg.IdempotencyToken) @@ -196,7 +237,7 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio result.ModelVersion, result.ModelID = resp.ModelVersion, resp.ModelID result.EnvironmentStagingFailed = resp.environmentState() == stagingStateFailed if !jsonOut { - cmdio.LogString(ctx, fmt.Sprintf("Version %s of %s is ready.", resp.ModelVersion, cfg.RegisteredModel)) + cmdio.LogString(ctx, fmt.Sprintf("Version %s of %s is ready.", stripControl(resp.ModelVersion), cfg.RegisteredModel)) if result.EnvironmentStagingFailed { cmdio.LogString(ctx, "Warning: the version's serving environment couldn't be prepared ahead of time, so the endpoint can be slower to start, or fail to start.") } @@ -206,21 +247,24 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio } entity := cfg.servedEntity(result.ModelVersion) - endpoint, err := serveVersion(ctx, w, cfg.Endpoint, entity, !jsonOut) + _, err = serveVersion(ctx, w, cfg.Endpoint, entity, !jsonOut) if err != nil { resume := deployResumeCommand(w.Config.Profile, opts, result.IdempotencyKey) if stopped := deployStopped(ctx, err); stopped != nil { return fmt.Errorf("%w while waiting for endpoint %s, which keeps updating. To continue, run:\n %s", stopped, cfg.Endpoint, resume) } + if errors.Is(err, errEndpointNotReady) { + return err + } return fmt.Errorf("serving version %s of %s on endpoint %s failed: %w\n\nTo try again without registering another version, run:\n %s", result.ModelVersion, cfg.RegisteredModel, cfg.Endpoint, err, resume) } result.Status = "READY" - result.EndpointURL = endpointInvocationsURL(w.Config.Host, endpoint.Name) + result.EndpointURL = endpointInvocationsURL(w.Config.Host, cfg.Endpoint) if jsonOut { return renderEnvelope(ctx, result) } out := cmd.OutOrStdout() - fmt.Fprintf(out, "Endpoint %s serves version %s of %s.\n", endpoint.Name, result.ModelVersion, cfg.RegisteredModel) + fmt.Fprintf(out, "Endpoint %s serves version %s of %s.\n", cfg.Endpoint, result.ModelVersion, cfg.RegisteredModel) fmt.Fprintln(out, "Query it at: "+result.EndpointURL) return nil } @@ -248,9 +292,9 @@ func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPacka // registerDeployment registers the version and waits until it is ready, showing // progress on a spinner when show is true. Its errors say how to continue. func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, code *modelPackageCode, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { - if show { - cmdio.LogString(ctx, fmt.Sprintf("Registering a version of %s with idempotency key %s", cfg.RegisteredModel, key)) - } + // Printed to stderr in every output mode, before the first request, so a run + // that is killed can still be continued with this key. + cmdio.LogString(ctx, fmt.Sprintf("Registering a version of %s with idempotency key %s", cfg.RegisteredModel, key)) req := cfg.registrationRequest(key, code) req.RetryFailedStaging = opts.retryFailedStaging @@ -272,12 +316,22 @@ func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg if cfg.Model.Path != nil { source = "the weights in " + *cfg.Model.Path } - return nil, fmt.Errorf("copying %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", source, resp.ModelVersion, cfg.RegisteredModel, resume) + return nil, withCode(codeStagingFailed, true, fmt.Errorf("copying %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", source, stripControl(resp.ModelVersion), cfg.RegisteredModel, resume)) } if stopped := deployStopped(ctx, err); stopped != nil { - return nil, fmt.Errorf("%w while registering a version of %s. Copying a checkpoint's weights continues, and the version is finalized when you continue. To continue, run:\n %s", stopped, cfg.RegisteredModel, resume) + continuing := "" + if resp != nil && resp.weightsState() != "" { + continuing = " Copying the weights continues, and the version is finalized when you continue." + } + return nil, fmt.Errorf("%w while registering a version of %s.%s To continue, run:\n %s", stopped, cfg.RegisteredModel, continuing, resume) + } + failed := fmt.Errorf("registering a version of %s with idempotency key %s failed: %w", cfg.RegisteredModel, key, err) + // A retryable failure is retried with the same key, so no other version is + // registered. + if _, retryable := deployErrorCode(err); retryable { + return nil, fmt.Errorf("%w\n\nTo try again, run:\n %s", failed, resume) } - return nil, fmt.Errorf("registering a version of %s with idempotency key %s failed: %w", cfg.RegisteredModel, key, err) + return nil, failed } // deployStopped returns why air deploy stopped waiting, or nil if it didn't. The @@ -297,17 +351,27 @@ func deployStopped(ctx context.Context, err error) error { } // deployErrorCode classifies a failed deployment for the JSON error envelope. +// Failures that aren't the user's input or the server's answer, such as a lost +// connection or a failed endpoint update, are worth retrying with the printed +// command. func deployErrorCode(err error) (code string, retryable bool) { switch { case errors.Is(err, errDeployInterrupted): - return "INTERRUPTED", true + return codeInterrupted, true case errors.Is(err, errDeployTimedOut): - return "TIMEOUT", true + return codeTimeout, true } - if apiErr, ok := errors.AsType[*apierr.APIError](err); ok && apiErr.ErrorCode != "" { - return apiErr.ErrorCode, apiErr.StatusCode == http.StatusTooManyRequests || apiErr.StatusCode >= http.StatusInternalServerError + if coded, ok := errors.AsType[*deployError](err); ok { + return coded.code, coded.retryable + } + if apiErr, ok := errors.AsType[*apierr.APIError](err); ok { + retryable = apiErr.StatusCode == http.StatusTooManyRequests || apiErr.StatusCode >= http.StatusInternalServerError + if !apiErrorCodeRe().MatchString(apiErr.ErrorCode) { + return codeInternalError, retryable + } + return apiErr.ErrorCode, retryable } - return "INVALID_ARGS", false + return codeInternalError, true } // validateDeployFlags rejects flags the config's model source can't use. @@ -315,7 +379,7 @@ func validateDeployFlags(cfg *deployConfig, opts deployOptions) error { if opts.timeout <= 0 { return fmt.Errorf("--timeout must be positive, got %s", opts.timeout) } - if opts.idempotencyKey != "" { + if opts.idempotencyKeySet { if !cfg.registers() { return errors.New("--idempotency-key applies only when air deploy registers a version; model.type is registered") } @@ -333,9 +397,9 @@ func validateDeployFlags(cfg *deployConfig, opts deployOptions) error { func registrationProgress(cfg *deployConfig, resp *registerModelPackageResponse) string { state := resp.weightsState() if state == "" { - return fmt.Sprintf("Registering version %s of %s…", resp.ModelVersion, cfg.RegisteredModel) + return fmt.Sprintf("Registering version %s of %s…", stripControl(resp.ModelVersion), cfg.RegisteredModel) } - return fmt.Sprintf("Copying the weights into version %s of %s (%s)…", resp.ModelVersion, cfg.RegisteredModel, state) + return fmt.Sprintf("Copying the weights into version %s of %s (%s)…", stripControl(resp.ModelVersion), cfg.RegisteredModel, stripControl(state)) } // serveVersion serves entity on the endpoint, showing the endpoint's state on a diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go index ce232b46ff4..2ae6ac0917c 100644 --- a/cmd/air/deploy_test.go +++ b/cmd/air/deploy_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http/httptest" "path/filepath" "strconv" @@ -17,6 +18,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/flags" + "github.com/databricks/cli/libs/retry" "github.com/databricks/cli/libs/shellquote" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" @@ -28,7 +30,7 @@ import ( // withDeadline returns the test's context with a deadline, which air deploy // always sets from --timeout. func withDeadline(t *testing.T) context.Context { - ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + ctx, cancel := context.WithTimeout(fastRetries(t.Context()), time.Minute) t.Cleanup(cancel) return ctx } @@ -52,7 +54,12 @@ func runDeployCmd(t *testing.T, out flags.Output, buf *bytes.Buffer, srvURL, con require.NoError(t, cmd.Flags().Set(flagValues[i], flagValues[i+1])) } - ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), out, nil, buf, buf, "", "")) + // JSON tests parse buf, so stderr's progress lines go elsewhere. + var stderr io.Writer = buf + if out == flags.OutputJSON { + stderr = io.Discard + } + ctx := cmdio.InContext(fastRetries(t.Context()), cmdio.NewIO(t.Context(), out, nil, buf, stderr, "", "")) if srvURL != "" { ctx = cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srvURL)) } @@ -91,6 +98,9 @@ func TestDeployCheckpointJSON(t *testing.T) { served := endpoints.endpoint.Config.ServedEntities[0] assert.Equal(t, "3", served.EntityVersion) assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, served.EnvironmentVars) + // scale_to_zero: false is sent, not left to the server's default. + require.Len(t, endpoints.rawWrites, 1) + assert.Contains(t, endpoints.rawWrites[0], `"scale_to_zero_enabled":false`) } func TestDeployCheckpointText(t *testing.T) { @@ -132,6 +142,20 @@ func TestDeployUsesTheConfigTokenThenANewKey(t *testing.T) { assert.NotContains(t, []string{"from-config", "from-flag"}, bodies[2].IdempotencyKey) } +func TestDeployPrintsTheKeyBeforeRegisteringInJSONMode(t *testing.T) { + srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) + + var stdout, stderr bytes.Buffer + cmd := withOutput(newDeployCommandWithPollInterval(time.Millisecond), flags.OutputJSON) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", checkpointDeployConfig))) + require.NoError(t, cmd.Flags().Set("idempotency-key", "key-1")) + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", "")) + cmd.SetContext(cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL))) + require.ErrorIs(t, cmd.RunE(cmd, nil), root.ErrAlreadyPrinted) + assert.Equal(t, "Registering a version of main.models.qwen with idempotency key key-1\n", stderr.String()) + assert.Contains(t, stdout.String(), `"code": "RESOURCE_CONFLICT"`) +} + func TestDeployRetriesFailedStagingOnce(t *testing.T) { srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress, stagingReady) @@ -154,7 +178,7 @@ func TestDeployFailedStagingSaysHowToRetry(t *testing.T) { } func TestDeployReportsRegistrationErrors(t *testing.T) { - srv, _ := deployServer(t, newFakeEndpoints(t, nil), "") + srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) var buf bytes.Buffer err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") @@ -162,7 +186,7 @@ func TestDeployReportsRegistrationErrors(t *testing.T) { } func TestDeployReportsErrorsAsJSON(t *testing.T) { - srv, _ := deployServer(t, newFakeEndpoints(t, nil), "") + srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) var buf bytes.Buffer err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig) @@ -192,6 +216,56 @@ func TestDeployChecksTheEndpointBeforeRegistering(t *testing.T) { assert.Empty(t, fake.requests()) } +func TestDeployReportsAnEndpointThatIsNotReady(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + endpoints.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateNotUpdating} + srv, _ := deployServer(t, endpoints, stagingReady) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + // Running air deploy again doesn't make the endpoint ready, so nothing says to. + assert.Equal(t, "ENDPOINT_NOT_READY", got.Error.Code) + assert.False(t, got.Error.Retryable) + assert.NotContains(t, got.Error.Message, "run:") +} + +func TestDeployTimesOutWhileCopyingTheWeights(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress) + fake.statuses = []string{statusInProgress} + + // Polls are spaced as the SDK's rate limiter allows, so the deadline, not the + // limiter, stops the wait. + var buf bytes.Buffer + cmd := withOutput(newDeployCommandWithPollInterval(100*time.Millisecond), flags.OutputJSON) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", checkpointDeployConfig))) + require.NoError(t, cmd.Flags().Set("idempotency-key", "key-1")) + require.NoError(t, cmd.Flags().Set("timeout", "500ms")) + ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &buf, io.Discard, "", "")) + cmd.SetContext(cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL))) + require.ErrorIs(t, cmd.RunE(cmd, nil), root.ErrAlreadyPrinted) + var got errorEnvelope + require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) + assert.Equal(t, "TIMEOUT", got.Error.Code) + assert.True(t, got.Error.Retryable) + assert.Contains(t, got.Error.Message, "air deploy timed out while registering a version of main.models.qwen. Copying the weights continues") + assert.Contains(t, got.Error.Message, " --idempotency-key key-1") + assert.Len(t, fake.requests(), 1) +} + +func TestDeployRetryableRegistrationFailureSaysHowToRetry(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), serverErrorResponse) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed: Something went wrong.\n\nTo try again, run:\n databricks air deploy -f ") + assert.Contains(t, err.Error(), " --idempotency-key key-1") + assert.Len(t, fake.requests(), retry.MaxRetries+1) +} + func TestDeployEndpointFailureSaysHowToRetry(t *testing.T) { endpoints := newFakeEndpoints(t, nil) endpoints.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed} @@ -291,6 +365,7 @@ func TestDeployFlagValidation(t *testing.T) { {"bad key", checkpointDeployConfig, []string{"idempotency-key", "a/b"}, "--idempotency-key must be 1 to 128 letters"}, {"retry for a registered version", registered, []string{"retry-failed-staging", "true"}, "--retry-failed-staging applies only to a checkpoint"}, {"zero timeout", checkpointDeployConfig, []string{"timeout", "0s"}, "--timeout must be positive"}, + {"empty key", checkpointDeployConfig, []string{"idempotency-key", ""}, `--idempotency-key must be 1 to 128 letters, digits, '.', '_' or '-', got ""`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -331,19 +406,25 @@ func TestDeployStopped(t *testing.T) { func TestDeployErrorCode(t *testing.T) { for _, tt := range []struct { + name string err error code string retryable bool }{ - {fmt.Errorf("x: %w", errDeployInterrupted), "INTERRUPTED", true}, - {fmt.Errorf("x: %w", errDeployTimedOut), "TIMEOUT", true}, - {fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "TEMPORARILY_UNAVAILABLE", StatusCode: 503}), "TEMPORARILY_UNAVAILABLE", true}, - {fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "INVALID_PARAMETER_VALUE", StatusCode: 400}), "INVALID_PARAMETER_VALUE", false}, - {errors.New("model: section is required"), "INVALID_ARGS", false}, + {"interrupted", fmt.Errorf("x: %w", errDeployInterrupted), "INTERRUPTED", true}, + {"timed out", fmt.Errorf("x: %w", errDeployTimedOut), "TIMEOUT", true}, + {"invalid input", withCode(codeInvalidArgs, false, errors.New("model: section is required")), "INVALID_ARGS", false}, + {"staging failed", withCode(codeStagingFailed, true, errors.New("copying failed")), "STAGING_FAILED", true}, + {"server unavailable", fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "TEMPORARILY_UNAVAILABLE", StatusCode: 503}), "TEMPORARILY_UNAVAILABLE", true}, + {"bad request", fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "INVALID_PARAMETER_VALUE", StatusCode: 400}), "INVALID_PARAMETER_VALUE", false}, + {"5xx without a body", &apierr.APIError{StatusCode: 502}, "INTERNAL_ERROR", true}, + {"error body without a code", &apierr.APIError{ErrorCode: "", StatusCode: 400}, "INTERNAL_ERROR", false}, + {"lost connection", errors.New("connection reset by peer"), "INTERNAL_ERROR", true}, + {"endpoint not ready", withCode(codeEndpointNotReady, false, errEndpointNotReady), "ENDPOINT_NOT_READY", false}, } { code, retryable := deployErrorCode(tt.err) - assert.Equal(t, tt.code, code, tt.err.Error()) - assert.Equal(t, tt.retryable, retryable, tt.err.Error()) + assert.Equal(t, tt.code, code, tt.name) + assert.Equal(t, tt.retryable, retryable, tt.name) } } diff --git a/cmd/air/deploycode.go b/cmd/air/deploycode.go index eb2c1ae1ee0..433941d77af 100644 --- a/cmd/air/deploycode.go +++ b/cmd/air/deploycode.go @@ -1,9 +1,12 @@ package aircmd import ( + "bytes" "context" + "errors" "fmt" "os" + "os/exec" "path/filepath" "regexp" "slices" @@ -30,28 +33,31 @@ const pythonCacheDir = "__pycache__" var codePathSegmentRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*$`) }) // collectModelCode reads a model from code from code_source: model.entry and the -// other files a code_source snapshot would include, as UTF-8 text. Files git -// ignores aren't sent, and nor are hidden files and directories or __pycache__ -// directories, whose names the server can't place or which hold no source. -// configPath locates a relative code_source.root_path. -// -// The files are stored in the model version, where anyone who can read the model -// can read them. Outside a git repository no .gitignore filters them, so the user -// must list them in code_source.include_paths. +// other files under code_source.include_paths, as UTF-8 text, selected as a +// code_source snapshot selects them. Files git ignores aren't sent, and nor are +// hidden files and directories or __pycache__ directories, whose names the +// server can't place or which hold no source. configPath locates a relative +// code_source.root_path. func collectModelCode(ctx context.Context, cfg *deployConfig, configPath string) (*modelPackageCode, error) { root, err := resolveRootPath(ctx, cfg.CodeSource.RootPath, filepath.Dir(configPath)) if err != nil { return nil, err } isGitRepo := newGitRepo(root).isRepository(ctx) - if !isGitRepo && len(cfg.CodeSource.IncludePaths) == 0 { - return nil, fmt.Errorf("code_source.include_paths is required because %s isn't in a git repository: air deploy stores the files in the model version, so list the ones to send, e.g. include_paths: [%s]", filepath.ToSlash(root), *cfg.Model.Entry) - } files, err := snapshotFiles(ctx, root, cfg.CodeSource.IncludePaths, isGitRepo) if err != nil { return nil, err } files = slices.DeleteFunc(files, skippedCodeFile) + if isGitRepo { + ignored, err := ignoredTrackedFiles(ctx, root, files) + if err != nil { + return nil, err + } + if len(ignored) > 0 { + return nil, fmt.Errorf("code_source file %s matches a .gitignore rule but is tracked by git; air deploy doesn't send ignored files, so leave it out of code_source.include_paths or stop tracking it", ignored[0]) + } + } code := &modelPackageCode{EntryFile: *cfg.Model.Entry, Environment: cfg.servingEnvironment()} if cfg.Task != nil { @@ -86,6 +92,34 @@ func collectModelCode(ctx context.Context, cfg *deployConfig, configPath string) return code, nil } +// ignoredTrackedFiles returns the files that a .gitignore rule matches although +// git tracks them: ls-files lists tracked files whatever the rules say, and the +// rules are the user's statement of what not to share. +func ignoredTrackedFiles(ctx context.Context, root string, files []snapshotFile) ([]string, error) { + var paths bytes.Buffer + for _, f := range files { + paths.WriteString(filepath.ToSlash(f.rel)) + paths.WriteByte(0) + } + cmd := exec.CommandContext(ctx, "git", "-C", root, "check-ignore", "--no-index", "-z", "--stdin") + cmd.Stdin = &paths + out, err := cmd.Output() + // check-ignore exits 1 when no path is ignored. + if exitErr, ok := errors.AsType[*exec.ExitError](err); ok && exitErr.ExitCode() == 1 { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err) + } + var ignored []string + for raw := range bytes.SplitSeq(out, []byte{0}) { + if len(raw) > 0 { + ignored = append(ignored, string(raw)) + } + } + return ignored, nil +} + // checkCodeFiles checks the files to send against the server's limits, from // their listing alone. func checkCodeFiles(files []snapshotFile, dependencies []string) error { diff --git a/cmd/air/deploycode_test.go b/cmd/air/deploycode_test.go index c5d50fb88ce..8fa2fa7e0ab 100644 --- a/cmd/air/deploycode_test.go +++ b/cmd/air/deploycode_test.go @@ -126,16 +126,23 @@ func TestCollectModelCodeLeavesOutIgnoredFiles(t *testing.T) { assert.Equal(t, []string{"agent.py", "tools/a.py"}, codeFilePaths(code)) } -func TestCollectModelCodeOutsideGitNeedsIncludePaths(t *testing.T) { - cfg, configPath := codeTreeConfig(t, t.TempDir(), map[string]string{"agent.py": "x\n", "credentials.json": "{}\n"}) +func TestCollectModelCodeRefusesTrackedIgnoredFiles(t *testing.T) { + cfg, configPath := writeCodeTree(t, map[string]string{"agent.py": "x\n", "creds.json": "{}\n"}) + repo := cfg.CodeSource.RootPath + commitAll(t, repo, "init") + writeRepoFile(t, repo, ".gitignore", "creds.json\n") + cfg.CodeSource.IncludePaths = []string{"agent.py", "creds.json"} + _, err := collectModelCode(t.Context(), cfg, configPath) - assert.ErrorContains(t, err, "code_source.include_paths is required because") - assert.ErrorContains(t, err, "isn't in a git repository") + assert.ErrorContains(t, err, "code_source file creds.json matches a .gitignore rule but is tracked by git") +} - cfg.CodeSource.IncludePaths = []string{"agent.py"} +func TestCollectModelCodeOutsideGit(t *testing.T) { + cfg, configPath := codeTreeConfig(t, t.TempDir(), map[string]string{"agent.py": "x\n", "tools/a.py": "a\n", "credentials.json": "{}\n"}) + cfg.CodeSource.IncludePaths = []string{"agent.py", "tools"} code, err := collectModelCode(t.Context(), cfg, configPath) require.NoError(t, err) - assert.Equal(t, []string{"agent.py"}, codeFilePaths(code)) + assert.Equal(t, []string{"agent.py", "tools/a.py"}, codeFilePaths(code)) } func TestCollectModelCodeChecksSizesBeforeReading(t *testing.T) { diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index 4b035cd616e..29619ef9db7 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -12,7 +12,7 @@ import ( "strconv" "strings" "sync" - "unicode/utf8" + "unicode/utf16" "github.com/databricks/databricks-sdk-go/service/serving" "go.yaml.in/yaml/v3" @@ -66,6 +66,13 @@ var deployIdempotencyKeyRe = sync.OnceValue(func() *regexp.Regexp { return regex // modelVersionRe matches a Unity Catalog model version number. var modelVersionRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[1-9][0-9]*$`) }) +// endpointNameRe matches a serving endpoint name: letters, digits, '-' and '_'. +// The name is part of the API's URL paths, so nothing else may reach a request. +var endpointNameRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_-]+$`) }) + +// workloadSizes are the concurrency tiers a served entity scales to. +var workloadSizes = []string{"Small", "Medium", "Large"} + // entryFileRe matches the name of a model from code's entry file, which the server // places at the model's root. var entryFileRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$`) }) @@ -78,9 +85,9 @@ type deployConfig struct { Task *string `yaml:"task" help:"The model's task as MLflow names it: llm/v1/chat, llm/v1/completions or llm/v1/embeddings for a checkpoint, or e.g. agent/v1/responses for a model from code. A model from code without a task has no signature." required:"when model.type is checkpoint" group:"Model"` Command *string `yaml:"command" help:"Shell command that starts the model server of a checkpoint, at most 32,768 characters. It runs from the directory where model.path appears under its own name." required:"when model.type is checkpoint" group:"Model"` Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment version to serve on and the packages to install, for a checkpoint or a model from code." group:"Model"` - CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of a model from code, sent with the registration and stored in the model version, where anyone who can read the model can read it. Hidden files and directories (names starting with '.') and __pycache__ directories aren't sent; run with --dry-run to list the files that are." required:"when model.type is code" group:"Model"` + CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of a model from code, sent with the registration and stored in the model version, where anyone who can read the model can read it. Only the files under include_paths are sent, leaving out files git ignores, hidden files and directories (names starting with '.') and __pycache__ directories; run with --dry-run to list them." required:"when model.type is code" group:"Model"` - Endpoint string `yaml:"endpoint" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` + Endpoint string `yaml:"endpoint" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead, with the compute, environment variables and secrets in this file replacing the ones it has; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` Compute *deployComputeConfig `yaml:"compute" help:"Serving compute and how the endpoint scales." required:"yes" group:"Serving"` EnvVariables map[string]string `yaml:"env_variables" help:"Plain environment variables of the served model, as NAME: value. A name here cannot also appear in secrets." group:"Serving"` Secrets map[string]string `yaml:"secrets" help:"Environment variables of the served model sourced from secrets, as NAME: scope/key." group:"Serving"` @@ -106,7 +113,7 @@ type deployEnvironmentConfig struct { // deployCodeSourceConfig is the `code_source` block of a model from code. type deployCodeSourceConfig struct { RootPath string `yaml:"root_path" help:"Directory holding the model's code. A relative path resolves against the YAML file's directory." required:"yes"` - IncludePaths []string `yaml:"include_paths" help:"Send only these paths, relative to root_path and without \"..\". Omit to send every file that git doesn't ignore; required when root_path isn't in a git repository."` + IncludePaths []string `yaml:"include_paths" help:"Files and directories to send, relative to root_path and without \"..\", e.g. [agent.py, tools]." required:"yes"` } // deployComputeConfig is the `compute` block: the served entity's workload. @@ -146,8 +153,8 @@ func (c *deployConfig) validate() error { if err := c.validateSource(); err != nil { return err } - if strings.TrimSpace(c.Endpoint) == "" { - return errors.New("endpoint is required") + if !endpointNameRe().MatchString(c.Endpoint) { + return fmt.Errorf("endpoint must be a serving endpoint name of letters, digits, '-' and '_', got %q", c.Endpoint) } if c.Compute == nil { return errors.New("compute: section is required") @@ -155,8 +162,8 @@ func (c *deployConfig) validate() error { if workloadTypes := servingWorkloadTypes(); !slices.Contains(workloadTypes, c.Compute.WorkloadType) { return fmt.Errorf("compute.workload_type must be one of %s, got %q", strings.Join(workloadTypes, ", "), c.Compute.WorkloadType) } - if strings.TrimSpace(c.Compute.WorkloadSize) == "" { - return errors.New("compute.workload_size is required") + if !slices.Contains(workloadSizes, c.Compute.WorkloadSize) { + return fmt.Errorf("compute.workload_size must be one of %s, got %q", strings.Join(workloadSizes, ", "), c.Compute.WorkloadSize) } if c.Model.Type == modelTypeCheckpoint && !c.servesOnGPU() { return fmt.Errorf("a checkpoint is served on GPU; set compute.workload_type to a GPU workload type such as GPU_MEDIUM, got %q", c.Compute.WorkloadType) @@ -207,7 +214,7 @@ func (c *deployConfig) validateSource() error { if m.Path != nil { v := *m.Path - if !strings.HasPrefix(v, "/Volumes/") || utf8.RuneCountInString(v) > maxVolumePathChars { + if !strings.HasPrefix(v, "/Volumes/") || serverLength(v) > maxVolumePathChars { return fmt.Errorf("model.path must be a Unity Catalog volume directory as /Volumes////, of at most %d characters, got %q", maxVolumePathChars, v) } } @@ -227,7 +234,7 @@ func (c *deployConfig) validateSource() error { if err := validateCommand(*c.Command); err != nil { return err } - if n := utf8.RuneCountInString(*c.Command); n > maxServingCommandChars { + if n := serverLength(*c.Command); n > maxServingCommandChars { return fmt.Errorf("command must be at most %d characters, got %d", maxServingCommandChars, n) } } @@ -237,6 +244,18 @@ func (c *deployConfig) validateSource() error { } } if c.CodeSource != nil { + // The files are stored in the model version for anyone who can read the + // model, so the user names them rather than sending a whole directory. + if c.CodeSource.IncludePaths == nil { + return errors.New("code_source.include_paths is required: list the files and directories to send, e.g. include_paths: [agent.py, tools]") + } + // git reads a leading ':' as pathspec magic, such as ":!x" for everything + // but x, which would send files the list doesn't name. + for _, p := range c.CodeSource.IncludePaths { + if strings.HasPrefix(strings.TrimSpace(p), ":") { + return fmt.Errorf("code_source.include_paths must name files and directories, got %q", p) + } + } snapshot := snapshotSourceConfig{RootPath: c.CodeSource.RootPath, IncludePaths: c.CodeSource.IncludePaths} if err := snapshot.validate("code_source"); err != nil { return err @@ -270,6 +289,16 @@ func validateDeployIdempotencyKey(key, source string) error { return nil } +// serverLength is the length of s as the server's limits count it, in UTF-16 +// code units. +func serverLength(s string) int { + n := 0 + for _, r := range s { + n += utf16.RuneLen(r) + } + return n +} + // servingWorkloadTypes lists the served entity workload types, sorted. func servingWorkloadTypes() []string { var workloadType serving.ServingModelWorkloadType diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index 80f64e76012..6d5efc47302 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -33,6 +33,7 @@ model: task: agent/v1/responses code_source: root_path: . + include_paths: [agent.py, tools] environment: version: 5 dependencies: [mlflow==3.5.0] @@ -155,9 +156,11 @@ func TestDeployConfigValidation(t *testing.T) { {"agent task on a checkpoint", replaceFirst("task: llm/v1/chat", "task: agent/v1/responses"), `task of a checkpoint must be one of llm/v1/chat, llm/v1/completions, llm/v1/embeddings, got "agent/v1/responses"`}, {"zero environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: 0"), "environment.version must be a serverless environment version"}, {"empty dependency", replaceFirst("[einops==0.8.0]", `[""]`), "environment.dependencies[0] cannot be empty"}, - {"no endpoint", dropLine("endpoint:"), "endpoint is required"}, + {"no endpoint", dropLine("endpoint:"), `endpoint must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, + {"endpoint with a path", replaceFirst("endpoint: qwen-chat", "endpoint: x/../../jobs"), "endpoint must be a serving endpoint name"}, {"no compute", dropBlock("compute:"), "compute: section is required"}, - {"no workload size", dropLine(" workload_size:"), "compute.workload_size is required"}, + {"no workload size", dropLine(" workload_size:"), `compute.workload_size must be one of Small, Medium, Large, got ""`}, + {"lowercase workload size", replaceFirst("workload_size: Small", "workload_size: medium"), "compute.workload_size must be one of Small, Medium, Large"}, {"secret without key", replaceFirst("llm/hf_token", "llm"), "expected format 'scope/key'"}, {"env var and secret", replaceFirst("LOG_LEVEL: info", "HF_TOKEN: x"), `"HF_TOKEN" is set in both env_variables and secrets`}, {"bad idempotency token", appendText("idempotency_token: a b\n"), "idempotency_token must be 1 to 128 letters"}, @@ -185,6 +188,8 @@ func TestDeployConfigSourceFields(t *testing.T) { {"code with command", "model: {type: code, entry: agent.py}\ncode_source: {root_path: .}\ncommand: x\n", "command can't be set when model.type is code"}, {"code entry in a directory", "model: {type: code, entry: src/agent.py}\ncode_source: {root_path: .}\n", "model.entry must name a .py file directly under code_source.root_path"}, {"code entry not python", "model: {type: code, entry: agent.ipynb}\ncode_source: {root_path: .}\n", "model.entry must name a .py file"}, + {"code without include_paths", "model: {type: code, entry: agent.py}\ncode_source: {root_path: .}\n", "code_source.include_paths is required: list the files and directories to send"}, + {"code include pathspec magic", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\":!creds.json\"]}\n", `code_source.include_paths must name files and directories, got ":!creds.json"`}, {"code include traversal", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [../x]}\n", "code_source.include_paths cannot contain '..' traversal"}, {"saved model with task", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\ntask: llm/v1/chat\n", "task can't be set when model.type is saved_model_dir"}, {"saved model with environment", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\nenvironment: {version: 6}\n", "environment can't be set when model.type is saved_model_dir"}, diff --git a/cmd/air/deployendpoint.go b/cmd/air/deployendpoint.go index 7a5d4d5f90a..e240dbc1f0b 100644 --- a/cmd/air/deployendpoint.go +++ b/cmd/air/deployendpoint.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/serving" @@ -30,13 +31,25 @@ func (c *deployConfig) servedEntity(version string) serving.ServedEntityInput { // getEndpoint returns the endpoint named name, or nil if it doesn't exist. func getEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string) (*serving.ServingEndpointDetailed, error) { - endpoint, err := w.ServingEndpoints.GetByName(ctx, name) + endpoint, err := retry.With(ctx, retry.IsTransientRead, func() (*serving.ServingEndpointDetailed, error) { + return w.ServingEndpoints.GetByName(ctx, name) + }) if errors.Is(err, apierr.ErrResourceDoesNotExist) { return nil, nil } return endpoint, err } +// waitNotUpdating waits until the endpoint has no update in progress. The SDK's +// wait stops at the first failed poll, so a transient failure starts it again: +// an endpoint can take many minutes to start. Creates and config updates aren't +// repeated this way, since a repeat after an applied write would fail. +func waitNotUpdating(ctx context.Context, w *databricks.WorkspaceClient, name string, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { + return retry.With(ctx, retry.IsTransientRead, func() (*serving.ServingEndpointDetailed, error) { + return w.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, name, untilDeadline(ctx), progress) + }) +} + // checkEndpoint fails when the endpoint named name, if it exists, serves // something air deploy won't replace with a version of model. air deploy checks // it before registering, so a wrong endpoint name fails before any weights are @@ -51,11 +64,13 @@ func checkEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name, mod // checkReplaceable fails unless the endpoint serves at most one entity, a version // of model, in its current and pending configs. Replacing anything else would -// remove another model or a traffic split the user didn't ask to change. +// remove another model or a traffic split the user didn't ask to change. The +// endpoint lists every served model under served_entities too (served_models +// mirrors it), so served_entities covers both. func checkReplaceable(endpoint *serving.ServingEndpointDetailed, model string) error { for _, served := range [][]serving.ServedEntityOutput{currentEntities(endpoint), pendingEntities(endpoint)} { if len(served) > 1 || (len(served) == 1 && !strings.EqualFold(served[0].EntityName, model)) { - return fmt.Errorf("endpoint %s serves %s, and air deploy replaces only a single version of %s; set endpoint to another name, or change this endpoint with databricks serving-endpoints update-config", endpoint.Name, describeEntities(served), model) + return withCode(codeInvalidArgs, false, fmt.Errorf("endpoint %s serves %s, and air deploy replaces only a single version of %s; set endpoint to another name, or change this endpoint with databricks serving-endpoints update-config", stripControl(endpoint.Name), describeEntities(served), model)) } } return nil @@ -85,7 +100,7 @@ func serveOnEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name st return waitServes(ctx, w, name, entity, progress) } // The endpoint takes no other update until this one finishes. - endpoint, err = w.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, name, untilDeadline(ctx), progress) + endpoint, err = waitNotUpdating(ctx, w, name, progress) if err != nil { return nil, err } @@ -109,20 +124,25 @@ func serveOnEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name st // waitServes waits until the endpoint's update finishes, then checks that it // serves entity. func waitServes(ctx context.Context, w *databricks.WorkspaceClient, name string, entity serving.ServedEntityInput, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { - endpoint, err := w.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, name, untilDeadline(ctx), progress) + endpoint, err := waitNotUpdating(ctx, w, name, progress) if err != nil { return nil, err } return checkServes(endpoint, entity) } +// errEndpointNotReady reports an endpoint that serves the version but isn't ready, +// which running air deploy again doesn't change. +var errEndpointNotReady = errors.New("endpoint isn't ready") + // checkServes returns endpoint if it serves only entity and is ready to serve. func checkServes(endpoint *serving.ServingEndpointDetailed, entity serving.ServedEntityInput) (*serving.ServingEndpointDetailed, error) { + name := stripControl(endpoint.Name) if !servesOnly(currentEntities(endpoint), entity) { - return nil, fmt.Errorf("endpoint %s was changed to serve %s while air deploy waited for it", endpoint.Name, describeEntities(currentEntities(endpoint))) + return nil, fmt.Errorf("endpoint %s was changed to serve %s while air deploy waited for it", name, describeEntities(currentEntities(endpoint))) } if endpoint.State == nil || endpoint.State.Ready != serving.EndpointStateReadyReady { - return nil, fmt.Errorf("endpoint %s serves version %s of %s but isn't ready; check its events in the Serving UI", endpoint.Name, entity.EntityVersion, entity.EntityName) + return nil, withCode(codeEndpointNotReady, false, fmt.Errorf("%w: endpoint %s serves version %s of %s but isn't ready; check its events and logs in the Serving UI", errEndpointNotReady, name, entity.EntityVersion, entity.EntityName)) } return endpoint, nil } @@ -152,11 +172,12 @@ func describeEntities(served []serving.ServedEntityOutput) string { } names := make([]string, 0, len(served)) for _, s := range served { + name := stripControl(s.EntityName) if s.EntityVersion == "" { - names = append(names, s.EntityName) + names = append(names, name) continue } - names = append(names, fmt.Sprintf("version %s of %s", s.EntityVersion, s.EntityName)) + names = append(names, fmt.Sprintf("version %s of %s", stripControl(s.EntityVersion), name)) } return strings.Join(names, " and ") } diff --git a/cmd/air/deployendpoint_test.go b/cmd/air/deployendpoint_test.go index 3733a6a4ae9..e267a1bdaf4 100644 --- a/cmd/air/deployendpoint_test.go +++ b/cmd/air/deployendpoint_test.go @@ -27,6 +27,8 @@ type fakeEndpoints struct { writes []string // rawWrites holds the body of every create and config update, as sent. rawWrites []string + // failGets is how many GETs answer 500 before the endpoint is served again. + failGets int } func newFakeEndpoints(t *testing.T, existing *serving.ServingEndpointDetailed) *fakeEndpoints { @@ -64,6 +66,12 @@ func (f *fakeEndpoints) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.endpoint.PendingConfig = nil f.endpoint.State = &f.finalState case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, servingEndpointsPath+"/"): + if f.failGets > 0 && f.endpoint != nil { + f.failGets-- + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error_code": "INTERNAL_ERROR", "message": "Something went wrong."}`)) + return + } if f.endpoint == nil { w.WriteHeader(http.StatusNotFound) _, _ = w.Write([]byte(`{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "Endpoint not found"}`)) @@ -141,9 +149,6 @@ func TestServeOnEndpointCreatesAMissingEndpoint(t *testing.T) { require.NoError(t, err) assert.Equal(t, []string{"create"}, fake.writes) assert.Equal(t, "3", endpoint.Config.ServedEntities[0].EntityVersion) - // scale_to_zero: false is sent, not left to the server's default. - require.Len(t, fake.rawWrites, 1) - assert.Contains(t, fake.rawWrites[0], `"scale_to_zero_enabled":false`) } func TestServeOnEndpointUpdatesAnotherVersion(t *testing.T) { @@ -247,6 +252,18 @@ func TestCheckEndpointAllowsAMissingEndpoint(t *testing.T) { assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen")) } +func TestServeOnEndpointWaitsThroughATransientFailure(t *testing.T) { + fake := newFakeEndpoints(t, nil) + // The create succeeds; the first poll of the wait then fails. + fake.failGets = 1 + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"create"}, fake.writes) + assert.Zero(t, fake.failGets) +} + func TestServeOnEndpointReportsAFailedUpdate(t *testing.T) { fake := newFakeEndpoints(t, nil) fake.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateUpdateFailed} diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go index f09867aa04f..a5ae7277553 100644 --- a/cmd/air/deployregister.go +++ b/cmd/air/deployregister.go @@ -8,6 +8,7 @@ import ( "time" "github.com/databricks/cli/libs/auth" + "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/client" ) @@ -15,16 +16,24 @@ import ( // registerModelPackagePath registers a model whose files Model Serving writes. It // runs as the caller and is idempotent per key: a repeat request with the same key // and inputs continues the registration, and finalizes a checkpoint's version once -// its weights are staged. +// its weights are staged. Staging never finalizes the version by itself. const registerModelPackagePath = "/api/2.0/serving-endpoints:registerModelPackage" -// registrationPollInterval is how often air deploy repeats the registration while -// a checkpoint's weights are staged. +// stagingStatusPath reports the recorded staging progress of a model version, a +// read. +const stagingStatusPath = "/api/2.0/serving-endpoints:checkModelDeploymentStagingStatus" + +// registrationPollInterval is how often air deploy checks on a checkpoint's +// weights while they are staged. const registrationPollInterval = 15 * time.Second -// stagingStateFailed is the ModelDeploymentStagingStatus state of an artifact that -// failed to stage and won't be retried automatically. -const stagingStateFailed = "FAILED" +// ModelDeploymentStagingStatus states air deploy acts on. +const ( + // stagingStateInProgress: an upload is queued, running or waiting to be retried. + stagingStateInProgress = "IN_PROGRESS" + // stagingStateFailed: the upload failed and won't be retried automatically. + stagingStateFailed = "FAILED" +) // registerModelPackageRequest is the RegisterModelPackageRequest body. Exactly one // of Checkpoint, Code and SavedModel is set. @@ -86,6 +95,17 @@ type artifactStagingStatus struct { State string `json:"state,omitempty"` } +// stagingStatusRequest is the CheckModelDeploymentStagingStatusRequest body. +type stagingStatusRequest struct { + Model string `json:"model"` + ModelVersion string `json:"model_version"` +} + +// stagingStatusResponse is the CheckModelDeploymentStagingStatusResponse body. +type stagingStatusResponse struct { + Status *modelStagingStatus `json:"status,omitempty"` +} + // weightsState is the staging state of a checkpoint's weights, or "" when the // response reports none. func (r *registerModelPackageResponse) weightsState() string { @@ -134,22 +154,26 @@ func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) r return req } -// registerUntilReady sends req until the version it registers is ready to serve, -// calling progress with every response. A checkpoint's version becomes ready once -// its weights are staged; the other sources are ready after the first request. -// It returns errStagingFailed, with the last response, when staging the weights -// failed. +// registerUntilReady registers req's version and waits until it is ready to +// serve, calling progress with every registration response and staging update. +// The other sources are ready after the first request. A checkpoint's version +// becomes ready once its weights are staged: while they copy, it polls the staging +// status, a read, then repeats the registration, which finalizes the version. On +// error it returns the last response it got, nil before the first one, and +// errStagingFailed when staging failed. func registerUntilReady(ctx context.Context, w *databricks.WorkspaceClient, req registerModelPackageRequest, interval time.Duration, progress func(*registerModelPackageResponse)) (*registerModelPackageResponse, error) { apiClient, err := client.New(w.Config) if err != nil { return nil, fmt.Errorf("failed to create API client: %w", err) } - headers := auth.WorkspaceIDHeaders(w.Config) + p := modelServingPoller{api: apiClient, headers: auth.WorkspaceIDHeaders(w.Config), interval: interval} + var last *registerModelPackageResponse for { var resp registerModelPackageResponse - if err := apiClient.Do(ctx, http.MethodPost, registerModelPackagePath, headers, nil, req, &resp); err != nil { - return nil, err + if err := p.post(ctx, registerModelPackagePath, req, &resp); err != nil { + return last, err } + last = &resp progress(&resp) if resp.Ready { return &resp, nil @@ -160,13 +184,50 @@ func registerUntilReady(ctx context.Context, w *databricks.WorkspaceClient, req // Only the first request restages a failed checkpoint: repeating it would // restart a staging that keeps failing instead of reporting the failure. req.RetryFailedStaging = false + if err := p.waitForStaging(ctx, &resp, progress); err != nil { + return &resp, err + } + } +} - timer := time.NewTimer(interval) - select { - case <-ctx.Done(): - timer.Stop() - return &resp, ctx.Err() - case <-timer.C: +// modelServingPoller sends the requests of a registration that can take many +// minutes. +type modelServingPoller struct { + api *client.DatabricksClient + headers map[string]string + interval time.Duration +} + +// waitForStaging polls the staging status of resp's version, updating resp, until +// its weights are no longer being copied. +func (p modelServingPoller) waitForStaging(ctx context.Context, resp *registerModelPackageResponse, progress func(*registerModelPackageResponse)) error { + statusReq := stagingStatusRequest{Model: resp.RegisteredModel, ModelVersion: resp.ModelVersion} + for { + if err := sleepOrCancel(ctx, p.interval); err != nil { + return err + } + if resp.weightsState() != stagingStateInProgress { + return nil + } + var status stagingStatusResponse + if err := p.post(ctx, stagingStatusPath, statusReq, &status); err != nil { + return err + } + resp.StagingStatus = status.Status + progress(resp) + if resp.weightsState() != stagingStateInProgress { + return nil } } } + +// post sends a request, repeating it after a transient failure the SDK doesn't +// retry: a wait of many minutes outlasts the SDK's own retries. Both requests air +// deploy polls with are safe to repeat, the registration because it is +// idempotent per key. +func (p modelServingPoller) post(ctx context.Context, path string, body, out any) error { + _, err := retry.With(ctx, retry.IsTransientRead, func() (struct{}, error) { + return struct{}{}, p.api.Do(ctx, http.MethodPost, path, p.headers, nil, body, out) + }) + return err +} diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go index 90732e14df5..f548faf2da2 100644 --- a/cmd/air/deployregister_test.go +++ b/cmd/air/deployregister_test.go @@ -10,19 +10,30 @@ import ( "testing" "time" + "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/retry" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// fakeModelServing serves registerModelPackage with the given responses in turn, -// repeating the last, and records every registration body. Other paths are served -// by endpoints when set, else get an empty object (the SDK's config probe). +// Canned fake responses that stand for an error rather than a body. +const ( + conflictResponse = "conflict" + serverErrorResponse = "server error" +) + +// fakeModelServing serves registerModelPackage with responses, and the staging +// status API with statuses, each in turn and then repeating the last; statuses +// default to SUCCEEDED. It records every request. Other paths are served by +// endpoints when set, else get an empty object (the SDK's config probe). type fakeModelServing struct { - t *testing.T - mu sync.Mutex - responses []string - bodies []registerModelPackageRequest - endpoints http.HandlerFunc + t *testing.T + mu sync.Mutex + responses []string + statuses []string + bodies []registerModelPackageRequest + statusRequests []stagingStatusRequest + endpoints http.HandlerFunc } func newFakeModelServing(t *testing.T, responses ...string) *fakeModelServing { @@ -30,31 +41,61 @@ func newFakeModelServing(t *testing.T, responses ...string) *fakeModelServing { } func (f *fakeModelServing) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != registerModelPackagePath { + switch r.URL.Path { + case registerModelPackagePath: + var body registerModelPackageRequest + if !f.decode(w, r, &body) { + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.bodies = append(f.bodies, body) + f.respond(w, f.responses, len(f.bodies)) + case stagingStatusPath: + var body stagingStatusRequest + if !f.decode(w, r, &body) { + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.statusRequests = append(f.statusRequests, body) + statuses := f.statuses + if len(statuses) == 0 { + statuses = []string{statusSucceeded} + } + f.respond(w, statuses, len(f.statusRequests)) + default: if f.endpoints != nil { f.endpoints(w, r) return } _, _ = w.Write([]byte(`{}`)) - return } +} + +// decode reads a request body into v, failing the request (not the handler +// goroutine) when it can't. +func (f *fakeModelServing) decode(w http.ResponseWriter, r *http.Request, v any) bool { raw, err := io.ReadAll(r.Body) - var body registerModelPackageRequest - if !assert.NoError(f.t, err) || !assert.NoError(f.t, json.Unmarshal(raw, &body)) { + if !assert.NoError(f.t, err) || !assert.NoError(f.t, json.Unmarshal(raw, v)) { w.WriteHeader(http.StatusInternalServerError) - return + return false } + return true +} - f.mu.Lock() - defer f.mu.Unlock() - f.bodies = append(f.bodies, body) - i := min(len(f.bodies), len(f.responses)) - 1 - if f.responses[i] == "" { +// respond writes the nth (1-based) of responses, repeating the last. +func (f *fakeModelServing) respond(w http.ResponseWriter, responses []string, n int) { + switch body := responses[min(n, len(responses))-1]; body { + case conflictResponse: w.WriteHeader(http.StatusConflict) _, _ = w.Write([]byte(`{"error_code": "RESOURCE_CONFLICT", "message": "The idempotency key can't be used for this registration. Use a new key."}`)) - return + case serverErrorResponse: + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error_code": "INTERNAL_ERROR", "message": "Something went wrong."}`)) + default: + _, _ = w.Write([]byte(body)) } - _, _ = w.Write([]byte(f.responses[i])) } func (f *fakeModelServing) start() *httptest.Server { @@ -76,8 +117,17 @@ const ( "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` stagingFailed = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, "staging_status": {"state": "FAILED", "model_artifact": {"state": "FAILED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + + statusInProgress = `{"status": {"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + statusSucceeded = `{"status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + statusFailed = `{"status": {"state": "FAILED", "model_artifact": {"state": "FAILED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` ) +// fastRetries shortens the pause before air deploy repeats a failed request. +func fastRetries(ctx context.Context) context.Context { + return env.Set(ctx, retry.IntervalMsVariable, "1") +} + func checkpointRequest() registerModelPackageRequest { return registerModelPackageRequest{ RegisteredModel: "main.models.qwen", @@ -91,45 +141,75 @@ func checkpointRequest() registerModelPackageRequest { } } -func TestRegisterUntilReadyRepeatsTheSameRequest(t *testing.T) { - fake := newFakeModelServing(t, stagingInProgress, stagingInProgress, stagingReady) +func registerTestCheckpoint(t *testing.T, fake *fakeModelServing, req registerModelPackageRequest) (*registerModelPackageResponse, []string, error) { + t.Helper() srv := fake.start() - var states []string - req := checkpointRequest() - req.RetryFailedStaging = true - resp, err := registerUntilReady(t.Context(), newTestWorkspaceClient(t, srv.URL), req, time.Millisecond, func(r *registerModelPackageResponse) { + resp, err := registerUntilReady(fastRetries(t.Context()), newTestWorkspaceClient(t, srv.URL), req, time.Millisecond, func(r *registerModelPackageResponse) { states = append(states, r.weightsState()) }) + return resp, states, err +} + +func TestRegisterUntilReadyPollsStagingThenFinalizes(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress, stagingReady) + fake.statuses = []string{statusInProgress, statusSucceeded} + req := checkpointRequest() + req.RetryFailedStaging = true + + resp, states, err := registerTestCheckpoint(t, fake, req) require.NoError(t, err) assert.True(t, resp.Ready) assert.Equal(t, "3", resp.ModelVersion) - assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED"}, states) + assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED", "SUCCEEDED"}, states) + // The staging status is polled for the registered version; the registration is + // repeated once the weights are staged, which finalizes the version. + assert.Equal(t, []stagingStatusRequest{{Model: "main.models.qwen", ModelVersion: "3"}, {Model: "main.models.qwen", ModelVersion: "3"}}, fake.statusRequests) bodies := fake.requests() - require.Len(t, bodies, 3) + require.Len(t, bodies, 2) assert.Equal(t, req, bodies[0]) - // Only the first request restages; the repeats continue the same registration. + // Only the first request restages; the repeat continues the same registration. req.RetryFailedStaging = false assert.Equal(t, req, bodies[1]) - assert.Equal(t, req, bodies[2]) } func TestRegisterUntilReadyReportsFailedStaging(t *testing.T) { fake := newFakeModelServing(t, stagingInProgress, stagingFailed) - srv := fake.start() + fake.statuses = []string{statusFailed} - resp, err := registerUntilReady(t.Context(), newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Millisecond, func(*registerModelPackageResponse) {}) + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) require.ErrorIs(t, err, errStagingFailed) assert.Equal(t, "3", resp.ModelVersion) + // The failure the status reports is confirmed by the registration itself. assert.Len(t, fake.requests(), 2) } +func TestRegisterUntilReadyRetriesTransientErrors(t *testing.T) { + fake := newFakeModelServing(t, serverErrorResponse, stagingInProgress, stagingReady) + fake.statuses = []string{serverErrorResponse, statusSucceeded} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Len(t, fake.requests(), 3) + assert.Len(t, fake.statusRequests, 2) +} + +func TestRegisterUntilReadyGivesUpOnPersistentErrors(t *testing.T) { + fake := newFakeModelServing(t, serverErrorResponse) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, "Something went wrong.") + assert.Len(t, fake.requests(), retry.MaxRetries+1) +} + func TestRegisterUntilReadyReturnsServerErrors(t *testing.T) { - srv := newFakeModelServing(t, "").start() + fake := newFakeModelServing(t, conflictResponse) - _, err := registerUntilReady(t.Context(), newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Millisecond, func(*registerModelPackageResponse) {}) + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) assert.ErrorContains(t, err, "The idempotency key can't be used for this registration") + assert.Len(t, fake.requests(), 1) } func TestRegisterUntilReadyStopsWhenCanceled(t *testing.T) { diff --git a/cmd/air/snapshot_package.go b/cmd/air/snapshot_package.go index c1dc9c5f89d..3a4f6ff6ddd 100644 --- a/cmd/air/snapshot_package.go +++ b/cmd/air/snapshot_package.go @@ -146,7 +146,9 @@ func snapshotFiles(ctx context.Context, repoPath string, includePaths []string, if output, err := cmd.CombinedOutput(); err != nil { return nil, fmt.Errorf("failed to initialize temporary git metadata: %w: %s", err, strings.TrimSpace(string(output))) } - args = []string{"--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} + // -C makes ls-files list paths relative to repoPath; without it they are + // relative to the process's working directory, which may be a subdirectory. + args = []string{"-C", repoPath, "--git-dir", gitDir, "--work-tree", repoPath, "ls-files", "-z", "--cached", "--others", "--exclude-standard"} } if len(includePaths) > 0 { diff --git a/cmd/air/snapshot_package_test.go b/cmd/air/snapshot_package_test.go index 068997e2671..37c7bdbfec3 100644 --- a/cmd/air/snapshot_package_test.go +++ b/cmd/air/snapshot_package_test.go @@ -324,3 +324,21 @@ func TestSnapshotFiles_RejectsIncludePathsMatchingNothing(t *testing.T) { assert.Contains(t, err.Error(), "contains no files") assert.Contains(t, err.Error(), "include_paths") } + +func TestSnapshotFilesOutsideGitFromASubdirectory(t *testing.T) { + dir := t.TempDir() + writeRepoFile(t, dir, ".gitignore", "/creds.json\n") + writeRepoFile(t, dir, "creds.json", "{}") + writeRepoFile(t, dir, "agent.py", "x") + writeRepoFile(t, dir, "tools/a.py", "a") + // The listing must be relative to the root, not to the working directory. + t.Chdir(filepath.Join(dir, "tools")) + + files, err := snapshotFiles(t.Context(), dir, nil, false) + require.NoError(t, err) + var rels []string + for _, f := range files { + rels = append(rels, filepath.ToSlash(f.rel)) + } + assert.ElementsMatch(t, []string{".gitignore", "agent.py", "tools/a.py"}, rels) +} From 8d2d9f9272d5cfdb04a786b95028f24c31e42e66 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 15 Nov 2026 06:38:05 +0000 Subject: [PATCH 03/11] air deploy: retry each endpoint poll, stop on a not-ready code model - Poll the endpoint per GET with libs/retry, so transient failures spread over a long start don't end the wait. - Don't resend a model from code that isn't ready after registering. - Keep git's stderr in check-ignore failures; reject include_paths patterns and padding; don't mark a local client error retryable. Co-authored-by: Isaac --- cmd/air/deploycode.go | 4 +++- cmd/air/deployconfig.go | 7 +++--- cmd/air/deployconfig_test.go | 4 +++- cmd/air/deployendpoint.go | 39 +++++++++++++++++++++++++++------- cmd/air/deployendpoint_test.go | 33 ++++++++++++++++++---------- cmd/air/deployregister.go | 7 +++++- cmd/air/deployregister_test.go | 9 ++++++++ 7 files changed, 78 insertions(+), 25 deletions(-) diff --git a/cmd/air/deploycode.go b/cmd/air/deploycode.go index 433941d77af..721fadd7825 100644 --- a/cmd/air/deploycode.go +++ b/cmd/air/deploycode.go @@ -101,15 +101,17 @@ func ignoredTrackedFiles(ctx context.Context, root string, files []snapshotFile) paths.WriteString(filepath.ToSlash(f.rel)) paths.WriteByte(0) } + var stderr bytes.Buffer cmd := exec.CommandContext(ctx, "git", "-C", root, "check-ignore", "--no-index", "-z", "--stdin") cmd.Stdin = &paths + cmd.Stderr = &stderr out, err := cmd.Output() // check-ignore exits 1 when no path is ignored. if exitErr, ok := errors.AsType[*exec.ExitError](err); ok && exitErr.ExitCode() == 1 { return nil, nil } if err != nil { - return nil, fmt.Errorf("failed to evaluate git ignore rules: %w", err) + return nil, fmt.Errorf("failed to evaluate git ignore rules: %w: %s", err, strings.TrimSpace(stderr.String())) } var ignored []string for raw := range bytes.SplitSeq(out, []byte{0}) { diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index 29619ef9db7..9c3bdac4cf0 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -250,10 +250,11 @@ func (c *deployConfig) validateSource() error { return errors.New("code_source.include_paths is required: list the files and directories to send, e.g. include_paths: [agent.py, tools]") } // git reads a leading ':' as pathspec magic, such as ":!x" for everything - // but x, which would send files the list doesn't name. + // but x, and expands globs, either of which would send files the list + // doesn't name; padding would match nothing. for _, p := range c.CodeSource.IncludePaths { - if strings.HasPrefix(strings.TrimSpace(p), ":") { - return fmt.Errorf("code_source.include_paths must name files and directories, got %q", p) + if strings.HasPrefix(p, ":") || strings.ContainsAny(p, "*?[") || p != strings.TrimSpace(p) { + return fmt.Errorf("code_source.include_paths must name files and directories, without patterns or surrounding spaces, got %q", p) } } snapshot := snapshotSourceConfig{RootPath: c.CodeSource.RootPath, IncludePaths: c.CodeSource.IncludePaths} diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index 6d5efc47302..722d1acb350 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -189,7 +189,9 @@ func TestDeployConfigSourceFields(t *testing.T) { {"code entry in a directory", "model: {type: code, entry: src/agent.py}\ncode_source: {root_path: .}\n", "model.entry must name a .py file directly under code_source.root_path"}, {"code entry not python", "model: {type: code, entry: agent.ipynb}\ncode_source: {root_path: .}\n", "model.entry must name a .py file"}, {"code without include_paths", "model: {type: code, entry: agent.py}\ncode_source: {root_path: .}\n", "code_source.include_paths is required: list the files and directories to send"}, - {"code include pathspec magic", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\":!creds.json\"]}\n", `code_source.include_paths must name files and directories, got ":!creds.json"`}, + {"code include pathspec magic", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\":!creds.json\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got ":!creds.json"`}, + {"code include glob", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\"*\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got "*"`}, + {"code include padded", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\" agent.py\"]}\n", `got " agent.py"`}, {"code include traversal", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [../x]}\n", "code_source.include_paths cannot contain '..' traversal"}, {"saved model with task", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\ntask: llm/v1/chat\n", "task can't be set when model.type is saved_model_dir"}, {"saved model with environment", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\nenvironment: {version: 6}\n", "environment can't be set when model.type is saved_model_dir"}, diff --git a/cmd/air/deployendpoint.go b/cmd/air/deployendpoint.go index e240dbc1f0b..1effc2a8303 100644 --- a/cmd/air/deployendpoint.go +++ b/cmd/air/deployendpoint.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/serving" ) @@ -40,13 +41,34 @@ func getEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string return endpoint, err } -// waitNotUpdating waits until the endpoint has no update in progress. The SDK's -// wait stops at the first failed poll, so a transient failure starts it again: -// an endpoint can take many minutes to start. Creates and config updates aren't -// repeated this way, since a repeat after an applied write would fail. +// waitNotUpdating waits until the endpoint has no update in progress, calling +// progress on every poll. It polls as the SDK's WaitGetServingEndpointNotUpdating +// does, except that each poll repeats a transient failure, which ends the SDK's +// wait: an endpoint can take many minutes to start. Creates and config updates +// aren't repeated this way, since a repeat after an applied write would fail. func waitNotUpdating(ctx context.Context, w *databricks.WorkspaceClient, name string, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { - return retry.With(ctx, retry.IsTransientRead, func() (*serving.ServingEndpointDetailed, error) { - return w.ServingEndpoints.WaitGetServingEndpointNotUpdating(ctx, name, untilDeadline(ctx), progress) + return retries.Poll(ctx, untilDeadline(ctx), func() (*serving.ServingEndpointDetailed, *retries.Err) { + endpoint, err := retry.With(ctx, retry.IsTransientRead, func() (*serving.ServingEndpointDetailed, error) { + return w.ServingEndpoints.GetByName(ctx, name) + }) + if err != nil { + return nil, retries.Halt(err) + } + if progress != nil { + progress(endpoint) + } + var state serving.EndpointStateConfigUpdate + if endpoint.State != nil { + state = endpoint.State.ConfigUpdate + } + switch state { + case serving.EndpointStateConfigUpdateNotUpdating: + return endpoint, nil + case serving.EndpointStateConfigUpdateUpdateFailed, serving.EndpointStateConfigUpdateUpdateCanceled: + return nil, retries.Halt(fmt.Errorf("the endpoint's update ended in %s; check its events and logs in the Serving UI", state)) + default: + return nil, retries.Continues(string(state)) + } }) } @@ -131,8 +153,9 @@ func waitServes(ctx context.Context, w *databricks.WorkspaceClient, name string, return checkServes(endpoint, entity) } -// errEndpointNotReady reports an endpoint that serves the version but isn't ready, -// which running air deploy again doesn't change. +// errEndpointNotReady reports an endpoint that serves the version but isn't ready. +// Running air deploy again doesn't make it ready; it reports success once the +// endpoint recovers, so the error isn't marked retryable. var errEndpointNotReady = errors.New("endpoint isn't ready") // checkServes returns endpoint if it serves only entity and is ready to serve. diff --git a/cmd/air/deployendpoint_test.go b/cmd/air/deployendpoint_test.go index e267a1bdaf4..0a4dc57e019 100644 --- a/cmd/air/deployendpoint_test.go +++ b/cmd/air/deployendpoint_test.go @@ -27,8 +27,11 @@ type fakeEndpoints struct { writes []string // rawWrites holds the body of every create and config update, as sent. rawWrites []string - // failGets is how many GETs answer 500 before the endpoint is served again. - failGets int + // gets counts the GETs of an existing endpoint. Those numbered in failOnGets + // answer 500, and from readyAfterGets on (when set) the endpoint is ready. + gets int + failOnGets map[int]bool + readyAfterGets int } func newFakeEndpoints(t *testing.T, existing *serving.ServingEndpointDetailed) *fakeEndpoints { @@ -66,11 +69,16 @@ func (f *fakeEndpoints) ServeHTTP(w http.ResponseWriter, r *http.Request) { f.endpoint.PendingConfig = nil f.endpoint.State = &f.finalState case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, servingEndpointsPath+"/"): - if f.failGets > 0 && f.endpoint != nil { - f.failGets-- - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(`{"error_code": "INTERNAL_ERROR", "message": "Something went wrong."}`)) - return + if f.endpoint != nil { + f.gets++ + if f.failOnGets[f.gets] { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"error_code": "INTERNAL_ERROR", "message": "Something went wrong."}`)) + return + } + if f.readyAfterGets > 0 && f.gets >= f.readyAfterGets { + f.endpoint.State = &readyState + } } if f.endpoint == nil { w.WriteHeader(http.StatusNotFound) @@ -252,16 +260,19 @@ func TestCheckEndpointAllowsAMissingEndpoint(t *testing.T) { assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen")) } -func TestServeOnEndpointWaitsThroughATransientFailure(t *testing.T) { +func TestServeOnEndpointWaitsThroughTransientFailures(t *testing.T) { fake := newFakeEndpoints(t, nil) - // The create succeeds; the first poll of the wait then fails. - fake.failGets = 1 + // The endpoint starts over three polls, and the first GET of each fails: more + // failures than one request may retry, spread over the wait. + fake.finalState = serving.EndpointState{Ready: serving.EndpointStateReadyNotReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress} + fake.failOnGets = map[int]bool{1: true, 3: true, 5: true} + fake.readyAfterGets = 6 srv := fake.start() _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), nil) require.NoError(t, err) assert.Equal(t, []string{"create"}, fake.writes) - assert.Zero(t, fake.failGets) + assert.Equal(t, 6, fake.gets) } func TestServeOnEndpointReportsAFailedUpdate(t *testing.T) { diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go index a5ae7277553..cbc128fadb8 100644 --- a/cmd/air/deployregister.go +++ b/cmd/air/deployregister.go @@ -164,7 +164,7 @@ func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) r func registerUntilReady(ctx context.Context, w *databricks.WorkspaceClient, req registerModelPackageRequest, interval time.Duration, progress func(*registerModelPackageResponse)) (*registerModelPackageResponse, error) { apiClient, err := client.New(w.Config) if err != nil { - return nil, fmt.Errorf("failed to create API client: %w", err) + return nil, withCode(codeInternalError, false, fmt.Errorf("failed to create API client: %w", err)) } p := modelServingPoller{api: apiClient, headers: auth.WorkspaceIDHeaders(w.Config), interval: interval} var last *registerModelPackageResponse @@ -181,6 +181,11 @@ func registerUntilReady(ctx context.Context, w *databricks.WorkspaceClient, req if resp.weightsState() == stagingStateFailed { return &resp, errStagingFailed } + // Only a checkpoint waits on staging; repeating any other registration + // would resend its files to no effect. + if req.Checkpoint == nil { + return &resp, fmt.Errorf("version %s of %s isn't ready after registering", stripControl(resp.ModelVersion), req.RegisteredModel) + } // Only the first request restages a failed checkpoint: repeating it would // restart a staging that keeps failing instead of reporting the failure. req.RetryFailedStaging = false diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go index f548faf2da2..22e4f6b567b 100644 --- a/cmd/air/deployregister_test.go +++ b/cmd/air/deployregister_test.go @@ -196,6 +196,15 @@ func TestRegisterUntilReadyRetriesTransientErrors(t *testing.T) { assert.Len(t, fake.statusRequests, 2) } +func TestRegisterUntilReadyDoesNotRepeatANotReadyModelFromCode(t *testing.T) { + fake := newFakeModelServing(t, `{"model_id": "m-2", "registered_model": "a.b.c", "model_version": "1", "ready": false}`) + req := registerModelPackageRequest{RegisteredModel: "a.b.c", IdempotencyKey: "k", Code: &modelPackageCode{EntryFile: "agent.py"}} + + _, _, err := registerTestCheckpoint(t, fake, req) + assert.ErrorContains(t, err, "version 1 of a.b.c isn't ready after registering") + assert.Len(t, fake.requests(), 1) +} + func TestRegisterUntilReadyGivesUpOnPersistentErrors(t *testing.T) { fake := newFakeModelServing(t, serverErrorResponse) From 261b2028fb2eab0149eb41b31b90dfb1ec6daa68 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Sun, 15 Nov 2026 16:43:22 +0000 Subject: [PATCH 04/11] air deploy: register once, poll staging, then finalize the version The server now leaves a registered version pending: air deploy registers it once, polls the read-only staging status until the weights are copied or fail, then calls finalizeModelPackage. A status without a state is waited out, a finalize failure prints the command that continues, and the finalize response must name the same version. Co-authored-by: Isaac --- cmd/air/deploy.go | 20 +++- cmd/air/deploy_test.go | 40 +++++-- cmd/air/deployregister.go | 150 +++++++++++++++++-------- cmd/air/deployregister_test.go | 200 ++++++++++++++++++++++++++------- 4 files changed, 305 insertions(+), 105 deletions(-) diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go index 8ac4dd903ba..4d2b0d876f5 100644 --- a/cmd/air/deploy.go +++ b/cmd/air/deploy.go @@ -289,7 +289,7 @@ func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPacka return nil } -// registerDeployment registers the version and waits until it is ready, showing +// registerDeployment registers the version and makes it ready to serve, showing // progress on a spinner when show is true. Its errors say how to continue. func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, code *modelPackageCode, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { // Printed to stderr in every output mode, before the first request, so a run @@ -306,7 +306,7 @@ func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg sp.Update(registrationProgress(cfg, resp)) } } - resp, err := registerUntilReady(ctx, w, req, pollInterval, progress) + resp, err := registerAndFinalize(ctx, w, req, pollInterval, progress) if err == nil { return resp, nil } @@ -320,11 +320,18 @@ func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg } if stopped := deployStopped(ctx, err); stopped != nil { continuing := "" - if resp != nil && resp.weightsState() != "" { - continuing = " Copying the weights continues, and the version is finalized when you continue." + switch { + case resp == nil: + case resp.weightsState() == stagingStateSucceeded: + continuing = " The weights are copied, so continuing finalizes the version." + case resp.weightsState() != "": + continuing = " Copying the weights continues meanwhile." } return nil, fmt.Errorf("%w while registering a version of %s.%s To continue, run:\n %s", stopped, cfg.RegisteredModel, continuing, resume) } + if _, ok := errors.AsType[*finalizeError](err); ok { + return nil, fmt.Errorf("finalizing version %s of %s failed: %w\n\nTo try again, run:\n %s", stripControl(resp.ModelVersion), cfg.RegisteredModel, err, resume) + } failed := fmt.Errorf("registering a version of %s with idempotency key %s failed: %w", cfg.RegisteredModel, key, err) // A retryable failure is retried with the same key, so no other version is // registered. @@ -396,8 +403,11 @@ func validateDeployFlags(cfg *deployConfig, opts deployOptions) error { // registrationProgress describes a registration response on the spinner. func registrationProgress(cfg *deployConfig, resp *registerModelPackageResponse) string { state := resp.weightsState() - if state == "" { + switch state { + case "": return fmt.Sprintf("Registering version %s of %s…", stripControl(resp.ModelVersion), cfg.RegisteredModel) + case stagingStateSucceeded: + return fmt.Sprintf("Finalizing version %s of %s…", stripControl(resp.ModelVersion), cfg.RegisteredModel) } return fmt.Sprintf("Copying the weights into version %s of %s (%s)…", stripControl(resp.ModelVersion), cfg.RegisteredModel, stripControl(state)) } diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go index 2ae6ac0917c..05cbdab8d6b 100644 --- a/cmd/air/deploy_test.go +++ b/cmd/air/deploy_test.go @@ -35,8 +35,9 @@ func withDeadline(t *testing.T) context.Context { return ctx } -// deployServer serves registerModelPackage with responses and the serving -// endpoints API from endpoints. +// deployServer serves registerModelPackage with responses, the staging status +// and finalizeModelPackage with their defaults, and the serving endpoints API +// from endpoints. func deployServer(t *testing.T, endpoints *fakeEndpoints, responses ...string) (*httptest.Server, *fakeModelServing) { t.Helper() fake := newFakeModelServing(t, responses...) @@ -70,7 +71,7 @@ func runDeployCmd(t *testing.T, out flags.Output, buf *bytes.Buffer, srvURL, con func TestDeployCheckpointJSON(t *testing.T) { endpoints := newFakeEndpoints(t, nil) - srv, fake := deployServer(t, endpoints, stagingInProgress, stagingReady) + srv, fake := deployServer(t, endpoints, stagingInProgress) var buf bytes.Buffer err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") @@ -91,9 +92,9 @@ func TestDeployCheckpointJSON(t *testing.T) { }, got.Data) bodies := fake.requests() - require.Len(t, bodies, 2) + require.Len(t, bodies, 1) assert.Equal(t, "key-1", bodies[0].IdempotencyKey) - assert.Equal(t, bodies[0], bodies[1]) + assert.Equal(t, []finalizeModelPackageRequest{{RegisteredModel: "main.models.qwen", ModelVersion: "3"}}, fake.finalizeRequests) assert.Equal(t, []string{"create"}, endpoints.writes) served := endpoints.endpoint.Config.ServedEntities[0] assert.Equal(t, "3", served.EntityVersion) @@ -156,15 +157,14 @@ func TestDeployPrintsTheKeyBeforeRegisteringInJSONMode(t *testing.T) { assert.Contains(t, stdout.String(), `"code": "RESOURCE_CONFLICT"`) } -func TestDeployRetriesFailedStagingOnce(t *testing.T) { - srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress, stagingReady) +func TestDeploySendsRetryFailedStaging(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress) var buf bytes.Buffer require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "retry-failed-staging", "true")) bodies := fake.requests() - require.Len(t, bodies, 2) + require.Len(t, bodies, 1) assert.True(t, bodies[0].RetryFailedStaging) - assert.False(t, bodies[1].RetryFailedStaging) } func TestDeployFailedStagingSaysHowToRetry(t *testing.T) { @@ -253,6 +253,28 @@ func TestDeployTimesOutWhileCopyingTheWeights(t *testing.T) { assert.Contains(t, got.Error.Message, "air deploy timed out while registering a version of main.models.qwen. Copying the weights continues") assert.Contains(t, got.Error.Message, " --idempotency-key key-1") assert.Len(t, fake.requests(), 1) + assert.Empty(t, fake.finalizeRequests) +} + +func TestDeployFinalizeFailureSaysHowToRetry(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingSucceeded) + fake.finalizations = []string{notFinalizableResponse} + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "finalizing version 3 of main.models.qwen failed: Version 3 of model main.models.qwen can't be finalized yet") + assert.Contains(t, err.Error(), "To try again, run:\n databricks air deploy -f ") + assert.Contains(t, err.Error(), " --idempotency-key key-1") +} + +func TestDeployWarnsWhenTheFinalizedEnvironmentFailedToStage(t *testing.T) { + srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress) + fake.finalizations = []string{strings.Replace(finalizedResponse, `"environment_artifact": {"state": "IN_PROGRESS"}`, `"environment_artifact": {"state": "FAILED"}`, 1)} + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig)) + assert.Contains(t, buf.String(), "Warning: the version's serving environment couldn't be prepared ahead of time") } func TestDeployRetryableRegistrationFailureSaysHowToRetry(t *testing.T) { diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go index cbc128fadb8..f3f24c99693 100644 --- a/cmd/air/deployregister.go +++ b/cmd/air/deployregister.go @@ -13,24 +13,33 @@ import ( "github.com/databricks/databricks-sdk-go/client" ) -// registerModelPackagePath registers a model whose files Model Serving writes. It -// runs as the caller and is idempotent per key: a repeat request with the same key -// and inputs continues the registration, and finalizes a checkpoint's version once -// its weights are staged. Staging never finalizes the version by itself. +// registerModelPackagePath registers a model whose files Model Serving writes and +// starts copying a checkpoint's weights into the version. It runs as the caller and +// is idempotent per key: a repeat request with the same key and inputs continues +// the registration. It leaves the version pending. const registerModelPackagePath = "/api/2.0/serving-endpoints:registerModelPackage" // stagingStatusPath reports the recorded staging progress of a model version, a // read. const stagingStatusPath = "/api/2.0/serving-endpoints:checkModelDeploymentStagingStatus" +// finalizeModelPackagePath finalizes a registered version once its weights are +// copied, so it can be served. It is idempotent: a finalized version is returned +// as it is. +const finalizeModelPackagePath = "/api/2.0/serving-endpoints:finalizeModelPackage" + // registrationPollInterval is how often air deploy checks on a checkpoint's // weights while they are staged. const registrationPollInterval = 15 * time.Second // ModelDeploymentStagingStatus states air deploy acts on. const ( + // stagingStateNotStaged: no staging record exists for the version. + stagingStateNotStaged = "NOT_STAGED" // stagingStateInProgress: an upload is queued, running or waiting to be retried. stagingStateInProgress = "IN_PROGRESS" + // stagingStateSucceeded: the upload is recorded as done. + stagingStateSucceeded = "SUCCEEDED" // stagingStateFailed: the upload failed and won't be retried automatically. stagingStateFailed = "FAILED" ) @@ -95,6 +104,19 @@ type artifactStagingStatus struct { State string `json:"state,omitempty"` } +// finalizeModelPackageRequest is the FinalizeModelPackageRequest body. +type finalizeModelPackageRequest struct { + RegisteredModel string `json:"registered_model"` + ModelVersion string `json:"model_version"` +} + +// finalizeModelPackageResponse is the FinalizeModelPackageResponse body, the +// fields air deploy reads. +type finalizeModelPackageResponse struct { + ModelVersion string `json:"model_version"` + StagingStatus *modelStagingStatus `json:"staging_status,omitempty"` +} + // stagingStatusRequest is the CheckModelDeploymentStagingStatusRequest body. type stagingStatusRequest struct { Model string `json:"model"` @@ -128,6 +150,14 @@ func (r *registerModelPackageResponse) environmentState() string { // into the version. Rerunning with --retry-failed-staging stages them again. var errStagingFailed = errors.New("staging failed") +// finalizeError is a failed request to finalize the registered version. +// Rerunning with the same key registers the same version again and retries it. +type finalizeError struct{ err error } + +func (e *finalizeError) Error() string { return e.err.Error() } + +func (e *finalizeError) Unwrap() error { return e.err } + // registrationRequest builds the request that registers cfg's model under key. // code is the collected model from code, and is nil for the other sources. func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) registerModelPackageRequest { @@ -154,45 +184,69 @@ func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) r return req } -// registerUntilReady registers req's version and waits until it is ready to -// serve, calling progress with every registration response and staging update. -// The other sources are ready after the first request. A checkpoint's version -// becomes ready once its weights are staged: while they copy, it polls the staging -// status, a read, then repeats the registration, which finalizes the version. On -// error it returns the last response it got, nil before the first one, and -// errStagingFailed when staging failed. -func registerUntilReady(ctx context.Context, w *databricks.WorkspaceClient, req registerModelPackageRequest, interval time.Duration, progress func(*registerModelPackageResponse)) (*registerModelPackageResponse, error) { +// registerAndFinalize registers req's version and makes it ready to serve, +// calling progress with the registration response and every staging update. A +// checkpoint's weights are copied into the version after registering: while they +// are, it polls the staging status, a read. It then finalizes the version, for +// every source; the server checks that the version is ready to finalize. A +// version that is already ready, such as one an earlier run finalized, is +// returned as it is. On error it returns the last response it got, nil before +// the first one; errStagingFailed when staging failed, and a *finalizeError when +// finalizing did. +func registerAndFinalize(ctx context.Context, w *databricks.WorkspaceClient, req registerModelPackageRequest, interval time.Duration, progress func(*registerModelPackageResponse)) (*registerModelPackageResponse, error) { apiClient, err := client.New(w.Config) if err != nil { return nil, withCode(codeInternalError, false, fmt.Errorf("failed to create API client: %w", err)) } p := modelServingPoller{api: apiClient, headers: auth.WorkspaceIDHeaders(w.Config), interval: interval} - var last *registerModelPackageResponse - for { - var resp registerModelPackageResponse - if err := p.post(ctx, registerModelPackagePath, req, &resp); err != nil { - return last, err - } - last = &resp - progress(&resp) - if resp.Ready { - return &resp, nil - } - if resp.weightsState() == stagingStateFailed { - return &resp, errStagingFailed + var resp registerModelPackageResponse + if err := p.post(ctx, registerModelPackagePath, req, &resp); err != nil { + return nil, err + } + if resp.ModelVersion == "" { + return nil, withCode(codeInternalError, true, fmt.Errorf("registering a version of %s returned no version", req.RegisteredModel)) + } + progress(&resp) + if resp.Ready { + return &resp, nil + } + if req.Checkpoint != nil { + if err := p.waitForWeights(ctx, req.RegisteredModel, &resp, progress); err != nil { + return &resp, err } - // Only a checkpoint waits on staging; repeating any other registration - // would resend its files to no effect. - if req.Checkpoint == nil { - return &resp, fmt.Errorf("version %s of %s isn't ready after registering", stripControl(resp.ModelVersion), req.RegisteredModel) + } + if resp.weightsState() == stagingStateFailed { + return &resp, errStagingFailed + } + var finalized finalizeModelPackageResponse + finalizeReq := finalizeModelPackageRequest{RegisteredModel: req.RegisteredModel, ModelVersion: resp.ModelVersion} + if err := p.post(ctx, finalizeModelPackagePath, finalizeReq, &finalized); err != nil { + return &resp, &finalizeError{err} + } + if finalized.ModelVersion != resp.ModelVersion { + return &resp, &finalizeError{withCode(codeInternalError, true, fmt.Errorf("the server finalized version %q instead", stripControl(finalized.ModelVersion)))} + } + resp.Ready = true + resp.StagingStatus = mergeStagingStatus(resp.StagingStatus, finalized.StagingStatus) + return &resp, nil +} + +// mergeStagingStatus is newer, with each artifact it doesn't report kept from +// older. +func mergeStagingStatus(older, newer *modelStagingStatus) *modelStagingStatus { + if newer == nil { + return older + } + merged := *newer + if older != nil { + if merged.ModelArtifact == nil { + merged.ModelArtifact = older.ModelArtifact } - // Only the first request restages a failed checkpoint: repeating it would - // restart a staging that keeps failing instead of reporting the failure. - req.RetryFailedStaging = false - if err := p.waitForStaging(ctx, &resp, progress); err != nil { - return &resp, err + if merged.EnvironmentArtifact == nil { + merged.EnvironmentArtifact = older.EnvironmentArtifact } } + return &merged } // modelServingPoller sends the requests of a registration that can take many @@ -203,33 +257,33 @@ type modelServingPoller struct { interval time.Duration } -// waitForStaging polls the staging status of resp's version, updating resp, until -// its weights are no longer being copied. -func (p modelServingPoller) waitForStaging(ctx context.Context, resp *registerModelPackageResponse, progress func(*registerModelPackageResponse)) error { - statusReq := stagingStatusRequest{Model: resp.RegisteredModel, ModelVersion: resp.ModelVersion} +// waitForWeights polls the staging status of resp's version of model, updating +// resp, until its weights are copied, failed to copy, or have no staging record. +// A state it doesn't know, or a status without one, is waited out like a copy in +// progress, until the context ends. +func (p modelServingPoller) waitForWeights(ctx context.Context, model string, resp *registerModelPackageResponse, progress func(*registerModelPackageResponse)) error { + statusReq := stagingStatusRequest{Model: model, ModelVersion: resp.ModelVersion} for { + switch resp.weightsState() { + case stagingStateSucceeded, stagingStateFailed, stagingStateNotStaged: + return nil + } if err := sleepOrCancel(ctx, p.interval); err != nil { return err } - if resp.weightsState() != stagingStateInProgress { - return nil - } var status stagingStatusResponse if err := p.post(ctx, stagingStatusPath, statusReq, &status); err != nil { return err } - resp.StagingStatus = status.Status + resp.StagingStatus = mergeStagingStatus(resp.StagingStatus, status.Status) progress(resp) - if resp.weightsState() != stagingStateInProgress { - return nil - } } } // post sends a request, repeating it after a transient failure the SDK doesn't -// retry: a wait of many minutes outlasts the SDK's own retries. Both requests air -// deploy polls with are safe to repeat, the registration because it is -// idempotent per key. +// retry: a wait of many minutes outlasts the SDK's own retries. Every request air +// deploy sends here is safe to repeat: the registration is idempotent per key, +// the status is a read, and finalizing a finalized version returns it. func (p modelServingPoller) post(ctx context.Context, path string, body, out any) error { _, err := retry.With(ctx, retry.IsTransientRead, func() (struct{}, error) { return struct{}{}, p.api.Do(ctx, http.MethodPost, path, p.headers, nil, body, out) diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go index 22e4f6b567b..2efeb1a9028 100644 --- a/cmd/air/deployregister_test.go +++ b/cmd/air/deployregister_test.go @@ -3,9 +3,11 @@ package aircmd import ( "context" "encoding/json" + "errors" "io" "net/http" "net/http/httptest" + "strings" "sync" "testing" "time" @@ -18,22 +20,26 @@ import ( // Canned fake responses that stand for an error rather than a body. const ( - conflictResponse = "conflict" - serverErrorResponse = "server error" + conflictResponse = "conflict" + serverErrorResponse = "server error" + notFinalizableResponse = "not finalizable" ) -// fakeModelServing serves registerModelPackage with responses, and the staging -// status API with statuses, each in turn and then repeating the last; statuses -// default to SUCCEEDED. It records every request. Other paths are served by -// endpoints when set, else get an empty object (the SDK's config probe). +// fakeModelServing serves registerModelPackage with responses, the staging status +// API with statuses, and finalizeModelPackage with finalizations, each in turn and +// then repeating the last; statuses default to SUCCEEDED, and finalizations to the +// finalized version. It records every request. Other paths are served by endpoints +// when set, else get an empty object (the SDK's config probe). type fakeModelServing struct { - t *testing.T - mu sync.Mutex - responses []string - statuses []string - bodies []registerModelPackageRequest - statusRequests []stagingStatusRequest - endpoints http.HandlerFunc + t *testing.T + mu sync.Mutex + responses []string + statuses []string + finalizations []string + bodies []registerModelPackageRequest + statusRequests []stagingStatusRequest + finalizeRequests []finalizeModelPackageRequest + endpoints http.HandlerFunc } func newFakeModelServing(t *testing.T, responses ...string) *fakeModelServing { @@ -64,6 +70,19 @@ func (f *fakeModelServing) ServeHTTP(w http.ResponseWriter, r *http.Request) { statuses = []string{statusSucceeded} } f.respond(w, statuses, len(f.statusRequests)) + case finalizeModelPackagePath: + var body finalizeModelPackageRequest + if !f.decode(w, r, &body) { + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.finalizeRequests = append(f.finalizeRequests, body) + finalizations := f.finalizations + if len(finalizations) == 0 { + finalizations = []string{finalizedResponse} + } + f.respond(w, finalizations, len(f.finalizeRequests)) default: if f.endpoints != nil { f.endpoints(w, r) @@ -93,6 +112,9 @@ func (f *fakeModelServing) respond(w http.ResponseWriter, responses []string, n case serverErrorResponse: w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"error_code": "INTERNAL_ERROR", "message": "Something went wrong."}`)) + case notFinalizableResponse: + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error_code": "BAD_REQUEST", "message": "Version 3 of model main.models.qwen can't be finalized yet: the checkpoint's weights haven't been copied into it."}`)) default: _, _ = w.Write([]byte(body)) } @@ -113,6 +135,8 @@ func (f *fakeModelServing) requests() []registerModelPackageRequest { const ( stagingInProgress = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, "staging_status": {"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + stagingSucceeded = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, + "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` stagingReady = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": true, "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` stagingFailed = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, @@ -121,6 +145,9 @@ const ( statusInProgress = `{"status": {"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "SUCCEEDED"}}}` statusSucceeded = `{"status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` statusFailed = `{"status": {"state": "FAILED", "model_artifact": {"state": "FAILED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + + finalizedResponse = `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", + "staging_status": {"state": "IN_PROGRESS", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "IN_PROGRESS"}}}` ) // fastRetries shortens the pause before air deploy repeats a failed request. @@ -145,14 +172,14 @@ func registerTestCheckpoint(t *testing.T, fake *fakeModelServing, req registerMo t.Helper() srv := fake.start() var states []string - resp, err := registerUntilReady(fastRetries(t.Context()), newTestWorkspaceClient(t, srv.URL), req, time.Millisecond, func(r *registerModelPackageResponse) { + resp, err := registerAndFinalize(fastRetries(t.Context()), newTestWorkspaceClient(t, srv.URL), req, time.Millisecond, func(r *registerModelPackageResponse) { states = append(states, r.weightsState()) }) return resp, states, err } -func TestRegisterUntilReadyPollsStagingThenFinalizes(t *testing.T) { - fake := newFakeModelServing(t, stagingInProgress, stagingReady) +func TestRegisterAndFinalizePollsStagingThenFinalizes(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) fake.statuses = []string{statusInProgress, statusSucceeded} req := checkpointRequest() req.RetryFailedStaging = true @@ -161,51 +188,136 @@ func TestRegisterUntilReadyPollsStagingThenFinalizes(t *testing.T) { require.NoError(t, err) assert.True(t, resp.Ready) assert.Equal(t, "3", resp.ModelVersion) - assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED", "SUCCEEDED"}, states) + assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED"}, states) - // The staging status is polled for the registered version; the registration is - // repeated once the weights are staged, which finalizes the version. + // One registration; the status is polled for the registered version until the + // weights are copied, and the version is then finalized. + assert.Equal(t, []registerModelPackageRequest{req}, fake.requests()) assert.Equal(t, []stagingStatusRequest{{Model: "main.models.qwen", ModelVersion: "3"}, {Model: "main.models.qwen", ModelVersion: "3"}}, fake.statusRequests) - bodies := fake.requests() - require.Len(t, bodies, 2) - assert.Equal(t, req, bodies[0]) - // Only the first request restages; the repeat continues the same registration. - req.RetryFailedStaging = false - assert.Equal(t, req, bodies[1]) + assert.Equal(t, []finalizeModelPackageRequest{{RegisteredModel: "main.models.qwen", ModelVersion: "3"}}, fake.finalizeRequests) } -func TestRegisterUntilReadyReportsFailedStaging(t *testing.T) { - fake := newFakeModelServing(t, stagingInProgress, stagingFailed) +func TestRegisterAndFinalizeReportsTheFinalizedStagingStatus(t *testing.T) { + fake := newFakeModelServing(t, stagingSucceeded) + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + // The environment is still being staged when the version is finalized. + assert.Equal(t, stagingStateInProgress, resp.environmentState()) + assert.Empty(t, fake.statusRequests) +} + +func TestRegisterAndFinalizeReturnsAReadyVersion(t *testing.T) { + fake := newFakeModelServing(t, stagingReady) + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Empty(t, fake.statusRequests) + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeReportsFailedStaging(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) fake.statuses = []string{statusFailed} resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) require.ErrorIs(t, err, errStagingFailed) assert.Equal(t, "3", resp.ModelVersion) - // The failure the status reports is confirmed by the registration itself. - assert.Len(t, fake.requests(), 2) + assert.Len(t, fake.requests(), 1) + assert.Empty(t, fake.finalizeRequests) } -func TestRegisterUntilReadyRetriesTransientErrors(t *testing.T) { - fake := newFakeModelServing(t, serverErrorResponse, stagingInProgress, stagingReady) - fake.statuses = []string{serverErrorResponse, statusSucceeded} +func TestRegisterAndFinalizeReportsAFailureTheRegistrationReports(t *testing.T) { + fake := newFakeModelServing(t, stagingFailed) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.ErrorIs(t, err, errStagingFailed) + assert.Empty(t, fake.statusRequests) + assert.Empty(t, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeReturnsWhyTheServerWontFinalize(t *testing.T) { + notStaged := strings.ReplaceAll(stagingInProgress, "IN_PROGRESS", "NOT_STAGED") + fake := newFakeModelServing(t, notStaged) + fake.finalizations = []string{notFinalizableResponse} resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + _, ok := errors.AsType[*finalizeError](err) + assert.True(t, ok) + assert.ErrorContains(t, err, "haven't been copied into it") + assert.False(t, resp.Ready) + // A version with no staging record isn't polled; the server says what to do. + assert.Empty(t, fake.statusRequests) + assert.Len(t, fake.finalizeRequests, 1) +} + +func TestRegisterAndFinalizeWaitsOutAStatusWithoutAState(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) + fake.statuses = []string{`{}`, `{"status": {"state": "IN_PROGRESS"}}`, statusSucceeded} + + resp, states, err := registerTestCheckpoint(t, fake, checkpointRequest()) require.NoError(t, err) assert.True(t, resp.Ready) - assert.Len(t, fake.requests(), 3) - assert.Len(t, fake.statusRequests, 2) + assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED"}, states) + assert.Len(t, fake.statusRequests, 3) + assert.Len(t, fake.finalizeRequests, 1) +} + +func TestRegisterAndFinalizeKeepsAnEnvironmentStateTheFinalizeOmits(t *testing.T) { + environmentFailed := strings.Replace(stagingSucceeded, `"environment_artifact": {"state": "SUCCEEDED"}`, `"environment_artifact": {"state": "FAILED"}`, 1) + fake := newFakeModelServing(t, environmentFailed) + fake.finalizations = []string{`{"model_version": "3", "staging_status": {"model_artifact": {"state": "SUCCEEDED"}}}`} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + assert.Equal(t, stagingStateFailed, resp.environmentState()) +} + +func TestRegisterAndFinalizeRejectsAnotherFinalizedVersion(t *testing.T) { + fake := newFakeModelServing(t, stagingSucceeded) + fake.finalizations = []string{`{"model_version": "4"}`} + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + _, ok := errors.AsType[*finalizeError](err) + assert.True(t, ok) + assert.ErrorContains(t, err, `the server finalized version "4" instead`) +} + +func TestRegisterAndFinalizeRejectsARegistrationWithoutAVersion(t *testing.T) { + fake := newFakeModelServing(t, `{"model_id": "m-1", "registered_model": "main.models.qwen", "ready": false}`) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, "registering a version of main.models.qwen returned no version") + assert.Empty(t, fake.finalizeRequests) } -func TestRegisterUntilReadyDoesNotRepeatANotReadyModelFromCode(t *testing.T) { +func TestRegisterAndFinalizeFinalizesAModelFromCodeWithoutPolling(t *testing.T) { fake := newFakeModelServing(t, `{"model_id": "m-2", "registered_model": "a.b.c", "model_version": "1", "ready": false}`) + fake.finalizations = []string{`{"model_id": "m-2", "registered_model": "a.b.c", "model_version": "1"}`} req := registerModelPackageRequest{RegisteredModel: "a.b.c", IdempotencyKey: "k", Code: &modelPackageCode{EntryFile: "agent.py"}} - _, _, err := registerTestCheckpoint(t, fake, req) - assert.ErrorContains(t, err, "version 1 of a.b.c isn't ready after registering") - assert.Len(t, fake.requests(), 1) + resp, _, err := registerTestCheckpoint(t, fake, req) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Empty(t, fake.statusRequests) + assert.Equal(t, []finalizeModelPackageRequest{{RegisteredModel: "a.b.c", ModelVersion: "1"}}, fake.finalizeRequests) +} + +func TestRegisterAndFinalizeRetriesTransientErrors(t *testing.T) { + fake := newFakeModelServing(t, serverErrorResponse, stagingInProgress) + fake.statuses = []string{serverErrorResponse, statusSucceeded} + fake.finalizations = []string{serverErrorResponse, finalizedResponse} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + require.NoError(t, err) + assert.True(t, resp.Ready) + assert.Len(t, fake.requests(), 2) + assert.Len(t, fake.statusRequests, 2) + assert.Len(t, fake.finalizeRequests, 2) } -func TestRegisterUntilReadyGivesUpOnPersistentErrors(t *testing.T) { +func TestRegisterAndFinalizeGivesUpOnPersistentErrors(t *testing.T) { fake := newFakeModelServing(t, serverErrorResponse) _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) @@ -213,7 +325,7 @@ func TestRegisterUntilReadyGivesUpOnPersistentErrors(t *testing.T) { assert.Len(t, fake.requests(), retry.MaxRetries+1) } -func TestRegisterUntilReadyReturnsServerErrors(t *testing.T) { +func TestRegisterAndFinalizeReturnsServerErrors(t *testing.T) { fake := newFakeModelServing(t, conflictResponse) _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) @@ -221,12 +333,14 @@ func TestRegisterUntilReadyReturnsServerErrors(t *testing.T) { assert.Len(t, fake.requests(), 1) } -func TestRegisterUntilReadyStopsWhenCanceled(t *testing.T) { - srv := newFakeModelServing(t, stagingInProgress).start() +func TestRegisterAndFinalizeStopsWhenCanceled(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) + srv := fake.start() ctx, cancel := context.WithCancel(t.Context()) - _, err := registerUntilReady(ctx, newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Hour, func(*registerModelPackageResponse) { cancel() }) + _, err := registerAndFinalize(ctx, newTestWorkspaceClient(t, srv.URL), checkpointRequest(), time.Hour, func(*registerModelPackageResponse) { cancel() }) assert.ErrorIs(t, err, context.Canceled) + assert.Empty(t, fake.finalizeRequests) } func TestRegisterModelPackageRequestJSON(t *testing.T) { From 4b16295c158eeeb4cd5662621667a5cd52f0421d Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 16 Nov 2026 19:30:03 +0000 Subject: [PATCH 05/11] air deploy: accept audio checkpoints and print their audio URL A checkpoint can declare llm/v1/audio/transcriptions or llm/v1/audio/translations, which the registration API accepts. Their requests are multipart uploads to the endpoint's audio route, so the CLI prints that route instead of invocations. Co-authored-by: Isaac --- cmd/air/deploy.go | 18 ++++++++++++++---- cmd/air/deploy_test.go | 16 ++++++++++++++++ cmd/air/deployconfig.go | 11 +++++++++-- cmd/air/deployconfig_test.go | 12 +++++++++++- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go index 4d2b0d876f5..0b912366275 100644 --- a/cmd/air/deploy.go +++ b/cmd/air/deploy.go @@ -259,7 +259,7 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio return fmt.Errorf("serving version %s of %s on endpoint %s failed: %w\n\nTo try again without registering another version, run:\n %s", result.ModelVersion, cfg.RegisteredModel, cfg.Endpoint, err, resume) } result.Status = "READY" - result.EndpointURL = endpointInvocationsURL(w.Config.Host, cfg.Endpoint) + result.EndpointURL = endpointQueryURL(w.Config.Host, cfg.Endpoint, cfg.Task) if jsonOut { return renderEnvelope(ctx, result) } @@ -450,7 +450,17 @@ func deployResumeCommand(profile string, opts deployOptions, key string) string return strings.Join(args, " ") } -// endpointInvocationsURL is the REST URL that queries the endpoint. -func endpointInvocationsURL(host, name string) string { - return strings.TrimSuffix(host, "/") + "/serving-endpoints/" + name + "/invocations" +// endpointQueryURL is the REST URL that queries the endpoint: an audio model's +// audio route, which takes a multipart upload, or invocations for any other model. +func endpointQueryURL(host, name string, task *string) string { + base := strings.TrimSuffix(host, "/") + "/serving-endpoints/" + name + if task != nil { + switch *task { + case taskAudioTranscriptions: + return base + "/audio/transcriptions" + case taskAudioTranslations: + return base + "/audio/translations" + } + } + return base + "/invocations" } diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go index 05cbdab8d6b..11119b585c0 100644 --- a/cmd/air/deploy_test.go +++ b/cmd/air/deploy_test.go @@ -467,3 +467,19 @@ func TestDeployHelpDocumentsAField(t *testing.T) { // The coding agent skill writes run YAML, so the deploy help leaves it out. assert.NotContains(t, out.String(), "databricks-ai-runtime") } + +func TestEndpointQueryURL(t *testing.T) { + task := func(s string) *string { return &s } + tests := []struct { + task *string + want string + }{ + {task("llm/v1/chat"), "https://h/serving-endpoints/ep/invocations"}, + {task(taskAudioTranscriptions), "https://h/serving-endpoints/ep/audio/transcriptions"}, + {task(taskAudioTranslations), "https://h/serving-endpoints/ep/audio/translations"}, + {nil, "https://h/serving-endpoints/ep/invocations"}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, endpointQueryURL("https://h/", "ep", tt.task)) + } +} diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index 9c3bdac4cf0..adc2d36fc2e 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -34,9 +34,16 @@ const ( // modelTypes lists model.type values in the order the help and errors name them. var modelTypes = []string{modelTypeCheckpoint, modelTypeCode, modelTypeSavedModelDir, modelTypeRegistered} +// Audio tasks: their requests are multipart uploads of a recording, sent to the +// endpoint's audio route rather than to invocations. +const ( + taskAudioTranscriptions = "llm/v1/audio/transcriptions" + taskAudioTranslations = "llm/v1/audio/translations" +) + // checkpointTasks are the tasks a checkpoint can serve, as the registration API // documents them; models from code serve agent tasks too. -var checkpointTasks = []string{"llm/v1/chat", "llm/v1/completions", "llm/v1/embeddings"} +var checkpointTasks = []string{"llm/v1/chat", "llm/v1/completions", "llm/v1/embeddings", taskAudioTranscriptions, taskAudioTranslations} // deployCommandPath is the command whose -h documents deployConfig. const deployCommandPath = "databricks air deploy" @@ -82,7 +89,7 @@ var entryFileRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompi type deployConfig struct { RegisteredModel string `yaml:"registered_model" help:"Unity Catalog model, as ... air deploy registers a new version of it, creating the model if it doesn't exist, unless model.type is registered." required:"yes" group:"Model"` Model *deployModelConfig `yaml:"model" help:"What to serve: a checkpoint, a model from code, a model directory saved by MLflow, or an existing version." required:"yes" group:"Model"` - Task *string `yaml:"task" help:"The model's task as MLflow names it: llm/v1/chat, llm/v1/completions or llm/v1/embeddings for a checkpoint, or e.g. agent/v1/responses for a model from code. A model from code without a task has no signature." required:"when model.type is checkpoint" group:"Model"` + Task *string `yaml:"task" help:"The model's task as MLflow names it: llm/v1/chat, llm/v1/completions, llm/v1/embeddings, llm/v1/audio/transcriptions or llm/v1/audio/translations for a checkpoint, or e.g. agent/v1/responses for a model from code. A model from code without a task has no signature." required:"when model.type is checkpoint" group:"Model"` Command *string `yaml:"command" help:"Shell command that starts the model server of a checkpoint, at most 32,768 characters. It runs from the directory where model.path appears under its own name." required:"when model.type is checkpoint" group:"Model"` Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment version to serve on and the packages to install, for a checkpoint or a model from code." group:"Model"` CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of a model from code, sent with the registration and stored in the model version, where anyone who can read the model can read it. Only the files under include_paths are sent, leaving out files git ignores, hidden files and directories (names starting with '.') and __pycache__ directories; run with --dry-run to list them." required:"when model.type is code" group:"Model"` diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index 722d1acb350..3532516ca98 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -70,6 +70,16 @@ func TestLoadDeployConfigCheckpoint(t *testing.T) { }, req) } +func TestLoadDeployConfigAudioCheckpoint(t *testing.T) { + for _, task := range []string{taskAudioTranscriptions, taskAudioTranslations} { + t.Run(task, func(t *testing.T) { + cfg, err := loadDeployConfigText(t, strings.Replace(checkpointDeployConfig, "task: llm/v1/chat", "task: "+task, 1)) + require.NoError(t, err) + assert.Equal(t, task, cfg.registrationRequest("key-1", nil).Checkpoint.Task) + }) + } +} + func TestLoadDeployConfigCodeOnCPU(t *testing.T) { cfg, err := loadDeployConfigText(t, codeDeployConfig) require.NoError(t, err) @@ -153,7 +163,7 @@ func TestDeployConfigValidation(t *testing.T) { {"checkpoint on CPU", replaceFirst("GPU_MEDIUM", "CPU"), "a checkpoint is served on GPU"}, {"unknown workload type", replaceFirst("GPU_MEDIUM", "GPU_MEDUIM"), `compute.workload_type must be one of CPU, CPU_LARGE, CPU_MEDIUM, GPU_LARGE, GPU_MEDIUM, GPU_SMALL, GPU_XLARGE, GPU_XLARGE_8, MULTIGPU_MEDIUM, got "GPU_MEDUIM"`}, {"lowercase workload type", replaceFirst("GPU_MEDIUM", "gpu_medium"), "compute.workload_type must be one of"}, - {"agent task on a checkpoint", replaceFirst("task: llm/v1/chat", "task: agent/v1/responses"), `task of a checkpoint must be one of llm/v1/chat, llm/v1/completions, llm/v1/embeddings, got "agent/v1/responses"`}, + {"agent task on a checkpoint", replaceFirst("task: llm/v1/chat", "task: agent/v1/responses"), `task of a checkpoint must be one of llm/v1/chat, llm/v1/completions, llm/v1/embeddings, llm/v1/audio/transcriptions, llm/v1/audio/translations, got "agent/v1/responses"`}, {"zero environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: 0"), "environment.version must be a serverless environment version"}, {"empty dependency", replaceFirst("[einops==0.8.0]", `[""]`), "environment.dependencies[0] cannot be empty"}, {"no endpoint", dropLine("endpoint:"), `endpoint must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, From 7a054240bfda431d7d29221846da547ffc0bbf59 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Mon, 16 Nov 2026 23:33:58 +0000 Subject: [PATCH 06/11] air deploy: one config for a checkpoint served by a command or by Python code A checkpoint is model.path plus exactly one of command or model.entry, with model.model_config, model.streamable and the Databricks AI environment (environment.version: databricks_ai_v6) for Python code. The server decides which tasks a command can serve. Co-authored-by: Isaac --- cmd/air/deploy.go | 2 +- cmd/air/deployconfig.go | 111 ++++++++++++++++++++++++--------- cmd/air/deployconfig_test.go | 73 ++++++++++++++++++++-- cmd/air/deployregister.go | 53 ++++++++++++---- cmd/air/deployregister_test.go | 23 +++++++ 5 files changed, 214 insertions(+), 48 deletions(-) diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go index 0b912366275..cc7f0105142 100644 --- a/cmd/air/deploy.go +++ b/cmd/air/deploy.go @@ -202,7 +202,7 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio } var code *modelPackageCode - if cfg.Model.Type == modelTypeCode { + if cfg.Model.Entry != nil { code, err = collectModelCode(ctx, cfg, opts.file) if err != nil { return withCode(codeInvalidArgs, false, err) diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index adc2d36fc2e..1bffa98433b 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -41,10 +41,6 @@ const ( taskAudioTranslations = "llm/v1/audio/translations" ) -// checkpointTasks are the tasks a checkpoint can serve, as the registration API -// documents them; models from code serve agent tasks too. -var checkpointTasks = []string{"llm/v1/chat", "llm/v1/completions", "llm/v1/embeddings", taskAudioTranscriptions, taskAudioTranslations} - // deployCommandPath is the command whose -h documents deployConfig. const deployCommandPath = "databricks air deploy" @@ -89,10 +85,10 @@ var entryFileRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompi type deployConfig struct { RegisteredModel string `yaml:"registered_model" help:"Unity Catalog model, as ... air deploy registers a new version of it, creating the model if it doesn't exist, unless model.type is registered." required:"yes" group:"Model"` Model *deployModelConfig `yaml:"model" help:"What to serve: a checkpoint, a model from code, a model directory saved by MLflow, or an existing version." required:"yes" group:"Model"` - Task *string `yaml:"task" help:"The model's task as MLflow names it: llm/v1/chat, llm/v1/completions, llm/v1/embeddings, llm/v1/audio/transcriptions or llm/v1/audio/translations for a checkpoint, or e.g. agent/v1/responses for a model from code. A model from code without a task has no signature." required:"when model.type is checkpoint" group:"Model"` - Command *string `yaml:"command" help:"Shell command that starts the model server of a checkpoint, at most 32,768 characters. It runs from the directory where model.path appears under its own name." required:"when model.type is checkpoint" group:"Model"` - Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment version to serve on and the packages to install, for a checkpoint or a model from code." group:"Model"` - CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of a model from code, sent with the registration and stored in the model version, where anyone who can read the model can read it. Only the files under include_paths are sent, leaving out files git ignores, hidden files and directories (names starting with '.') and __pycache__ directories; run with --dry-run to list them." required:"when model.type is code" group:"Model"` + Task *string `yaml:"task" help:"The model's task as MLflow names it. Required with command, and then one Model Serving serves a command for, such as llm/v1/chat, llm/v1/embeddings or llm/v1/audio/transcriptions. Optional with model.entry, e.g. llm/v1/chat or agent/v1/responses, where it selects MLflow's signature; a model without a task has no signature." required:"when command is set" group:"Model"` + Command *string `yaml:"command" help:"Shell command that starts a model server over a checkpoint's weights, at most 32,768 characters, serving task on port 8080. It runs from the directory where model.path appears under its own name. Set either command or model.entry." required:"when model.type is checkpoint and model.entry is not set" group:"Model"` + Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment to serve on and the packages to install, for a checkpoint or a model from code." group:"Model"` + CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of model.entry, sent with the registration and stored in the model version, where anyone who can read the model can read it. Only the files under include_paths are sent, leaving out files git ignores, hidden files and directories (names starting with '.') and __pycache__ directories; run with --dry-run to list them." required:"when model.entry is set" group:"Model"` Endpoint string `yaml:"endpoint" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead, with the compute, environment variables and secrets in this file replacing the ones it has; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` Compute *deployComputeConfig `yaml:"compute" help:"Serving compute and how the endpoint scales." required:"yes" group:"Serving"` @@ -105,16 +101,18 @@ type deployConfig struct { // deployModelConfig is the `model` block: where the served model comes from. type deployModelConfig struct { - Type string `yaml:"type" help:"One of checkpoint (weights in a volume, started by command), code (a Python file from code_source), saved_model_dir (a model directory saved by MLflow in a volume) or registered (an existing version of registered_model)." required:"yes"` - Path *string `yaml:"path" help:"Unity Catalog volume directory, as /Volumes////, holding a checkpoint's weights or a model saved by MLflow. Model Serving copies it into the version." required:"when type is checkpoint or saved_model_dir"` - Entry *string `yaml:"entry" help:"Python file directly under code_source.root_path that builds the model and calls mlflow.models.set_model, e.g. agent.py." required:"when type is code"` - Version *string `yaml:"version" help:"Version of registered_model to serve, e.g. 3." required:"when type is registered"` + Type string `yaml:"type" help:"One of checkpoint (weights in a volume, served by command or by the Python file model.entry), code (a Python file from code_source, no weights), saved_model_dir (a model directory saved by MLflow in a volume) or registered (an existing version of registered_model)." required:"yes"` + Path *string `yaml:"path" help:"Unity Catalog volume directory, as /Volumes////, holding a checkpoint's weights or a model saved by MLflow. Model Serving copies it, with its subdirectories, into the version, where it appears under its own name: command finds it there, and model.entry reads its path as context.artifacts[\"\"]." required:"when type is checkpoint or saved_model_dir"` + Entry *string `yaml:"entry" help:"Python file directly under code_source.root_path that builds the model and calls mlflow.models.set_model, e.g. model.py. Model Serving's model server loads it and calls its predict; for a checkpoint, set either entry or command." required:"when type is code"` + ModelConfig map[string]any `yaml:"model_config" help:"Settings model.entry reads as context.model_config, as MLflow's log_model(model_config=...) records them."` + Streamable *bool `yaml:"streamable" help:"Whether model.entry answers streaming requests with predict_stream. MLflow detects this from the class when it logs a model; here it is declared. Defaults to true for a chat or agent task and false otherwise."` + Version *string `yaml:"version" help:"Version of registered_model to serve, e.g. 3." required:"when type is registered"` } // deployEnvironmentConfig is the `environment` block of a deployment. type deployEnvironmentConfig struct { - Version *int `yaml:"version" help:"Serverless environment version to serve on, e.g. 6 (default: 6). The model uses its Python version."` - Dependencies []string `yaml:"dependencies" help:"pip requirement lines to install, e.g. einops==0.8.0, at most 1,000. A model from code must list mlflow or mlflow-skinny."` + Version *stringOrInt `yaml:"version" help:"Serverless environment to serve on: 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment (default: 6), as for air run. The model uses its Python version. Standard includes CUDA but not PyTorch; the Databricks AI environment adds PyTorch, Transformers and vLLM, and needs GPU compute. A command that doesn't install SGLang runs on the Databricks AI environment either way."` + Dependencies []string `yaml:"dependencies" help:"pip requirement lines to install, e.g. einops==0.8.0, at most 1,000. A model from code must list mlflow or mlflow-skinny."` } // deployCodeSourceConfig is the `code_source` block of a model from code. @@ -134,7 +132,7 @@ type deployComputeConfig struct { // addition to the required ones in requiredSourceFields. Every other source field // is rejected, so a field the source can't use is never silently dropped. var sourceFields = map[string][]string{ - modelTypeCheckpoint: {"environment", "idempotency_token", "mlflow_experiment_path"}, + modelTypeCheckpoint: {"model.entry", "model.model_config", "model.streamable", "task", "command", "code_source", "environment", "idempotency_token", "mlflow_experiment_path"}, modelTypeCode: {"task", "environment", "idempotency_token", "mlflow_experiment_path"}, modelTypeSavedModelDir: {"idempotency_token", "mlflow_experiment_path"}, modelTypeRegistered: {}, @@ -142,7 +140,7 @@ var sourceFields = map[string][]string{ // requiredSourceFields lists the fields each model.type requires. var requiredSourceFields = map[string][]string{ - modelTypeCheckpoint: {"model.path", "task", "command"}, + modelTypeCheckpoint: {"model.path"}, modelTypeCode: {"model.entry", "code_source"}, modelTypeSavedModelDir: {"model.path"}, modelTypeRegistered: {"model.version"}, @@ -199,6 +197,8 @@ func (c *deployConfig) validateSource() error { set := map[string]bool{ "model.path": m.Path != nil, "model.entry": m.Entry != nil, + "model.model_config": m.ModelConfig != nil, + "model.streamable": m.Streamable != nil, "model.version": m.Version != nil, "task": c.Task != nil, "command": c.Command != nil, @@ -219,12 +219,19 @@ func (c *deployConfig) validateSource() error { } } + if m.Type == modelTypeCheckpoint { + if err := c.validateCheckpoint(); err != nil { + return err + } + } if m.Path != nil { - v := *m.Path - if !strings.HasPrefix(v, "/Volumes/") || serverLength(v) > maxVolumePathChars { - return fmt.Errorf("model.path must be a Unity Catalog volume directory as /Volumes////, of at most %d characters, got %q", maxVolumePathChars, v) + if err := validateVolumeDirectory("model.path", *m.Path); err != nil { + return err } } + if c.CodeSource != nil && m.Entry == nil { + return errors.New("code_source can't be set without model.entry, the file in it that builds the model") + } if m.Entry != nil && !entryFileRe().MatchString(*m.Entry) { return fmt.Errorf("model.entry must name a .py file directly under code_source.root_path, e.g. agent.py, got %q", *m.Entry) } @@ -234,9 +241,6 @@ func (c *deployConfig) validateSource() error { if c.Task != nil && strings.TrimSpace(*c.Task) == "" { return errors.New("task cannot be empty") } - if m.Type == modelTypeCheckpoint && !slices.Contains(checkpointTasks, *c.Task) { - return fmt.Errorf("task of a checkpoint must be one of %s, got %q", strings.Join(checkpointTasks, ", "), *c.Task) - } if c.Command != nil { if err := validateCommand(*c.Command); err != nil { return err @@ -273,9 +277,39 @@ func (c *deployConfig) validateSource() error { } // validate checks the environment version and dependency lines. +// validateCheckpoint checks the fields of a checkpoint: weights to copy, and +// exactly one way to serve them. +func (c *deployConfig) validateCheckpoint() error { + m := c.Model + if (c.Command != nil) == (m.Entry != nil) { + return errors.New("exactly one of command and model.entry is required when model.type is checkpoint: a command that serves the weights, or a Python file Model Serving's model server loads") + } + if c.Command != nil && c.Task == nil { + return errors.New("task is required when command is set: the task Model Serving routes to the command, such as llm/v1/chat") + } + if m.Entry != nil && c.CodeSource == nil { + return errors.New("code_source is required when model.entry is set: the directory that holds the file") + } + if m.Entry == nil && (m.ModelConfig != nil || m.Streamable != nil) { + return errors.New("model.model_config and model.streamable apply to model.entry, not to command") + } + return nil +} + +// validateVolumeDirectory checks a Unity Catalog volume directory the server +// copies into the version. +func validateVolumeDirectory(field, v string) error { + if !strings.HasPrefix(v, "/Volumes/") || serverLength(v) > maxVolumePathChars { + return fmt.Errorf("%s must be a Unity Catalog volume directory as /Volumes////, of at most %d characters, got %q", field, maxVolumePathChars, v) + } + return nil +} + func (e *deployEnvironmentConfig) validate() error { - if e.Version != nil && *e.Version < 1 { - return fmt.Errorf("environment.version must be a serverless environment version such as 6, got %d", *e.Version) + if e.Version != nil { + if _, _, err := e.runtimeVersion(); err != nil { + return err + } } if len(e.Dependencies) > maxDependencies { return fmt.Errorf("environment.dependencies can list at most %d packages, got %d", maxDependencies, len(e.Dependencies)) @@ -329,13 +363,28 @@ func (c *deployConfig) servesOnGPU() bool { return !strings.HasPrefix(c.Compute.WorkloadType, cpuWorkloadTypePrefix) } +// runtimeVersion is environment.version as a version number, and whether it names +// the Databricks AI environment, as air run reads the same field. +func (e *deployEnvironmentConfig) runtimeVersion() (int, bool, error) { + if e == nil || e.Version == nil { + return defaultServingEnvironmentVersion, false, nil + } + normalized, err := validateRuntimeVersion(e.Version.raw, "environment.version") + if err != nil { + return 0, false, err + } + ai := strings.HasPrefix(normalized, databricksAIPrefix) + major, err := strconv.Atoi(strings.TrimPrefix(normalized, databricksAIPrefix)) + if err != nil || major < 1 { + return 0, false, fmt.Errorf("environment.version must be a serverless environment version such as 6 or databricks_ai_v6, got %q", e.Version.raw) + } + return major, ai, nil +} + // servingRuntime is the serverless runtime the model is served on, as the server // names it: client., with a -gpu suffix on GPU compute. func (c *deployConfig) servingRuntime() string { - version := defaultServingEnvironmentVersion - if c.Environment != nil && c.Environment.Version != nil { - version = *c.Environment.Version - } + version, _, _ := c.Environment.runtimeVersion() runtime := "client." + strconv.Itoa(version) if c.servesOnGPU() { runtime += "-gpu" @@ -343,9 +392,13 @@ func (c *deployConfig) servingRuntime() string { return runtime } -// servingEnvironment is the registration's runtime and dependencies. +// servingEnvironment is the registration's runtime, dependencies and base +// environment. func (c *deployConfig) servingEnvironment() modelPackageEnvironment { environment := modelPackageEnvironment{Runtime: c.servingRuntime()} + if _, ai, _ := c.Environment.runtimeVersion(); ai { + environment.BaseEnvironment = baseEnvironmentDatabricksAI + } if c.Environment != nil { environment.Dependencies = c.Environment.Dependencies } diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index 3532516ca98..b2a8870d6a0 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -80,6 +80,63 @@ func TestLoadDeployConfigAudioCheckpoint(t *testing.T) { } } +const pythonCheckpointDeployConfig = `registered_model: main.ml.whisper +model: + type: checkpoint + path: /Volumes/main/weights/whisper + entry: model.py + model_config: {language: en, beams: 4} + streamable: false +task: llm/v1/chat +code_source: + root_path: . + include_paths: [model.py] +environment: + version: databricks_ai_v6 + dependencies: [soundfile==0.13.1] +endpoint: whisper +compute: + workload_type: GPU_MEDIUM + workload_size: Small +` + +func TestLoadDeployConfigPythonCheckpoint(t *testing.T) { + cfg, err := loadDeployConfigText(t, pythonCheckpointDeployConfig) + require.NoError(t, err) + assert.Equal(t, "client.6-gpu", cfg.servingRuntime()) + + streamable := false + code := &modelPackageCode{EntryFile: "model.py", EntrySource: "import mlflow\n", Files: []modelPackageFile{{Path: "util/a.py", Content: "x = 1\n"}}} + req := cfg.registrationRequest("key-1", code) + assert.Equal(t, registerModelPackageRequest{ + RegisteredModel: "main.ml.whisper", + IdempotencyKey: "key-1", + Checkpoint: &modelPackageCheckpoint{ + VolumePath: "/Volumes/main/weights/whisper", + PythonModel: &modelPackagePythonModel{ + EntryFile: "model.py", + EntrySource: "import mlflow\n", + Files: []modelPackageFile{{Path: "util/a.py", Content: "x = 1\n"}}, + ModelConfig: `{"beams":4,"language":"en"}`, + Streamable: &streamable, + }, + Task: "llm/v1/chat", + Environment: modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"soundfile==0.13.1"}, BaseEnvironment: "databricks-ai"}, + }, + }, req) +} + +func TestLoadDeployConfigPythonCheckpointWithoutTask(t *testing.T) { + text := strings.Replace(pythonCheckpointDeployConfig, "task: llm/v1/chat\n", "", 1) + text = strings.Replace(text, " model_config: {language: en, beams: 4}\n streamable: false\n", "", 1) + cfg, err := loadDeployConfigText(t, text) + require.NoError(t, err) + req := cfg.registrationRequest("k", &modelPackageCode{EntryFile: "model.py"}) + assert.Empty(t, req.Checkpoint.Task) + assert.Nil(t, req.Checkpoint.PythonModel.Streamable) + assert.Empty(t, req.Checkpoint.PythonModel.ModelConfig) +} + func TestLoadDeployConfigCodeOnCPU(t *testing.T) { cfg, err := loadDeployConfigText(t, codeDeployConfig) require.NoError(t, err) @@ -151,10 +208,15 @@ func TestDeployConfigValidation(t *testing.T) { {"bad model name", replaceFirst("main.models.qwen", "qwen"), "registered_model must be a Unity Catalog model name"}, {"backticked model name", replaceFirst("main.models.qwen", "main.models.`qwen`"), "registered_model must be a Unity Catalog model name"}, {"unknown type", replaceFirst("type: checkpoint", "type: pickle"), `model.type must be one of checkpoint, code, saved_model_dir, registered, got "pickle"`}, - {"checkpoint without task", replaceFirst("task: llm/v1/chat\n", ""), "task is required when model.type is checkpoint"}, - {"checkpoint without command", dropLine("command:"), "command is required when model.type is checkpoint"}, - {"checkpoint with entry", replaceFirst(" type: checkpoint\n", " type: checkpoint\n entry: a.py\n"), "model.entry can't be set when model.type is checkpoint"}, - {"checkpoint with code_source", appendText("code_source: {root_path: .}\n"), "code_source can't be set when model.type is checkpoint"}, + {"checkpoint without task", replaceFirst("task: llm/v1/chat\n", ""), "task is required when command is set"}, + {"checkpoint without command", dropLine("command:"), "exactly one of command and model.entry is required when model.type is checkpoint"}, + {"checkpoint with command and entry", replaceFirst(" type: checkpoint\n", " type: checkpoint\n entry: a.py\n"), "exactly one of command and model.entry is required when model.type is checkpoint"}, + {"checkpoint with code_source but no entry", appendText("code_source: {root_path: .}\n"), "code_source can't be set without model.entry"}, + {"checkpoint without weights", dropLine(" path:"), "model.path is required when model.type is checkpoint"}, + {"checkpoint with artifacts", replaceFirst(" type: checkpoint\n", " type: checkpoint\n artifacts: {tok: /Volumes/a/b/c/d}\n"), "field artifacts not found"}, + {"model_config with a command", replaceFirst(" type: checkpoint\n", " type: checkpoint\n model_config: {k: v}\n"), "model.model_config and model.streamable apply to model.entry, not to command"}, + {"AI environment before version 5", replaceFirst(" dependencies: [einops==0.8.0]", " version: databricks_ai_v4"), "databricks_ai_v in environment.version requires AI Runtime version 5 or higher"}, + {"bad environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: six"), "unsupported client image version"}, {"path outside volumes", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/dbfs/weights"), "model.path must be a Unity Catalog volume directory"}, {"path too long", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("a", 1100)), "model.path must be a Unity Catalog volume directory"}, {"empty command", replaceFirst("command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", `command: " "`), "command cannot be empty"}, @@ -163,8 +225,7 @@ func TestDeployConfigValidation(t *testing.T) { {"checkpoint on CPU", replaceFirst("GPU_MEDIUM", "CPU"), "a checkpoint is served on GPU"}, {"unknown workload type", replaceFirst("GPU_MEDIUM", "GPU_MEDUIM"), `compute.workload_type must be one of CPU, CPU_LARGE, CPU_MEDIUM, GPU_LARGE, GPU_MEDIUM, GPU_SMALL, GPU_XLARGE, GPU_XLARGE_8, MULTIGPU_MEDIUM, got "GPU_MEDUIM"`}, {"lowercase workload type", replaceFirst("GPU_MEDIUM", "gpu_medium"), "compute.workload_type must be one of"}, - {"agent task on a checkpoint", replaceFirst("task: llm/v1/chat", "task: agent/v1/responses"), `task of a checkpoint must be one of llm/v1/chat, llm/v1/completions, llm/v1/embeddings, llm/v1/audio/transcriptions, llm/v1/audio/translations, got "agent/v1/responses"`}, - {"zero environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: 0"), "environment.version must be a serverless environment version"}, + {"zero environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: 0"), "environment.version must be a serverless environment version such as 6 or databricks_ai_v6"}, {"empty dependency", replaceFirst("[einops==0.8.0]", `[""]`), "environment.dependencies[0] cannot be empty"}, {"no endpoint", dropLine("endpoint:"), `endpoint must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, {"endpoint with a path", replaceFirst("endpoint: qwen-chat", "endpoint: x/../../jobs"), "endpoint must be a serving endpoint name"}, diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go index f3f24c99693..b54f27302c4 100644 --- a/cmd/air/deployregister.go +++ b/cmd/air/deployregister.go @@ -2,6 +2,7 @@ package aircmd import ( "context" + "encoding/json" "errors" "fmt" "net/http" @@ -57,12 +58,25 @@ type registerModelPackageRequest struct { } type modelPackageCheckpoint struct { - VolumePath string `json:"volume_path"` - ServingCommand string `json:"serving_command"` - Task string `json:"task"` - Environment modelPackageEnvironment `json:"environment"` + VolumePath string `json:"volume_path"` + ServingCommand string `json:"serving_command,omitempty"` + PythonModel *modelPackagePythonModel `json:"python_model,omitempty"` + Task string `json:"task,omitempty"` + Environment modelPackageEnvironment `json:"environment"` } +// modelPackagePythonModel is the Python code that serves a checkpoint's weights. +type modelPackagePythonModel struct { + EntryFile string `json:"entry_file"` + EntrySource string `json:"entry_source"` + Files []modelPackageFile `json:"files,omitempty"` + ModelConfig string `json:"model_config,omitempty"` + Streamable *bool `json:"streamable,omitempty"` +} + +// baseEnvironmentDatabricksAI names the Databricks AI environment to the server. +const baseEnvironmentDatabricksAI = "databricks-ai" + type modelPackageCode struct { EntryFile string `json:"entry_file"` EntrySource string `json:"entry_source"` @@ -81,8 +95,9 @@ type modelPackageSavedModel struct { } type modelPackageEnvironment struct { - Runtime string `json:"runtime"` - Dependencies []string `json:"dependencies,omitempty"` + Runtime string `json:"runtime"` + Dependencies []string `json:"dependencies,omitempty"` + BaseEnvironment string `json:"base_environment,omitempty"` } // registerModelPackageResponse is the RegisterModelPackageResponse body. @@ -159,7 +174,7 @@ func (e *finalizeError) Error() string { return e.err.Error() } func (e *finalizeError) Unwrap() error { return e.err } // registrationRequest builds the request that registers cfg's model under key. -// code is the collected model from code, and is nil for the other sources. +// code is the collected code of model.entry, and is nil when there is none. func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) registerModelPackageRequest { req := registerModelPackageRequest{ RegisteredModel: c.RegisteredModel, @@ -170,12 +185,26 @@ func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) r } switch c.Model.Type { case modelTypeCheckpoint: - req.Checkpoint = &modelPackageCheckpoint{ - VolumePath: *c.Model.Path, - ServingCommand: *c.Command, - Task: *c.Task, - Environment: c.servingEnvironment(), + checkpoint := &modelPackageCheckpoint{VolumePath: *c.Model.Path, Environment: c.servingEnvironment()} + if c.Task != nil { + checkpoint.Task = *c.Task + } + if c.Command != nil { + checkpoint.ServingCommand = *c.Command + } else { + checkpoint.PythonModel = &modelPackagePythonModel{ + EntryFile: code.EntryFile, + EntrySource: code.EntrySource, + Files: code.Files, + Streamable: c.Model.Streamable, + } + if c.Model.ModelConfig != nil { + // The YAML decoded as JSON-compatible values; the server reads a JSON object. + raw, _ := json.Marshal(c.Model.ModelConfig) + checkpoint.PythonModel.ModelConfig = string(raw) + } } + req.Checkpoint = checkpoint case modelTypeCode: req.Code = code case modelTypeSavedModelDir: diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go index 2efeb1a9028..b9a43f7330c 100644 --- a/cmd/air/deployregister_test.go +++ b/cmd/air/deployregister_test.go @@ -343,6 +343,29 @@ func TestRegisterAndFinalizeStopsWhenCanceled(t *testing.T) { assert.Empty(t, fake.finalizeRequests) } +func TestRegisterPythonCheckpointRequestJSON(t *testing.T) { + streamable := true + raw, err := json.Marshal(registerModelPackageRequest{ + RegisteredModel: "a.b.c", + IdempotencyKey: "k", + Checkpoint: &modelPackageCheckpoint{ + VolumePath: "/Volumes/a/b/c/w", + PythonModel: &modelPackagePythonModel{EntryFile: "model.py", EntrySource: "x = 1\n", ModelConfig: `{"k":"v"}`, Streamable: &streamable}, + Environment: modelPackageEnvironment{Runtime: "client.6-gpu", BaseEnvironment: "databricks-ai"}, + }, + }) + require.NoError(t, err) + assert.JSONEq(t, `{ + "registered_model": "a.b.c", + "idempotency_key": "k", + "checkpoint": { + "volume_path": "/Volumes/a/b/c/w", + "python_model": {"entry_file": "model.py", "entry_source": "x = 1\n", "model_config": "{\"k\":\"v\"}", "streamable": true}, + "environment": {"runtime": "client.6-gpu", "base_environment": "databricks-ai"} + } + }`, string(raw)) +} + func TestRegisterModelPackageRequestJSON(t *testing.T) { raw, err := json.Marshal(registerModelPackageRequest{ RegisteredModel: "a.b.c", From ea8ee4a582122bc23433fdb55d3acd08586250e1 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 17 Nov 2026 01:14:46 +0000 Subject: [PATCH 07/11] air deploy: serve checkpoints only, and read each staging status as sent Drop the code and saved_model_dir model types until the server handles them; Python code now serves a checkpoint through model.entry, so the code collection stays and builds the checkpoint's python_model. Stop merging staging statuses: every registration, poll and finalize response must report a known state for both the weights and the serving environment, and a response that doesn't is an error rather than progress. Each new status replaces the last one. Co-authored-by: Isaac --- cmd/air/deploy.go | 32 +++---- cmd/air/deploy_test.go | 37 ++++---- cmd/air/deploycode.go | 25 ++--- cmd/air/deploycode_test.go | 32 +++---- cmd/air/deployconfig.go | 56 ++++++----- cmd/air/deployconfig_test.go | 69 +++++++------- cmd/air/deployregister.go | 164 ++++++++++++++++----------------- cmd/air/deployregister_test.go | 116 ++++++++++++++--------- 8 files changed, 286 insertions(+), 245 deletions(-) diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go index cc7f0105142..70aaff99db7 100644 --- a/cmd/air/deploy.go +++ b/cmd/air/deploy.go @@ -70,7 +70,8 @@ func withCode(code string, retryable bool, err error) error { type deployResult struct { Status string `json:"status"` DryRun bool `json:"dry_run,omitempty"` - // CodeFiles lists, on a dry run of a model from code, the files it would send. + // CodeFiles lists, on a dry run of a checkpoint served by model.entry, the + // files it would send. CodeFiles []string `json:"code_files,omitempty"` RegisteredModel string `json:"registered_model,omitempty"` ModelVersion string `json:"model_version,omitempty"` @@ -110,11 +111,10 @@ func newDeployCommandWithPollInterval(pollInterval time.Duration) *cobra.Command Long: `Register a model in Unity Catalog and serve it on a Model Serving endpoint. The deployment is described by a YAML file (see --file). The model is a -checkpoint (weights in a Unity Catalog volume and the command that serves -them), a model from code, a model directory saved by MLflow, or an existing -model version. air deploy registers a new version, waits until it is ready, -then creates or updates the endpoint to serve it and waits until the endpoint -is ready. +checkpoint (weights in a Unity Catalog volume, served by a command or by a +Python file that builds an MLflow model) or an existing model version. +air deploy registers a new version, waits until it is ready, then creates or +updates the endpoint to serve it and waits until the endpoint is ready. Registration is idempotent: run air deploy again with the idempotency key it prints to continue an interrupted deployment. To look up a YAML field, pass @@ -201,9 +201,9 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio return withCode(codeInvalidArgs, false, err) } - var code *modelPackageCode + var code *modelPackagePythonModel if cfg.Model.Entry != nil { - code, err = collectModelCode(ctx, cfg, opts.file) + code, err = collectEntryCode(ctx, cfg, opts.file) if err != nil { return withCode(codeInvalidArgs, false, err) } @@ -269,9 +269,9 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio return nil } -// renderDeployDryRun reports a valid config and, for a model from code, the -// files a deployment would send. -func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPackageCode, jsonOut bool) error { +// renderDeployDryRun reports a valid config and, for a checkpoint served by +// model.entry, the files a deployment would send. +func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPackagePythonModel, jsonOut bool) error { var files []string if code != nil { files = codeFilePaths(code) @@ -280,7 +280,7 @@ func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPacka return renderEnvelope(ctx, deployResult{Status: "DRY_RUN_OK", DryRun: true, CodeFiles: files}) } if code != nil { - cmdio.LogString(ctx, fmt.Sprintf("Dry run: the model from code would include %d file(s):", len(files))) + cmdio.LogString(ctx, fmt.Sprintf("Dry run: model.entry would send %d file(s):", len(files))) for _, f := range files { cmdio.LogString(ctx, " "+f) } @@ -291,7 +291,7 @@ func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPacka // registerDeployment registers the version and makes it ready to serve, showing // progress on a spinner when show is true. Its errors say how to continue. -func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, code *modelPackageCode, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { +func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, code *modelPackagePythonModel, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { // Printed to stderr in every output mode, before the first request, so a run // that is killed can still be continued with this key. cmdio.LogString(ctx, fmt.Sprintf("Registering a version of %s with idempotency key %s", cfg.RegisteredModel, key)) @@ -312,11 +312,7 @@ func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg } resume := deployResumeCommand(w.Config.Profile, opts, key) if errors.Is(err, errStagingFailed) { - source := "the checkpoint's weights" - if cfg.Model.Path != nil { - source = "the weights in " + *cfg.Model.Path - } - return nil, withCode(codeStagingFailed, true, fmt.Errorf("copying %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", source, stripControl(resp.ModelVersion), cfg.RegisteredModel, resume)) + return nil, withCode(codeStagingFailed, true, fmt.Errorf("copying the weights in %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", *cfg.Model.Path, stripControl(resp.ModelVersion), cfg.RegisteredModel, resume)) } if stopped := deployStopped(ctx, err); stopped != nil { continuing := "" diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go index 11119b585c0..805bf00a995 100644 --- a/cmd/air/deploy_test.go +++ b/cmd/air/deploy_test.go @@ -317,23 +317,26 @@ compute: {workload_type: GPU_MEDIUM, workload_size: Small} assert.NotContains(t, buf.String(), "idempotency_key") } -func TestDeployCodeSendsTheFiles(t *testing.T) { +func TestDeployEntrySendsThePythonModel(t *testing.T) { cfg, _ := writeCodeTree(t, map[string]string{"agent.py": "import tools\n", "tools/__init__.py": "x = 1\n"}) - codeReady := `{"model_id": "m-2", "registered_model": "main.agents.support", "model_version": "1", "ready": true}` - srv, fake := deployServer(t, newFakeEndpoints(t, nil), codeReady) + ready := `{"model_id": "m-2", "registered_model": "main.agents.support", "model_version": "1", "ready": true, + "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` + srv, fake := deployServer(t, newFakeEndpoints(t, nil), ready) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, codeConfigAt(cfg.CodeSource.RootPath))) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, entryConfigAt(cfg.CodeSource.RootPath))) bodies := fake.requests() require.Len(t, bodies, 1) - assert.Equal(t, &modelPackageCode{ - EntryFile: "agent.py", - EntrySource: "import tools\n", - Files: []modelPackageFile{{Path: "tools/__init__.py", Content: "x = 1\n"}}, + assert.Equal(t, &modelPackageCheckpoint{ + VolumePath: "/Volumes/main/models/weights/support", + PythonModel: &modelPackagePythonModel{ + EntryFile: "agent.py", + EntrySource: "import tools\n", + Files: []modelPackageFile{{Path: "tools/__init__.py", Content: "x = 1\n"}}, + }, Task: "agent/v1/responses", - Environment: modelPackageEnvironment{Runtime: "client.5", Dependencies: []string{"mlflow==3.5.0"}}, - }, bodies[0].Code) - assert.Nil(t, bodies[0].Checkpoint) + Environment: modelPackageEnvironment{Runtime: "client.5-gpu", Dependencies: []string{"mlflow==3.5.0"}}, + }, bodies[0].Checkpoint) } func TestDeployDryRunNeedsNoWorkspace(t *testing.T) { @@ -358,11 +361,11 @@ func TestDeployDryRunListsCodeFiles(t *testing.T) { cfg, _ := writeCodeTree(t, map[string]string{"agent.py": "x\n", "tools/a.py": "a\n"}) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", codeConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) - assert.Contains(t, buf.String(), "Dry run: the model from code would include 2 file(s):\n agent.py\n tools/a.py\n") + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", entryConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) + assert.Contains(t, buf.String(), "Dry run: model.entry would send 2 file(s):\n agent.py\n tools/a.py\n") buf.Reset() - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", codeConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", entryConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) var got struct { Data deployResult `json:"data"` } @@ -370,9 +373,9 @@ func TestDeployDryRunListsCodeFiles(t *testing.T) { assert.Equal(t, []string{"agent.py", "tools/a.py"}, got.Data.CodeFiles) } -// codeConfigAt is codeDeployConfig with its code at root. -func codeConfigAt(root string) string { - return strings.Replace(codeDeployConfig, "root_path: .", "root_path: "+strconv.Quote(filepath.ToSlash(root)), 1) +// entryConfigAt is entryDeployConfig with its code at root. +func entryConfigAt(root string) string { + return strings.Replace(entryDeployConfig, "root_path: .", "root_path: "+strconv.Quote(filepath.ToSlash(root)), 1) } func TestDeployFlagValidation(t *testing.T) { diff --git a/cmd/air/deploycode.go b/cmd/air/deploycode.go index 721fadd7825..a2afa4ca6fe 100644 --- a/cmd/air/deploycode.go +++ b/cmd/air/deploycode.go @@ -15,7 +15,7 @@ import ( "unicode/utf8" ) -// Limits of a model from code, as the server enforces them. +// Limits of the Python code of model.entry, as the server enforces them. const ( // maxCodePackageBytes caps the code files plus the dependency lines. The server // also counts the small environment file it writes, so this check rejects only @@ -32,13 +32,13 @@ const pythonCacheDir = "__pycache__" // codePathSegmentRe matches a segment of a code file path the server can place. var codePathSegmentRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*$`) }) -// collectModelCode reads a model from code from code_source: model.entry and the -// other files under code_source.include_paths, as UTF-8 text, selected as a -// code_source snapshot selects them. Files git ignores aren't sent, and nor are +// collectEntryCode reads the Python code that serves a checkpoint from +// code_source: model.entry and the other files under code_source.include_paths, +// as UTF-8 text, selected as a code_source snapshot selects them. Files git ignores aren't sent, and nor are // hidden files and directories or __pycache__ directories, whose names the // server can't place or which hold no source. configPath locates a relative // code_source.root_path. -func collectModelCode(ctx context.Context, cfg *deployConfig, configPath string) (*modelPackageCode, error) { +func collectEntryCode(ctx context.Context, cfg *deployConfig, configPath string) (*modelPackagePythonModel, error) { root, err := resolveRootPath(ctx, cfg.CodeSource.RootPath, filepath.Dir(configPath)) if err != nil { return nil, err @@ -59,13 +59,14 @@ func collectModelCode(ctx context.Context, cfg *deployConfig, configPath string) } } - code := &modelPackageCode{EntryFile: *cfg.Model.Entry, Environment: cfg.servingEnvironment()} - if cfg.Task != nil { - code.Task = *cfg.Task + code := &modelPackagePythonModel{EntryFile: *cfg.Model.Entry} + var dependencies []string + if cfg.Environment != nil { + dependencies = cfg.Environment.Dependencies } // Check the listing before reading anything, so a root_path that holds large // data fails fast instead of being read into memory. - if err := checkCodeFiles(files, code.Environment.Dependencies); err != nil { + if err := checkCodeFiles(files, dependencies); err != nil { return nil, err } @@ -127,7 +128,7 @@ func ignoredTrackedFiles(ctx context.Context, root string, files []snapshotFile) func checkCodeFiles(files []snapshotFile, dependencies []string) error { // One of the files is the entry file, which the limit doesn't count. if len(files) > maxCodeFiles+1 { - return fmt.Errorf("code_source sends %d files, but a model from code can include at most %d besides model.entry; narrow it with code_source.include_paths", len(files), maxCodeFiles) + return fmt.Errorf("code_source sends %d files, but model.entry can bring at most %d other files; narrow it with code_source.include_paths", len(files), maxCodeFiles) } var size int64 for _, dependency := range dependencies { @@ -143,14 +144,14 @@ func checkCodeFiles(files []snapshotFile, dependencies []string) error { } size += f.size if size > maxCodePackageBytes { - return fmt.Errorf("code_source files and environment.dependencies total more than %d bytes, the most a model from code can hold; narrow code_source.include_paths", maxCodePackageBytes) + return fmt.Errorf("code_source files and environment.dependencies total more than %d bytes, the most a checkpoint's Python code can hold; narrow code_source.include_paths", maxCodePackageBytes) } } return nil } // codeFilePaths lists the files of code, entry file first, as dry runs show them. -func codeFilePaths(code *modelPackageCode) []string { +func codeFilePaths(code *modelPackagePythonModel) []string { paths := []string{code.EntryFile} for _, f := range code.Files { paths = append(paths, f.Path) diff --git a/cmd/air/deploycode_test.go b/cmd/air/deploycode_test.go index 8fa2fa7e0ab..fe57b18acdf 100644 --- a/cmd/air/deploycode_test.go +++ b/cmd/air/deploycode_test.go @@ -25,13 +25,13 @@ func codeTreeConfig(t *testing.T, dir string, files map[string]string) (*deployC for rel, content := range files { writeRepoFile(t, dir, filepath.FromSlash(rel), content) } - entry, task := "agent.py", "agent/v1/responses" + entry, task, path := "agent.py", "agent/v1/responses", "/Volumes/main/models/weights/support" cfg := &deployConfig{ - Model: &deployModelConfig{Type: modelTypeCode, Entry: &entry}, + Model: &deployModelConfig{Type: modelTypeCheckpoint, Path: &path, Entry: &entry}, Task: &task, CodeSource: &deployCodeSourceConfig{RootPath: dir}, Environment: &deployEnvironmentConfig{Dependencies: []string{"mlflow==3.5.0"}}, - Compute: &deployComputeConfig{WorkloadType: "CPU", WorkloadSize: "Small"}, + Compute: &deployComputeConfig{WorkloadType: "GPU_SMALL", WorkloadSize: "Small"}, } return cfg, filepath.Join(dir, "deploy.yaml") } @@ -50,12 +50,10 @@ func TestCollectModelCode(t *testing.T) { "configs/prompts/__init__.py.md": "not hidden\n", }) - code, err := collectModelCode(t.Context(), cfg, configPath) + code, err := collectEntryCode(t.Context(), cfg, configPath) require.NoError(t, err) assert.Equal(t, "agent.py", code.EntryFile) assert.Equal(t, "import tools.search\n", code.EntrySource) - assert.Equal(t, "agent/v1/responses", code.Task) - assert.Equal(t, modelPackageEnvironment{Runtime: "client.6", Dependencies: []string{"mlflow==3.5.0"}}, code.Environment) var paths []string for _, f := range code.Files { @@ -74,9 +72,9 @@ func TestCollectModelCodeIsDeterministic(t *testing.T) { cfg, configPath := writeCodeTree(t, map[string]string{ "agent.py": "x = 1\n", "b.py": "b\n", "a/c.py": "c\n", "a.py": "a\n", }) - first, err := collectModelCode(t.Context(), cfg, configPath) + first, err := collectEntryCode(t.Context(), cfg, configPath) require.NoError(t, err) - second, err := collectModelCode(t.Context(), cfg, configPath) + second, err := collectEntryCode(t.Context(), cfg, configPath) require.NoError(t, err) assert.Equal(t, first, second) } @@ -86,7 +84,7 @@ func TestCollectModelCodeIncludePaths(t *testing.T) { "agent.py": "x = 1\n", "tools/a.py": "a\n", "data/big.csv": "1,2\n", }) cfg.CodeSource.IncludePaths = []string{"agent.py", "tools"} - code, err := collectModelCode(t.Context(), cfg, configPath) + code, err := collectEntryCode(t.Context(), cfg, configPath) require.NoError(t, err) require.Len(t, code.Files, 1) assert.Equal(t, "tools/a.py", code.Files[0].Path) @@ -110,7 +108,7 @@ func TestCollectModelCodeErrors(t *testing.T) { t.Run(tt.name, func(t *testing.T) { cfg, configPath := writeCodeTree(t, tt.files) cfg.CodeSource.IncludePaths = tt.include - _, err := collectModelCode(t.Context(), cfg, configPath) + _, err := collectEntryCode(t.Context(), cfg, configPath) assert.ErrorContains(t, err, tt.wantErr) }) } @@ -121,7 +119,7 @@ func TestCollectModelCodeLeavesOutIgnoredFiles(t *testing.T) { "agent.py": "x\n", "credentials.json": "{}\n", "tools/a.py": "a\n", }) require.NoError(t, os.WriteFile(filepath.Join(cfg.CodeSource.RootPath, ".gitignore"), []byte("credentials.json\n"), 0o600)) - code, err := collectModelCode(t.Context(), cfg, configPath) + code, err := collectEntryCode(t.Context(), cfg, configPath) require.NoError(t, err) assert.Equal(t, []string{"agent.py", "tools/a.py"}, codeFilePaths(code)) } @@ -133,14 +131,14 @@ func TestCollectModelCodeRefusesTrackedIgnoredFiles(t *testing.T) { writeRepoFile(t, repo, ".gitignore", "creds.json\n") cfg.CodeSource.IncludePaths = []string{"agent.py", "creds.json"} - _, err := collectModelCode(t.Context(), cfg, configPath) + _, err := collectEntryCode(t.Context(), cfg, configPath) assert.ErrorContains(t, err, "code_source file creds.json matches a .gitignore rule but is tracked by git") } func TestCollectModelCodeOutsideGit(t *testing.T) { cfg, configPath := codeTreeConfig(t, t.TempDir(), map[string]string{"agent.py": "x\n", "tools/a.py": "a\n", "credentials.json": "{}\n"}) cfg.CodeSource.IncludePaths = []string{"agent.py", "tools"} - code, err := collectModelCode(t.Context(), cfg, configPath) + code, err := collectEntryCode(t.Context(), cfg, configPath) require.NoError(t, err) assert.Equal(t, []string{"agent.py", "tools/a.py"}, codeFilePaths(code)) } @@ -151,7 +149,7 @@ func TestCollectModelCodeChecksSizesBeforeReading(t *testing.T) { cfg, configPath := writeCodeTree(t, map[string]string{ "agent.py": "x\n", "data.bin": strings.Repeat("\xff", maxCodePackageBytes+1), }) - _, err := collectModelCode(t.Context(), cfg, configPath) + _, err := collectEntryCode(t.Context(), cfg, configPath) assert.ErrorContains(t, err, "total more than 4194304 bytes") } @@ -161,7 +159,7 @@ func TestCollectModelCodeRejectsSymlink(t *testing.T) { if err := os.Symlink("real.py", filepath.Join(dir, "link.py")); err != nil { t.Skipf("symlinks unavailable: %v", err) } - _, err := collectModelCode(t.Context(), cfg, configPath) + _, err := collectEntryCode(t.Context(), cfg, configPath) assert.ErrorContains(t, err, "code_source path link.py isn't a regular file") } @@ -171,8 +169,8 @@ func TestCollectModelCodeTooManyFiles(t *testing.T) { files[fmt.Sprintf("pkg/m%d.py", i)] = "" } cfg, configPath := writeCodeTree(t, files) - _, err := collectModelCode(t.Context(), cfg, configPath) - assert.ErrorContains(t, err, "code_source sends 1002 files, but a model from code can include at most 1000 besides model.entry") + _, err := collectEntryCode(t.Context(), cfg, configPath) + assert.ErrorContains(t, err, "code_source sends 1002 files, but model.entry can bring at most 1000 other files") } func TestSentCodePath(t *testing.T) { diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index 1bffa98433b..b3a7ed0bd89 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -25,14 +25,12 @@ import ( // Model sources, as model.type. const ( - modelTypeCheckpoint = "checkpoint" - modelTypeCode = "code" - modelTypeSavedModelDir = "saved_model_dir" - modelTypeRegistered = "registered" + modelTypeCheckpoint = "checkpoint" + modelTypeRegistered = "registered" ) // modelTypes lists model.type values in the order the help and errors name them. -var modelTypes = []string{modelTypeCheckpoint, modelTypeCode, modelTypeSavedModelDir, modelTypeRegistered} +var modelTypes = []string{modelTypeCheckpoint, modelTypeRegistered} // Audio tasks: their requests are multipart uploads of a recording, sent to the // endpoint's audio route rather than to invocations. @@ -76,18 +74,18 @@ var endpointNameRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCo // workloadSizes are the concurrency tiers a served entity scales to. var workloadSizes = []string{"Small", "Medium", "Large"} -// entryFileRe matches the name of a model from code's entry file, which the server -// places at the model's root. +// entryFileRe matches the name of model.entry, the Python file the server places +// at the model's root. var entryFileRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$`) }) // deployConfig is the `air deploy` YAML schema: a model to register in Unity // Catalog and the serving endpoint that serves it. type deployConfig struct { RegisteredModel string `yaml:"registered_model" help:"Unity Catalog model, as ... air deploy registers a new version of it, creating the model if it doesn't exist, unless model.type is registered." required:"yes" group:"Model"` - Model *deployModelConfig `yaml:"model" help:"What to serve: a checkpoint, a model from code, a model directory saved by MLflow, or an existing version." required:"yes" group:"Model"` + Model *deployModelConfig `yaml:"model" help:"What to serve: a checkpoint, or an existing version." required:"yes" group:"Model"` Task *string `yaml:"task" help:"The model's task as MLflow names it. Required with command, and then one Model Serving serves a command for, such as llm/v1/chat, llm/v1/embeddings or llm/v1/audio/transcriptions. Optional with model.entry, e.g. llm/v1/chat or agent/v1/responses, where it selects MLflow's signature; a model without a task has no signature." required:"when command is set" group:"Model"` Command *string `yaml:"command" help:"Shell command that starts a model server over a checkpoint's weights, at most 32,768 characters, serving task on port 8080. It runs from the directory where model.path appears under its own name. Set either command or model.entry." required:"when model.type is checkpoint and model.entry is not set" group:"Model"` - Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment to serve on and the packages to install, for a checkpoint or a model from code." group:"Model"` + Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment to serve a checkpoint on and the packages to install." group:"Model"` CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of model.entry, sent with the registration and stored in the model version, where anyone who can read the model can read it. Only the files under include_paths are sent, leaving out files git ignores, hidden files and directories (names starting with '.') and __pycache__ directories; run with --dry-run to list them." required:"when model.entry is set" group:"Model"` Endpoint string `yaml:"endpoint" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead, with the compute, environment variables and secrets in this file replacing the ones it has; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` @@ -101,9 +99,9 @@ type deployConfig struct { // deployModelConfig is the `model` block: where the served model comes from. type deployModelConfig struct { - Type string `yaml:"type" help:"One of checkpoint (weights in a volume, served by command or by the Python file model.entry), code (a Python file from code_source, no weights), saved_model_dir (a model directory saved by MLflow in a volume) or registered (an existing version of registered_model)." required:"yes"` - Path *string `yaml:"path" help:"Unity Catalog volume directory, as /Volumes////, holding a checkpoint's weights or a model saved by MLflow. Model Serving copies it, with its subdirectories, into the version, where it appears under its own name: command finds it there, and model.entry reads its path as context.artifacts[\"\"]." required:"when type is checkpoint or saved_model_dir"` - Entry *string `yaml:"entry" help:"Python file directly under code_source.root_path that builds the model and calls mlflow.models.set_model, e.g. model.py. Model Serving's model server loads it and calls its predict; for a checkpoint, set either entry or command." required:"when type is code"` + Type string `yaml:"type" help:"One of checkpoint (weights in a volume, served by command or by the Python file model.entry) or registered (an existing version of registered_model)." required:"yes"` + Path *string `yaml:"path" help:"Unity Catalog volume directory, as /Volumes////, holding a checkpoint's weights. Model Serving copies it, with its subdirectories, into the version, where it appears under its own name: command finds it there, and model.entry reads its path as context.artifacts[\"\"]." required:"when type is checkpoint"` + Entry *string `yaml:"entry" help:"Python file directly under code_source.root_path that builds the model and calls mlflow.models.set_model, e.g. model.py. Model Serving's model server loads it and calls its predict. Set either entry or command." required:"when type is checkpoint and command is not set"` ModelConfig map[string]any `yaml:"model_config" help:"Settings model.entry reads as context.model_config, as MLflow's log_model(model_config=...) records them."` Streamable *bool `yaml:"streamable" help:"Whether model.entry answers streaming requests with predict_stream. MLflow detects this from the class when it logs a model; here it is declared. Defaults to true for a chat or agent task and false otherwise."` Version *string `yaml:"version" help:"Version of registered_model to serve, e.g. 3." required:"when type is registered"` @@ -112,10 +110,10 @@ type deployModelConfig struct { // deployEnvironmentConfig is the `environment` block of a deployment. type deployEnvironmentConfig struct { Version *stringOrInt `yaml:"version" help:"Serverless environment to serve on: 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment (default: 6), as for air run. The model uses its Python version. Standard includes CUDA but not PyTorch; the Databricks AI environment adds PyTorch, Transformers and vLLM, and needs GPU compute. A command that doesn't install SGLang runs on the Databricks AI environment either way."` - Dependencies []string `yaml:"dependencies" help:"pip requirement lines to install, e.g. einops==0.8.0, at most 1,000. A model from code must list mlflow or mlflow-skinny."` + Dependencies []string `yaml:"dependencies" help:"pip requirement lines to install, e.g. einops==0.8.0, at most 1,000."` } -// deployCodeSourceConfig is the `code_source` block of a model from code. +// deployCodeSourceConfig is the `code_source` block: the local code of model.entry. type deployCodeSourceConfig struct { RootPath string `yaml:"root_path" help:"Directory holding the model's code. A relative path resolves against the YAML file's directory." required:"yes"` IncludePaths []string `yaml:"include_paths" help:"Files and directories to send, relative to root_path and without \"..\", e.g. [agent.py, tools]." required:"yes"` @@ -132,18 +130,14 @@ type deployComputeConfig struct { // addition to the required ones in requiredSourceFields. Every other source field // is rejected, so a field the source can't use is never silently dropped. var sourceFields = map[string][]string{ - modelTypeCheckpoint: {"model.entry", "model.model_config", "model.streamable", "task", "command", "code_source", "environment", "idempotency_token", "mlflow_experiment_path"}, - modelTypeCode: {"task", "environment", "idempotency_token", "mlflow_experiment_path"}, - modelTypeSavedModelDir: {"idempotency_token", "mlflow_experiment_path"}, - modelTypeRegistered: {}, + modelTypeCheckpoint: {"model.entry", "model.model_config", "model.streamable", "task", "command", "code_source", "environment", "idempotency_token", "mlflow_experiment_path"}, + modelTypeRegistered: {}, } // requiredSourceFields lists the fields each model.type requires. var requiredSourceFields = map[string][]string{ - modelTypeCheckpoint: {"model.path"}, - modelTypeCode: {"model.entry", "code_source"}, - modelTypeSavedModelDir: {"model.path"}, - modelTypeRegistered: {"model.version"}, + modelTypeCheckpoint: {"model.path"}, + modelTypeRegistered: {"model.version"}, } // validate runs structural validation over the whole config, returning the @@ -276,7 +270,6 @@ func (c *deployConfig) validateSource() error { return nil } -// validate checks the environment version and dependency lines. // validateCheckpoint checks the fields of a checkpoint: weights to copy, and // exactly one way to serve them. func (c *deployConfig) validateCheckpoint() error { @@ -305,6 +298,7 @@ func validateVolumeDirectory(field, v string) error { return nil } +// validate checks the environment version and dependency lines. func (e *deployEnvironmentConfig) validate() error { if e.Version != nil { if _, _, err := e.runtimeVersion(); err != nil { @@ -438,6 +432,22 @@ func deploySchema() configField { task: llm/v1/chat command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --served-model-name base --host 0.0.0.0 --port 8080 endpoint: qwen-chat + compute: + workload_type: GPU_MEDIUM + workload_size: Small + + A checkpoint served by Python code, model.py, which calls mlflow.models.set_model: + registered_model: main.models.whisper + model: + type: checkpoint + path: /Volumes/main/models/weights/whisper + entry: model.py + code_source: + root_path: . + include_paths: [model.py] + environment: + version: databricks_ai_v6 + endpoint: whisper compute: workload_type: GPU_MEDIUM workload_size: Small`, diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index b2a8870d6a0..6525e8801de 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -26,9 +26,11 @@ env_variables: LOG_LEVEL: info ` -const codeDeployConfig = `registered_model: main.agents.support +// entryDeployConfig is a checkpoint served by Python code, model.entry. +const entryDeployConfig = `registered_model: main.agents.support model: - type: code + type: checkpoint + path: /Volumes/main/models/weights/support entry: agent.py task: agent/v1/responses code_source: @@ -39,7 +41,7 @@ environment: dependencies: [mlflow==3.5.0] endpoint: support-agent compute: - workload_type: CPU + workload_type: GPU_SMALL workload_size: Small scale_to_zero: true ` @@ -106,7 +108,7 @@ func TestLoadDeployConfigPythonCheckpoint(t *testing.T) { assert.Equal(t, "client.6-gpu", cfg.servingRuntime()) streamable := false - code := &modelPackageCode{EntryFile: "model.py", EntrySource: "import mlflow\n", Files: []modelPackageFile{{Path: "util/a.py", Content: "x = 1\n"}}} + code := &modelPackagePythonModel{EntryFile: "model.py", EntrySource: "import mlflow\n", Files: []modelPackageFile{{Path: "util/a.py", Content: "x = 1\n"}}} req := cfg.registrationRequest("key-1", code) assert.Equal(t, registerModelPackageRequest{ RegisteredModel: "main.ml.whisper", @@ -131,16 +133,16 @@ func TestLoadDeployConfigPythonCheckpointWithoutTask(t *testing.T) { text = strings.Replace(text, " model_config: {language: en, beams: 4}\n streamable: false\n", "", 1) cfg, err := loadDeployConfigText(t, text) require.NoError(t, err) - req := cfg.registrationRequest("k", &modelPackageCode{EntryFile: "model.py"}) + req := cfg.registrationRequest("k", &modelPackagePythonModel{EntryFile: "model.py"}) assert.Empty(t, req.Checkpoint.Task) assert.Nil(t, req.Checkpoint.PythonModel.Streamable) assert.Empty(t, req.Checkpoint.PythonModel.ModelConfig) } -func TestLoadDeployConfigCodeOnCPU(t *testing.T) { - cfg, err := loadDeployConfigText(t, codeDeployConfig) +func TestLoadDeployConfigEntryCheckpointServedEntity(t *testing.T) { + cfg, err := loadDeployConfigText(t, entryDeployConfig) require.NoError(t, err) - assert.Equal(t, "client.5", cfg.servingRuntime()) + assert.Equal(t, "client.5-gpu", cfg.servingRuntime()) assert.Nil(t, cfg.servedEnvironmentVars()) entity := cfg.servedEntity("4") @@ -161,20 +163,23 @@ compute: {workload_type: CPU, workload_size: Small} assert.Equal(t, "7", *cfg.Model.Version) } -func TestLoadDeployConfigSavedModelDir(t *testing.T) { - cfg, err := loadDeployConfigText(t, `registered_model: main.ml.churn -model: {type: saved_model_dir, path: /Volumes/main/ml/runs/model} -mlflow_experiment_path: /Shared/churn +func TestLoadDeployConfigRejectsUnsupportedModelTypes(t *testing.T) { + for _, modelType := range []string{"code", "saved_model_dir"} { + t.Run(modelType, func(t *testing.T) { + _, err := loadDeployConfigText(t, `registered_model: main.ml.churn +model: {type: `+modelType+`, path: /Volumes/main/ml/runs/model} endpoint: churn -compute: {workload_type: CPU, workload_size: Small} +compute: {workload_type: GPU_SMALL, workload_size: Small} `) + assert.ErrorContains(t, err, `model.type must be one of checkpoint, registered, got "`+modelType+`"`) + }) + } +} + +func TestLoadDeployConfigExperimentPath(t *testing.T) { + cfg, err := loadDeployConfigText(t, checkpointDeployConfig+"mlflow_experiment_path: /Shared/qwen\n") require.NoError(t, err) - assert.Equal(t, registerModelPackageRequest{ - RegisteredModel: "main.ml.churn", - IdempotencyKey: "k", - SavedModel: &modelPackageSavedModel{VolumePath: "/Volumes/main/ml/runs/model"}, - ExperimentPath: "/Shared/churn", - }, cfg.registrationRequest("k", nil)) + assert.Equal(t, "/Shared/qwen", cfg.registrationRequest("k", nil).ExperimentPath) } func TestLoadDeployConfigOverride(t *testing.T) { @@ -207,7 +212,7 @@ func TestDeployConfigValidation(t *testing.T) { }{ {"bad model name", replaceFirst("main.models.qwen", "qwen"), "registered_model must be a Unity Catalog model name"}, {"backticked model name", replaceFirst("main.models.qwen", "main.models.`qwen`"), "registered_model must be a Unity Catalog model name"}, - {"unknown type", replaceFirst("type: checkpoint", "type: pickle"), `model.type must be one of checkpoint, code, saved_model_dir, registered, got "pickle"`}, + {"unknown type", replaceFirst("type: checkpoint", "type: pickle"), `model.type must be one of checkpoint, registered, got "pickle"`}, {"checkpoint without task", replaceFirst("task: llm/v1/chat\n", ""), "task is required when command is set"}, {"checkpoint without command", dropLine("command:"), "exactly one of command and model.entry is required when model.type is checkpoint"}, {"checkpoint with command and entry", replaceFirst(" type: checkpoint\n", " type: checkpoint\n entry: a.py\n"), "exactly one of command and model.entry is required when model.type is checkpoint"}, @@ -248,24 +253,22 @@ func TestDeployConfigValidation(t *testing.T) { } func TestDeployConfigSourceFields(t *testing.T) { - base := "endpoint: e\ncompute: {workload_type: CPU, workload_size: Small}\nregistered_model: a.b.c\n" + base := "endpoint: e\ncompute: {workload_type: GPU_SMALL, workload_size: Small}\nregistered_model: a.b.c\n" tests := []struct { name string source string wantErr string }{ - {"code without entry", "model: {type: code}\ncode_source: {root_path: .}\n", "model.entry is required when model.type is code"}, - {"code without code_source", "model: {type: code, entry: agent.py}\n", "code_source is required when model.type is code"}, - {"code with command", "model: {type: code, entry: agent.py}\ncode_source: {root_path: .}\ncommand: x\n", "command can't be set when model.type is code"}, - {"code entry in a directory", "model: {type: code, entry: src/agent.py}\ncode_source: {root_path: .}\n", "model.entry must name a .py file directly under code_source.root_path"}, - {"code entry not python", "model: {type: code, entry: agent.ipynb}\ncode_source: {root_path: .}\n", "model.entry must name a .py file"}, - {"code without include_paths", "model: {type: code, entry: agent.py}\ncode_source: {root_path: .}\n", "code_source.include_paths is required: list the files and directories to send"}, - {"code include pathspec magic", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\":!creds.json\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got ":!creds.json"`}, - {"code include glob", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\"*\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got "*"`}, - {"code include padded", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\" agent.py\"]}\n", `got " agent.py"`}, - {"code include traversal", "model: {type: code, entry: agent.py}\ncode_source: {root_path: ., include_paths: [../x]}\n", "code_source.include_paths cannot contain '..' traversal"}, - {"saved model with task", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\ntask: llm/v1/chat\n", "task can't be set when model.type is saved_model_dir"}, - {"saved model with environment", "model: {type: saved_model_dir, path: /Volumes/a/b/c/m}\nenvironment: {version: 6}\n", "environment can't be set when model.type is saved_model_dir"}, + {"checkpoint entry without code_source", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\n", "code_source is required when model.entry is set"}, + {"checkpoint entry in a directory", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: src/agent.py}\ncode_source: {root_path: ., include_paths: [src]}\n", "model.entry must name a .py file directly under code_source.root_path"}, + {"checkpoint entry not python", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.ipynb}\ncode_source: {root_path: ., include_paths: [agent.ipynb]}\n", "model.entry must name a .py file"}, + {"checkpoint entry without include_paths", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: .}\n", "code_source.include_paths is required: list the files and directories to send"}, + {"checkpoint include pathspec magic", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\":!creds.json\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got ":!creds.json"`}, + {"checkpoint include glob", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\"*\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got "*"`}, + {"checkpoint include padded", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\" agent.py\"]}\n", `got " agent.py"`}, + {"checkpoint include traversal", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [../x]}\n", "code_source.include_paths cannot contain '..' traversal"}, + {"checkpoint entry with command", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [agent.py]}\ntask: llm/v1/chat\ncommand: x\n", "exactly one of command and model.entry is required"}, + {"registered with entry", "model: {type: registered, version: 3, entry: agent.py}\n", "model.entry can't be set when model.type is registered"}, {"registered without version", "model: {type: registered}\n", "model.version is required when model.type is registered"}, {"registered with path", "model: {type: registered, version: 3, path: /Volumes/a/b/c/m}\n", "model.path can't be set when model.type is registered"}, {"registered with token", "model: {type: registered, version: 3}\nidempotency_token: k\n", "idempotency_token can't be set when model.type is registered"}, diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go index b54f27302c4..92f94648113 100644 --- a/cmd/air/deployregister.go +++ b/cmd/air/deployregister.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "net/http" + "slices" "time" "github.com/databricks/cli/libs/auth" @@ -45,18 +46,17 @@ const ( stagingStateFailed = "FAILED" ) -// registerModelPackageRequest is the RegisterModelPackageRequest body. Exactly one -// of Checkpoint, Code and SavedModel is set. +// registerModelPackageRequest is the RegisterModelPackageRequest body. type registerModelPackageRequest struct { RegisteredModel string `json:"registered_model"` IdempotencyKey string `json:"idempotency_key"` Checkpoint *modelPackageCheckpoint `json:"checkpoint,omitempty"` - Code *modelPackageCode `json:"code,omitempty"` - SavedModel *modelPackageSavedModel `json:"saved_model,omitempty"` ExperimentPath string `json:"experiment_path,omitempty"` RetryFailedStaging bool `json:"retry_failed_staging,omitempty"` } +// modelPackageCheckpoint is a checkpoint's weights in a volume and what serves +// them: exactly one of ServingCommand and PythonModel is set. type modelPackageCheckpoint struct { VolumePath string `json:"volume_path"` ServingCommand string `json:"serving_command,omitempty"` @@ -77,23 +77,13 @@ type modelPackagePythonModel struct { // baseEnvironmentDatabricksAI names the Databricks AI environment to the server. const baseEnvironmentDatabricksAI = "databricks-ai" -type modelPackageCode struct { - EntryFile string `json:"entry_file"` - EntrySource string `json:"entry_source"` - Files []modelPackageFile `json:"files,omitempty"` - Task string `json:"task,omitempty"` - Environment modelPackageEnvironment `json:"environment"` -} - +// modelPackageFile is a source file the entry file imports, by path relative to +// code_source.root_path. type modelPackageFile struct { Path string `json:"path"` Content string `json:"content"` } -type modelPackageSavedModel struct { - VolumePath string `json:"volume_path"` -} - type modelPackageEnvironment struct { Runtime string `json:"runtime"` Dependencies []string `json:"dependencies,omitempty"` @@ -143,8 +133,8 @@ type stagingStatusResponse struct { Status *modelStagingStatus `json:"status,omitempty"` } -// weightsState is the staging state of a checkpoint's weights, or "" when the -// response reports none. +// weightsState is the staging state of a checkpoint's weights, or "" before a +// response reports it. func (r *registerModelPackageResponse) weightsState() string { if r.StagingStatus == nil || r.StagingStatus.ModelArtifact == nil { return "" @@ -152,8 +142,8 @@ func (r *registerModelPackageResponse) weightsState() string { return r.StagingStatus.ModelArtifact.State } -// environmentState is the staging state of the serving environment, or "" when the -// response reports none. +// environmentState is the staging state of the serving environment, or "" before +// a response reports it. func (r *registerModelPackageResponse) environmentState() string { if r.StagingStatus == nil || r.StagingStatus.EnvironmentArtifact == nil { return "" @@ -173,9 +163,10 @@ func (e *finalizeError) Error() string { return e.err.Error() } func (e *finalizeError) Unwrap() error { return e.err } -// registrationRequest builds the request that registers cfg's model under key. -// code is the collected code of model.entry, and is nil when there is none. -func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) registerModelPackageRequest { +// registrationRequest builds the request that registers cfg's checkpoint under key. +// code is the collected code of model.entry, and is nil when a command serves the +// checkpoint. +func (c *deployConfig) registrationRequest(key string, code *modelPackagePythonModel) registerModelPackageRequest { req := registerModelPackageRequest{ RegisteredModel: c.RegisteredModel, IdempotencyKey: key, @@ -183,45 +174,39 @@ func (c *deployConfig) registrationRequest(key string, code *modelPackageCode) r if c.MLflowExperimentPath != nil { req.ExperimentPath = *c.MLflowExperimentPath } - switch c.Model.Type { - case modelTypeCheckpoint: - checkpoint := &modelPackageCheckpoint{VolumePath: *c.Model.Path, Environment: c.servingEnvironment()} - if c.Task != nil { - checkpoint.Task = *c.Task + checkpoint := &modelPackageCheckpoint{VolumePath: *c.Model.Path, Environment: c.servingEnvironment()} + if c.Task != nil { + checkpoint.Task = *c.Task + } + if c.Command != nil { + checkpoint.ServingCommand = *c.Command + } else { + checkpoint.PythonModel = &modelPackagePythonModel{ + EntryFile: code.EntryFile, + EntrySource: code.EntrySource, + Files: code.Files, + Streamable: c.Model.Streamable, } - if c.Command != nil { - checkpoint.ServingCommand = *c.Command - } else { - checkpoint.PythonModel = &modelPackagePythonModel{ - EntryFile: code.EntryFile, - EntrySource: code.EntrySource, - Files: code.Files, - Streamable: c.Model.Streamable, - } - if c.Model.ModelConfig != nil { - // The YAML decoded as JSON-compatible values; the server reads a JSON object. - raw, _ := json.Marshal(c.Model.ModelConfig) - checkpoint.PythonModel.ModelConfig = string(raw) - } + if c.Model.ModelConfig != nil { + // The YAML decoded as JSON-compatible values; the server reads a JSON object. + raw, _ := json.Marshal(c.Model.ModelConfig) + checkpoint.PythonModel.ModelConfig = string(raw) } - req.Checkpoint = checkpoint - case modelTypeCode: - req.Code = code - case modelTypeSavedModelDir: - req.SavedModel = &modelPackageSavedModel{VolumePath: *c.Model.Path} } + req.Checkpoint = checkpoint return req } // registerAndFinalize registers req's version and makes it ready to serve, -// calling progress with the registration response and every staging update. A +// calling progress with the registration response and every staging update. The // checkpoint's weights are copied into the version after registering: while they -// are, it polls the staging status, a read. It then finalizes the version, for -// every source; the server checks that the version is ready to finalize. A -// version that is already ready, such as one an earlier run finalized, is -// returned as it is. On error it returns the last response it got, nil before -// the first one; errStagingFailed when staging failed, and a *finalizeError when -// finalizing did. +// are, it polls the staging status, a read. It then finalizes the version; the +// server checks that the version is ready to finalize. A version that is already +// ready, such as one an earlier run finalized, is returned as it is. Each +// response's staging status replaces the last one, and a status without both +// artifacts' states is an error, not progress. On error it returns the last +// response it got, nil before the first one; errStagingFailed when staging +// failed, and a *finalizeError when finalizing did. func registerAndFinalize(ctx context.Context, w *databricks.WorkspaceClient, req registerModelPackageRequest, interval time.Duration, progress func(*registerModelPackageResponse)) (*registerModelPackageResponse, error) { apiClient, err := client.New(w.Config) if err != nil { @@ -235,14 +220,15 @@ func registerAndFinalize(ctx context.Context, w *databricks.WorkspaceClient, req if resp.ModelVersion == "" { return nil, withCode(codeInternalError, true, fmt.Errorf("registering a version of %s returned no version", req.RegisteredModel)) } + if err := checkStagingStatus(resp.StagingStatus, req.RegisteredModel, resp.ModelVersion); err != nil { + return nil, err + } progress(&resp) if resp.Ready { return &resp, nil } - if req.Checkpoint != nil { - if err := p.waitForWeights(ctx, req.RegisteredModel, &resp, progress); err != nil { - return &resp, err - } + if err := p.waitForWeights(ctx, req.RegisteredModel, &resp, progress); err != nil { + return &resp, err } if resp.weightsState() == stagingStateFailed { return &resp, errStagingFailed @@ -255,27 +241,38 @@ func registerAndFinalize(ctx context.Context, w *databricks.WorkspaceClient, req if finalized.ModelVersion != resp.ModelVersion { return &resp, &finalizeError{withCode(codeInternalError, true, fmt.Errorf("the server finalized version %q instead", stripControl(finalized.ModelVersion)))} } + if err := checkStagingStatus(finalized.StagingStatus, req.RegisteredModel, resp.ModelVersion); err != nil { + return &resp, &finalizeError{err} + } resp.Ready = true - resp.StagingStatus = mergeStagingStatus(resp.StagingStatus, finalized.StagingStatus) + resp.StagingStatus = finalized.StagingStatus return &resp, nil } -// mergeStagingStatus is newer, with each artifact it doesn't report kept from -// older. -func mergeStagingStatus(older, newer *modelStagingStatus) *modelStagingStatus { - if newer == nil { - return older - } - merged := *newer - if older != nil { - if merged.ModelArtifact == nil { - merged.ModelArtifact = older.ModelArtifact - } - if merged.EnvironmentArtifact == nil { - merged.EnvironmentArtifact = older.EnvironmentArtifact - } +// knownStagingStates are the artifact states the staging status API reports. +var knownStagingStates = []string{stagingStateNotStaged, stagingStateInProgress, stagingStateSucceeded, stagingStateFailed} + +// checkStagingStatus checks that status, from a response about version of model, +// reports a known state for both the weights and the serving environment, as the +// server always does. Anything else is a response air deploy can't act on, so it +// fails rather than being read as progress until the timeout. +func checkStagingStatus(status *modelStagingStatus, model, version string) error { + var problem string + switch { + case status == nil: + problem = "it has no staging status" + case status.ModelArtifact == nil || status.ModelArtifact.State == "": + problem = "its staging status has no state for the weights" + case !slices.Contains(knownStagingStates, status.ModelArtifact.State): + problem = fmt.Sprintf("its staging status has an unknown state %q for the weights", stripControl(status.ModelArtifact.State)) + case status.EnvironmentArtifact == nil || status.EnvironmentArtifact.State == "": + problem = "its staging status has no state for the serving environment" + case !slices.Contains(knownStagingStates, status.EnvironmentArtifact.State): + problem = fmt.Sprintf("its staging status has an unknown state %q for the serving environment", stripControl(status.EnvironmentArtifact.State)) + default: + return nil } - return &merged + return withCode(codeInternalError, true, fmt.Errorf("air deploy can't read Model Serving's response about version %s of %s: %s", stripControl(version), model, problem)) } // modelServingPoller sends the requests of a registration that can take many @@ -286,17 +283,12 @@ type modelServingPoller struct { interval time.Duration } -// waitForWeights polls the staging status of resp's version of model, updating -// resp, until its weights are copied, failed to copy, or have no staging record. -// A state it doesn't know, or a status without one, is waited out like a copy in -// progress, until the context ends. +// waitForWeights polls the staging status of resp's version of model, replacing +// resp's status with each one, until its weights are copied, failed to copy, or +// have no staging record. resp's status was checked, and so is each new one. func (p modelServingPoller) waitForWeights(ctx context.Context, model string, resp *registerModelPackageResponse, progress func(*registerModelPackageResponse)) error { statusReq := stagingStatusRequest{Model: model, ModelVersion: resp.ModelVersion} - for { - switch resp.weightsState() { - case stagingStateSucceeded, stagingStateFailed, stagingStateNotStaged: - return nil - } + for resp.weightsState() == stagingStateInProgress { if err := sleepOrCancel(ctx, p.interval); err != nil { return err } @@ -304,9 +296,13 @@ func (p modelServingPoller) waitForWeights(ctx context.Context, model string, re if err := p.post(ctx, stagingStatusPath, statusReq, &status); err != nil { return err } - resp.StagingStatus = mergeStagingStatus(resp.StagingStatus, status.Status) + if err := checkStagingStatus(status.Status, model, resp.ModelVersion); err != nil { + return err + } + resp.StagingStatus = status.Status progress(resp) } + return nil } // post sends a request, repeating it after a transient failure the SDK doesn't diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go index b9a43f7330c..6c8948a0d19 100644 --- a/cmd/air/deployregister_test.go +++ b/cmd/air/deployregister_test.go @@ -252,26 +252,81 @@ func TestRegisterAndFinalizeReturnsWhyTheServerWontFinalize(t *testing.T) { assert.Len(t, fake.finalizeRequests, 1) } -func TestRegisterAndFinalizeWaitsOutAStatusWithoutAState(t *testing.T) { - fake := newFakeModelServing(t, stagingInProgress) - fake.statuses = []string{`{}`, `{"status": {"state": "IN_PROGRESS"}}`, statusSucceeded} +// Staging statuses that break the contract: the server always reports a known state +// for both the weights and the serving environment. +var malformedStagingStatuses = map[string]struct { + status string + wantErr string +}{ + "no status": {`null`, "it has no staging status"}, + "no weights": {`{"state": "IN_PROGRESS", "environment_artifact": {"state": "SUCCEEDED"}}`, "its staging status has no state for the weights"}, + "weights without a state": {`{"state": "IN_PROGRESS", "model_artifact": {}, "environment_artifact": {"state": "SUCCEEDED"}}`, "its staging status has no state for the weights"}, + "unknown weights state": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "COPYING"}, "environment_artifact": {"state": "SUCCEEDED"}}`, `unknown state "COPYING" for the weights`}, + "no environment": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}}`, "its staging status has no state for the serving environment"}, + "unknown environment state": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "IN_PROGRESS"}, "environment_artifact": {"state": "BUILDING"}}`, `unknown state "BUILDING" for the serving environment`}, + "control characters in the state": {`{"state": "IN_PROGRESS", "model_artifact": {"state": "X\u001b[31m"}, "environment_artifact": {"state": "SUCCEEDED"}}`, `unknown state "X" for the weights`}, +} - resp, states, err := registerTestCheckpoint(t, fake, checkpointRequest()) - require.NoError(t, err) - assert.True(t, resp.Ready) - assert.Equal(t, []string{"IN_PROGRESS", "IN_PROGRESS", "IN_PROGRESS", "SUCCEEDED"}, states) - assert.Len(t, fake.statusRequests, 3) - assert.Len(t, fake.finalizeRequests, 1) +func TestRegisterAndFinalizeRejectsAMalformedRegistrationStatus(t *testing.T) { + for name, tt := range malformedStagingStatuses { + t.Run(name, func(t *testing.T) { + fake := newFakeModelServing(t, `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3", "ready": false, "staging_status": `+tt.status+`}`) + + resp, states, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, "air deploy can't read Model Serving's response about version 3 of main.models.qwen: ") + assert.ErrorContains(t, err, tt.wantErr) + assert.Nil(t, resp) + assert.Empty(t, states) + assert.Empty(t, fake.statusRequests) + assert.Empty(t, fake.finalizeRequests) + }) + } } -func TestRegisterAndFinalizeKeepsAnEnvironmentStateTheFinalizeOmits(t *testing.T) { +func TestRegisterAndFinalizeRejectsAMalformedPolledStatus(t *testing.T) { + for name, tt := range malformedStagingStatuses { + t.Run(name, func(t *testing.T) { + fake := newFakeModelServing(t, stagingInProgress) + fake.statuses = []string{`{"status": ` + tt.status + `}`, statusSucceeded} + + resp, states, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, tt.wantErr) + code, retryable := deployErrorCode(err) + assert.Equal(t, codeInternalError, code) + assert.True(t, retryable) + // It stops at the malformed status instead of reading it as progress. + assert.Equal(t, []string{"IN_PROGRESS"}, states) + assert.Len(t, fake.statusRequests, 1) + assert.Empty(t, fake.finalizeRequests) + assert.False(t, resp.Ready) + }) + } +} + +func TestRegisterAndFinalizeRejectsAMalformedFinalizedStatus(t *testing.T) { + for name, tt := range malformedStagingStatuses { + t.Run(name, func(t *testing.T) { + fake := newFakeModelServing(t, stagingSucceeded) + fake.finalizations = []string{`{"model_version": "3", "staging_status": ` + tt.status + `}`} + + resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + _, ok := errors.AsType[*finalizeError](err) + assert.True(t, ok) + assert.ErrorContains(t, err, tt.wantErr) + assert.False(t, resp.Ready) + }) + } +} + +func TestRegisterAndFinalizeUsesTheFinalizedStatusAsItIs(t *testing.T) { environmentFailed := strings.Replace(stagingSucceeded, `"environment_artifact": {"state": "SUCCEEDED"}`, `"environment_artifact": {"state": "FAILED"}`, 1) fake := newFakeModelServing(t, environmentFailed) - fake.finalizations = []string{`{"model_version": "3", "staging_status": {"model_artifact": {"state": "SUCCEEDED"}}}`} + fake.finalizations = []string{`{"model_version": "3", "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}`} resp, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) require.NoError(t, err) - assert.Equal(t, stagingStateFailed, resp.environmentState()) + // The finalize response's status replaces the registration's, not merged with it. + assert.Equal(t, stagingStateSucceeded, resp.environmentState()) } func TestRegisterAndFinalizeRejectsAnotherFinalizedVersion(t *testing.T) { @@ -292,18 +347,6 @@ func TestRegisterAndFinalizeRejectsARegistrationWithoutAVersion(t *testing.T) { assert.Empty(t, fake.finalizeRequests) } -func TestRegisterAndFinalizeFinalizesAModelFromCodeWithoutPolling(t *testing.T) { - fake := newFakeModelServing(t, `{"model_id": "m-2", "registered_model": "a.b.c", "model_version": "1", "ready": false}`) - fake.finalizations = []string{`{"model_id": "m-2", "registered_model": "a.b.c", "model_version": "1"}`} - req := registerModelPackageRequest{RegisteredModel: "a.b.c", IdempotencyKey: "k", Code: &modelPackageCode{EntryFile: "agent.py"}} - - resp, _, err := registerTestCheckpoint(t, fake, req) - require.NoError(t, err) - assert.True(t, resp.Ready) - assert.Empty(t, fake.statusRequests) - assert.Equal(t, []finalizeModelPackageRequest{{RegisteredModel: "a.b.c", ModelVersion: "1"}}, fake.finalizeRequests) -} - func TestRegisterAndFinalizeRetriesTransientErrors(t *testing.T) { fake := newFakeModelServing(t, serverErrorResponse, stagingInProgress) fake.statuses = []string{serverErrorResponse, statusSucceeded} @@ -367,25 +410,16 @@ func TestRegisterPythonCheckpointRequestJSON(t *testing.T) { } func TestRegisterModelPackageRequestJSON(t *testing.T) { - raw, err := json.Marshal(registerModelPackageRequest{ - RegisteredModel: "a.b.c", - IdempotencyKey: "k", - Code: &modelPackageCode{ - EntryFile: "agent.py", - EntrySource: "x = 1\n", - Files: []modelPackageFile{{Path: "tools/a.py", Content: ""}}, - Environment: modelPackageEnvironment{Runtime: "client.6", Dependencies: []string{"mlflow"}}, - }, - }) + raw, err := json.Marshal(checkpointRequest()) require.NoError(t, err) assert.JSONEq(t, `{ - "registered_model": "a.b.c", - "idempotency_key": "k", - "code": { - "entry_file": "agent.py", - "entry_source": "x = 1\n", - "files": [{"path": "tools/a.py", "content": ""}], - "environment": {"runtime": "client.6", "dependencies": ["mlflow"]} + "registered_model": "main.models.qwen", + "idempotency_key": "key-1", + "checkpoint": { + "volume_path": "/Volumes/main/models/weights/qwen3_8b", + "serving_command": "python -m vllm.entrypoints.openai.api_server --model qwen3_8b", + "task": "llm/v1/chat", + "environment": {"runtime": "client.6-gpu"} } }`, string(raw)) } From 4ff964849b64f626f6ff4aa1700d9516097df15d Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 17 Nov 2026 02:42:40 +0000 Subject: [PATCH 08/11] air deploy: reject model_config JSON can't hold, and read only the code files listed model_config values YAML holds but JSON can't, such as .nan, .inf and mappings with keys that aren't strings, were dropped silently, so the model got no config; they are now rejected when the config loads. A code file replaced after it was listed, such as by a symlink, is refused instead of read. A registered version that isn't a number is an error. Co-authored-by: Isaac --- cmd/air/deploy.go | 5 ++++- cmd/air/deploycode.go | 39 +++++++++++++++++++++++++++++++++- cmd/air/deploycode_test.go | 28 ++++++++++++++++++++++++ cmd/air/deployconfig.go | 17 +++++++++++++++ cmd/air/deployconfig_test.go | 31 ++++++++++++++++++++++----- cmd/air/deployregister.go | 16 ++++++++------ cmd/air/deployregister_test.go | 8 +++++++ 7 files changed, 131 insertions(+), 13 deletions(-) diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go index 70aaff99db7..fb6d5730609 100644 --- a/cmd/air/deploy.go +++ b/cmd/air/deploy.go @@ -295,7 +295,10 @@ func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg // Printed to stderr in every output mode, before the first request, so a run // that is killed can still be continued with this key. cmdio.LogString(ctx, fmt.Sprintf("Registering a version of %s with idempotency key %s", cfg.RegisteredModel, key)) - req := cfg.registrationRequest(key, code) + req, err := cfg.registrationRequest(key, code) + if err != nil { + return nil, withCode(codeInvalidArgs, false, err) + } req.RetryFailedStaging = opts.retryFailedStaging progress := func(*registerModelPackageResponse) {} diff --git a/cmd/air/deploycode.go b/cmd/air/deploycode.go index a2afa4ca6fe..96fb72f129c 100644 --- a/cmd/air/deploycode.go +++ b/cmd/air/deploycode.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -73,7 +74,10 @@ func collectEntryCode(ctx context.Context, cfg *deployConfig, configPath string) foundEntry := false for _, f := range files { rel := filepath.ToSlash(f.rel) - content, err := os.ReadFile(filepath.Join(root, f.rel)) + content, err := readListedFile(filepath.Join(root, f.rel), f.size) + if errors.Is(err, errCodeFileChanged) { + return nil, fmt.Errorf("code_source file %s changed while air deploy was reading it; run air deploy again", rel) + } if err != nil { return nil, fmt.Errorf("failed to read code_source file %s: %w", rel, err) } @@ -93,6 +97,39 @@ func collectEntryCode(ctx context.Context, cfg *deployConfig, configPath string) return code, nil } +// errCodeFileChanged reports a code file that isn't the one listed anymore. +var errCodeFileChanged = errors.New("code file changed since it was listed") + +// readListedFile reads the regular file of size bytes listed at path. It opens +// the file before checking it, so a file replaced after the listing, such as by a +// symlink to a file outside the project, is refused rather than sent. +func readListedFile(path string, size int64) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + opened, err := f.Stat() + if err != nil { + return nil, err + } + atPath, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !atPath.Mode().IsRegular() || !os.SameFile(opened, atPath) || opened.Size() != size { + return nil, errCodeFileChanged + } + content, err := io.ReadAll(io.LimitReader(f, size+1)) + if err != nil { + return nil, err + } + if int64(len(content)) != size { + return nil, errCodeFileChanged + } + return content, nil +} + // ignoredTrackedFiles returns the files that a .gitignore rule matches although // git tracks them: ls-files lists tracked files whatever the rules say, and the // rules are the user's statement of what not to share. diff --git a/cmd/air/deploycode_test.go b/cmd/air/deploycode_test.go index fe57b18acdf..04717cd132c 100644 --- a/cmd/air/deploycode_test.go +++ b/cmd/air/deploycode_test.go @@ -163,6 +163,34 @@ func TestCollectModelCodeRejectsSymlink(t *testing.T) { assert.ErrorContains(t, err, "code_source path link.py isn't a regular file") } +func TestReadListedFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "model.py") + require.NoError(t, os.WriteFile(path, []byte("x = 1\n"), 0o600)) + content, err := readListedFile(path, 6) + require.NoError(t, err) + assert.Equal(t, "x = 1\n", string(content)) +} + +func TestReadListedFileRefusesAFileReplacedByASymlink(t *testing.T) { + dir := t.TempDir() + secret := filepath.Join(dir, "secret") + require.NoError(t, os.WriteFile(secret, []byte("token\n"), 0o600)) + path := filepath.Join(dir, "model.py") + if err := os.Symlink(secret, path); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + // Listed as a regular file of the secret's size, then replaced by the link. + _, err := readListedFile(path, 6) + assert.ErrorIs(t, err, errCodeFileChanged) +} + +func TestReadListedFileRefusesAFileThatChangedSize(t *testing.T) { + path := filepath.Join(t.TempDir(), "model.py") + require.NoError(t, os.WriteFile(path, []byte("x = 1\ny = 2\n"), 0o600)) + _, err := readListedFile(path, 6) + assert.ErrorIs(t, err, errCodeFileChanged) +} + func TestCollectModelCodeTooManyFiles(t *testing.T) { files := map[string]string{"agent.py": "x\n"} for i := range maxCodeFiles + 1 { diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index b3a7ed0bd89..109b130d028 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -3,6 +3,7 @@ package aircmd import ( "bytes" "context" + "encoding/json" "errors" "fmt" "maps" @@ -286,9 +287,25 @@ func (c *deployConfig) validateCheckpoint() error { if m.Entry == nil && (m.ModelConfig != nil || m.Streamable != nil) { return errors.New("model.model_config and model.streamable apply to model.entry, not to command") } + if m.ModelConfig != nil { + if _, err := modelConfigJSON(m.ModelConfig); err != nil { + return err + } + } return nil } +// modelConfigJSON encodes model_config as the JSON object the server reads. +// YAML can hold values JSON can't, such as .nan, .inf and mappings whose keys +// aren't strings, and those are rejected rather than dropped. +func modelConfigJSON(config map[string]any) (string, error) { + raw, err := json.Marshal(config) + if err != nil { + return "", fmt.Errorf("model.model_config must hold only values JSON can: strings, finite numbers, booleans, null, lists and mappings with string keys: %w", err) + } + return string(raw), nil +} + // validateVolumeDirectory checks a Unity Catalog volume directory the server // copies into the version. func validateVolumeDirectory(field, v string) error { diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index 6525e8801de..41af37c6ac6 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -59,7 +59,8 @@ func TestLoadDeployConfigCheckpoint(t *testing.T) { assert.Equal(t, modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}}, cfg.servingEnvironment()) assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, cfg.servedEnvironmentVars()) - req := cfg.registrationRequest("key-1", nil) + req, err := cfg.registrationRequest("key-1", nil) + require.NoError(t, err) assert.Equal(t, registerModelPackageRequest{ RegisteredModel: "main.models.qwen", IdempotencyKey: "key-1", @@ -77,7 +78,9 @@ func TestLoadDeployConfigAudioCheckpoint(t *testing.T) { t.Run(task, func(t *testing.T) { cfg, err := loadDeployConfigText(t, strings.Replace(checkpointDeployConfig, "task: llm/v1/chat", "task: "+task, 1)) require.NoError(t, err) - assert.Equal(t, task, cfg.registrationRequest("key-1", nil).Checkpoint.Task) + req, err := cfg.registrationRequest("key-1", nil) + require.NoError(t, err) + assert.Equal(t, task, req.Checkpoint.Task) }) } } @@ -109,7 +112,8 @@ func TestLoadDeployConfigPythonCheckpoint(t *testing.T) { streamable := false code := &modelPackagePythonModel{EntryFile: "model.py", EntrySource: "import mlflow\n", Files: []modelPackageFile{{Path: "util/a.py", Content: "x = 1\n"}}} - req := cfg.registrationRequest("key-1", code) + req, err := cfg.registrationRequest("key-1", code) + require.NoError(t, err) assert.Equal(t, registerModelPackageRequest{ RegisteredModel: "main.ml.whisper", IdempotencyKey: "key-1", @@ -133,7 +137,8 @@ func TestLoadDeployConfigPythonCheckpointWithoutTask(t *testing.T) { text = strings.Replace(text, " model_config: {language: en, beams: 4}\n streamable: false\n", "", 1) cfg, err := loadDeployConfigText(t, text) require.NoError(t, err) - req := cfg.registrationRequest("k", &modelPackagePythonModel{EntryFile: "model.py"}) + req, err := cfg.registrationRequest("k", &modelPackagePythonModel{EntryFile: "model.py"}) + require.NoError(t, err) assert.Empty(t, req.Checkpoint.Task) assert.Nil(t, req.Checkpoint.PythonModel.Streamable) assert.Empty(t, req.Checkpoint.PythonModel.ModelConfig) @@ -179,7 +184,23 @@ compute: {workload_type: GPU_SMALL, workload_size: Small} func TestLoadDeployConfigExperimentPath(t *testing.T) { cfg, err := loadDeployConfigText(t, checkpointDeployConfig+"mlflow_experiment_path: /Shared/qwen\n") require.NoError(t, err) - assert.Equal(t, "/Shared/qwen", cfg.registrationRequest("k", nil).ExperimentPath) + req, err := cfg.registrationRequest("k", nil) + require.NoError(t, err) + assert.Equal(t, "/Shared/qwen", req.ExperimentPath) +} + +func TestLoadDeployConfigRejectsModelConfigJSONCantHold(t *testing.T) { + for name, value := range map[string]string{ + "NaN": "{ratio: .nan}", + "infinity": "{limit: .inf}", + "non-string map keys": "{weights: {1: a}}", + } { + t.Run(name, func(t *testing.T) { + text := strings.Replace(pythonCheckpointDeployConfig, "{language: en, beams: 4}", value, 1) + _, err := loadDeployConfigText(t, text) + assert.ErrorContains(t, err, "model.model_config must hold only values JSON can") + }) + } } func TestLoadDeployConfigOverride(t *testing.T) { diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go index 92f94648113..b1d4207c64c 100644 --- a/cmd/air/deployregister.go +++ b/cmd/air/deployregister.go @@ -2,7 +2,6 @@ package aircmd import ( "context" - "encoding/json" "errors" "fmt" "net/http" @@ -166,7 +165,7 @@ func (e *finalizeError) Unwrap() error { return e.err } // registrationRequest builds the request that registers cfg's checkpoint under key. // code is the collected code of model.entry, and is nil when a command serves the // checkpoint. -func (c *deployConfig) registrationRequest(key string, code *modelPackagePythonModel) registerModelPackageRequest { +func (c *deployConfig) registrationRequest(key string, code *modelPackagePythonModel) (registerModelPackageRequest, error) { req := registerModelPackageRequest{ RegisteredModel: c.RegisteredModel, IdempotencyKey: key, @@ -188,13 +187,15 @@ func (c *deployConfig) registrationRequest(key string, code *modelPackagePythonM Streamable: c.Model.Streamable, } if c.Model.ModelConfig != nil { - // The YAML decoded as JSON-compatible values; the server reads a JSON object. - raw, _ := json.Marshal(c.Model.ModelConfig) - checkpoint.PythonModel.ModelConfig = string(raw) + raw, err := modelConfigJSON(c.Model.ModelConfig) + if err != nil { + return registerModelPackageRequest{}, err + } + checkpoint.PythonModel.ModelConfig = raw } } req.Checkpoint = checkpoint - return req + return req, nil } // registerAndFinalize registers req's version and makes it ready to serve, @@ -220,6 +221,9 @@ func registerAndFinalize(ctx context.Context, w *databricks.WorkspaceClient, req if resp.ModelVersion == "" { return nil, withCode(codeInternalError, true, fmt.Errorf("registering a version of %s returned no version", req.RegisteredModel)) } + if !modelVersionRe().MatchString(resp.ModelVersion) { + return nil, withCode(codeInternalError, true, fmt.Errorf("registering a version of %s returned %q, which isn't a version number", req.RegisteredModel, stripControl(resp.ModelVersion))) + } if err := checkStagingStatus(resp.StagingStatus, req.RegisteredModel, resp.ModelVersion); err != nil { return nil, err } diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go index 6c8948a0d19..0e9840a9fc1 100644 --- a/cmd/air/deployregister_test.go +++ b/cmd/air/deployregister_test.go @@ -347,6 +347,14 @@ func TestRegisterAndFinalizeRejectsARegistrationWithoutAVersion(t *testing.T) { assert.Empty(t, fake.finalizeRequests) } +func TestRegisterAndFinalizeRejectsAVersionThatIsntANumber(t *testing.T) { + fake := newFakeModelServing(t, `{"model_id": "m-1", "registered_model": "main.models.qwen", "model_version": "3; rm", "ready": false, "staging_status": {"model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}`) + + _, _, err := registerTestCheckpoint(t, fake, checkpointRequest()) + assert.ErrorContains(t, err, `registering a version of main.models.qwen returned "3; rm", which isn't a version number`) + assert.Empty(t, fake.finalizeRequests) +} + func TestRegisterAndFinalizeRetriesTransientErrors(t *testing.T) { fake := newFakeModelServing(t, serverErrorResponse, stagingInProgress) fake.statuses = []string{serverErrorResponse, statusSucceeded} From 7db8aecce4f94c1509c348213983b4e885459e1b Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 17 Nov 2026 02:48:52 +0000 Subject: [PATCH 09/11] air deploy: say which model_config values are refused: .nan and .inf Co-authored-by: Isaac --- cmd/air/deployconfig.go | 6 +++--- cmd/air/deployconfig_test.go | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index 109b130d028..8744ced25d7 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -296,12 +296,12 @@ func (c *deployConfig) validateCheckpoint() error { } // modelConfigJSON encodes model_config as the JSON object the server reads. -// YAML can hold values JSON can't, such as .nan, .inf and mappings whose keys -// aren't strings, and those are rejected rather than dropped. +// YAML can hold numbers JSON can't, .nan and .inf, and those are rejected rather +// than dropped. func modelConfigJSON(config map[string]any) (string, error) { raw, err := json.Marshal(config) if err != nil { - return "", fmt.Errorf("model.model_config must hold only values JSON can: strings, finite numbers, booleans, null, lists and mappings with string keys: %w", err) + return "", fmt.Errorf("model.model_config must hold only values JSON can, so no .nan or .inf: %w", err) } return string(raw), nil } diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index 41af37c6ac6..cff1af8a0e5 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -191,14 +191,14 @@ func TestLoadDeployConfigExperimentPath(t *testing.T) { func TestLoadDeployConfigRejectsModelConfigJSONCantHold(t *testing.T) { for name, value := range map[string]string{ - "NaN": "{ratio: .nan}", - "infinity": "{limit: .inf}", - "non-string map keys": "{weights: {1: a}}", + "NaN": "{ratio: .nan}", + "infinity": "{limit: .inf}", + "negative infinity": "{weights: [1, -.inf]}", } { t.Run(name, func(t *testing.T) { text := strings.Replace(pythonCheckpointDeployConfig, "{language: en, beams: 4}", value, 1) _, err := loadDeployConfigText(t, text) - assert.ErrorContains(t, err, "model.model_config must hold only values JSON can") + assert.ErrorContains(t, err, "model.model_config must hold only values JSON can, so no .nan or .inf") }) } } From 5ff35668317a77b910a7934f956a8ba20fdefe86 Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Tue, 17 Nov 2026 19:22:52 +0000 Subject: [PATCH 10/11] air deploy: name compute.scale_to_zero_enabled as the serving endpoint API does The compute block's fields now all carry the served entity's API names: workload_type, workload_size and scale_to_zero_enabled. Co-authored-by: Isaac --- cmd/air/deploy_test.go | 2 +- cmd/air/deployconfig.go | 6 +++--- cmd/air/deployconfig_test.go | 4 ++-- cmd/air/deployendpoint.go | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go index 805bf00a995..57276665480 100644 --- a/cmd/air/deploy_test.go +++ b/cmd/air/deploy_test.go @@ -99,7 +99,7 @@ func TestDeployCheckpointJSON(t *testing.T) { served := endpoints.endpoint.Config.ServedEntities[0] assert.Equal(t, "3", served.EntityVersion) assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, served.EnvironmentVars) - // scale_to_zero: false is sent, not left to the server's default. + // scale_to_zero_enabled: false is sent, not left to the server's default. require.Len(t, endpoints.rawWrites, 1) assert.Contains(t, endpoints.rawWrites[0], `"scale_to_zero_enabled":false`) } diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index 8744ced25d7..06e6a32064e 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -122,9 +122,9 @@ type deployCodeSourceConfig struct { // deployComputeConfig is the `compute` block: the served entity's workload. type deployComputeConfig struct { - WorkloadType string `yaml:"workload_type" help:"Serving compute, e.g. CPU, GPU_SMALL, GPU_MEDIUM or GPU_LARGE. A checkpoint is served on a GPU workload type." required:"yes"` - WorkloadSize string `yaml:"workload_size" help:"How far the endpoint scales: Small, Medium or Large." required:"yes"` - ScaleToZero bool `yaml:"scale_to_zero" help:"Whether the endpoint scales to zero when idle (default: false)."` + WorkloadType string `yaml:"workload_type" help:"Serving compute, e.g. CPU, GPU_SMALL, GPU_MEDIUM or GPU_LARGE. A checkpoint is served on a GPU workload type." required:"yes"` + WorkloadSize string `yaml:"workload_size" help:"How far the endpoint scales: Small, Medium or Large." required:"yes"` + ScaleToZeroEnabled bool `yaml:"scale_to_zero_enabled" help:"Whether the endpoint scales to zero when idle (default: false)."` } // sourceFields lists, per model.type, the optional fields that apply to it, in diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index cff1af8a0e5..8ea4542f99b 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -43,7 +43,7 @@ endpoint: support-agent compute: workload_type: GPU_SMALL workload_size: Small - scale_to_zero: true + scale_to_zero_enabled: true ` func loadDeployConfigText(t *testing.T, text string, overrides ...string) (*deployConfig, error) { @@ -212,7 +212,7 @@ func TestLoadDeployConfigOverride(t *testing.T) { func TestLoadDeployConfigRejectsUnknownOverride(t *testing.T) { _, err := loadDeployConfigText(t, checkpointDeployConfig, "compute.accelerator_type=GPU_1xH100") - assert.ErrorContains(t, err, `invalid --override "compute.accelerator_type": "accelerator_type" is not a known field; available fields are: scale_to_zero, workload_size, workload_type`) + assert.ErrorContains(t, err, `invalid --override "compute.accelerator_type": "accelerator_type" is not a known field; available fields are: scale_to_zero_enabled, workload_size, workload_type`) } func TestLoadDeployConfigRejectsUnknownField(t *testing.T) { diff --git a/cmd/air/deployendpoint.go b/cmd/air/deployendpoint.go index 1effc2a8303..a0481d40a57 100644 --- a/cmd/air/deployendpoint.go +++ b/cmd/air/deployendpoint.go @@ -22,9 +22,9 @@ func (c *deployConfig) servedEntity(version string) serving.ServedEntityInput { EntityVersion: version, WorkloadType: serving.ServingModelWorkloadType(c.Compute.WorkloadType), WorkloadSize: c.Compute.WorkloadSize, - ScaleToZeroEnabled: c.Compute.ScaleToZero, + ScaleToZeroEnabled: c.Compute.ScaleToZeroEnabled, EnvironmentVars: c.servedEnvironmentVars(), - // scale_to_zero: false is the user's choice, so send it rather than leave it + // scale_to_zero_enabled: false is the user's choice, so send it rather than leave it // to the server's default. ForceSendFields: []string{"ScaleToZeroEnabled"}, } From c830022ecae18360c7176aa2a68b977411d420fc Mon Sep 17 00:00:00 2001 From: Arpit Agarwal Date: Wed, 18 Nov 2026 20:10:30 +0000 Subject: [PATCH 11/11] air deploy: flat config for command, dedicated and registered deployments Replace the nested model: block and the Python-code path with a flat YAML whose fields select one of three variants: - checkpoint and command: a checkpoint served by the command the file sets - checkpoint without command: a checkpoint served with dedicated inference - neither: a model version already registered in Unity Catalog Compute is serving compute (accelerator_type, num_accelerators and workload_size, mapped to a serving workload type) for a command or a registered model, and dedicated compute (compute_profile, provisioned_model_units and reservation_term) for a dedicated checkpoint. A field the file's variant doesn't use is an error that names the field, the variant and why the file is read as that variant. Endpoints are read as JSON, since the SDK's served entity has no compute_profile, and dedicated endpoints are created and updated over raw JSON. A dry run prints the plan for each variant without making requests. Co-authored-by: Isaac --- cmd/air/deploy.go | 188 +++++++---- cmd/air/deploy_test.go | 252 ++++++++++----- cmd/air/deploycode.go | 229 -------------- cmd/air/deploycode_test.go | 216 ------------- cmd/air/deployconfig.go | 541 ++++++++++++++++--------------- cmd/air/deployconfig_test.go | 559 ++++++++++++++++++++------------- cmd/air/deployendpoint.go | 358 ++++++++++++++++----- cmd/air/deployendpoint_test.go | 282 ++++++++++++++--- cmd/air/deployregister.go | 97 +++--- cmd/air/deployregister_test.go | 39 ++- 10 files changed, 1508 insertions(+), 1253 deletions(-) delete mode 100644 cmd/air/deploycode.go delete mode 100644 cmd/air/deploycode_test.go diff --git a/cmd/air/deploy.go b/cmd/air/deploy.go index fb6d5730609..3d41ba6e990 100644 --- a/cmd/air/deploy.go +++ b/cmd/air/deploy.go @@ -2,11 +2,13 @@ package aircmd import ( "context" + "encoding/json" "errors" "fmt" "net/http" "path/filepath" "regexp" + "strconv" "strings" "sync" "time" @@ -19,7 +21,7 @@ import ( "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/retries" - "github.com/databricks/databricks-sdk-go/service/serving" + "github.com/databricks/databricks-sdk-go/service/catalog" "github.com/spf13/cobra" ) @@ -66,17 +68,29 @@ func withCode(code string, retryable bool, err error) error { return &deployError{code: code, retryable: retryable, err: err} } +// deployPlan is the deployment a dry run reports. +type deployPlan struct { + // Variant is the variant the file is read as: command, dedicated or registered. + Variant string `json:"variant"` + // Checkpoint is the checkpoint a new version is registered from. It is empty when + // the version is already registered. + Checkpoint string `json:"checkpoint,omitempty"` + // ModelVersion is the version served, empty when a new version is registered, or + // when the newest READY version is served, picked when deploying. + ModelVersion string `json:"model_version,omitempty"` + // ServedEntity is the entity the endpoint serves, as the endpoints API takes it. + ServedEntity any `json:"served_entity"` +} + // deployResult is the JSON payload for `air deploy`. type deployResult struct { - Status string `json:"status"` - DryRun bool `json:"dry_run,omitempty"` - // CodeFiles lists, on a dry run of a checkpoint served by model.entry, the - // files it would send. - CodeFiles []string `json:"code_files,omitempty"` - RegisteredModel string `json:"registered_model,omitempty"` - ModelVersion string `json:"model_version,omitempty"` - ModelID string `json:"model_id,omitempty"` - IdempotencyKey string `json:"idempotency_key,omitempty"` + Status string `json:"status"` + DryRun bool `json:"dry_run,omitempty"` + Plan *deployPlan `json:"plan,omitempty"` + RegisteredModel string `json:"registered_model,omitempty"` + ModelVersion string `json:"model_version,omitempty"` + ModelID string `json:"model_id,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` // EnvironmentStagingFailed is set when the version's serving environment // couldn't be prepared ahead of time, so the endpoint can be slower to start. EnvironmentStagingFailed bool `json:"environment_staging_failed,omitempty"` @@ -110,23 +124,28 @@ func newDeployCommandWithPollInterval(pollInterval time.Duration) *cobra.Command Short: "Register a model in Unity Catalog and serve it on an endpoint", Long: `Register a model in Unity Catalog and serve it on a Model Serving endpoint. -The deployment is described by a YAML file (see --file). The model is a -checkpoint (weights in a Unity Catalog volume, served by a command or by a -Python file that builds an MLflow model) or an existing model version. -air deploy registers a new version, waits until it is ready, then creates or -updates the endpoint to serve it and waits until the endpoint is ready. +The deployment is described by a YAML file (see --file). Its fields say what is +served, and which variant the file is read as: + checkpoint and command a checkpoint in a volume, served by a command you give + checkpoint a checkpoint served with dedicated inference + neither a model version already registered in Unity Catalog + +For a checkpoint, air deploy registers a new version, waits until it is ready, +then creates or updates the endpoint to serve it and waits until the endpoint is +ready. For a registered version, it only creates or updates the endpoint. Registration is idempotent: run air deploy again with the idempotency key it prints to continue an interrupted deployment. To look up a YAML field, pass -its path to -h as a separate argument: +its path to -h as a separate argument, and see one YAML example per variant +with -h config: - databricks air deploy -h config all fields - databricks air deploy -h config.model one section - databricks air deploy -h config.model.type one field`, + databricks air deploy -h config all fields, with examples + databricks air deploy -h config.compute one section + databricks air deploy -h config.checkpoint one field`, Example: ` # Register a checkpoint and serve it databricks air deploy -f deploy.yaml - # Validate the config, and list the code it would send, without deploying + # Validate the config, and show the plan, without deploying databricks air deploy -f deploy.yaml --dry-run # Continue a deployment that was interrupted @@ -159,8 +178,8 @@ its path to -h as a separate argument: // List flags in declaration order (most used first) rather than alphabetically. cmd.Flags().SortFlags = false cmd.Flags().StringVarP(&opts.file, "file", "f", "", "Path to the deployment YAML config") - cmd.Flags().StringArrayVar(&opts.overrides, "override", nil, "Override a YAML field, e.g. compute.workload_size=Medium (repeatable)") - cmd.Flags().BoolVar(&opts.dryRun, "dry-run", false, "Validate the config, and list the code it would send, without deploying") + cmd.Flags().StringArrayVar(&opts.overrides, "override", nil, "Override a YAML field, e.g. compute.num_accelerators=4 (repeatable)") + cmd.Flags().BoolVar(&opts.dryRun, "dry-run", false, "Validate the config, and show the plan, without deploying") cmd.Flags().StringVar(&opts.idempotencyKey, "idempotency-key", "", "Continue the registration that used this key, instead of registering another version") cmd.Flags().BoolVar(&opts.retryFailedStaging, "retry-failed-staging", false, "Copy a checkpoint's weights again after copying them failed") cmd.Flags().DurationVar(&opts.timeout, "timeout", defaultDeployTimeout, "How long to wait for the version and the endpoint to be ready") @@ -201,17 +220,9 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio return withCode(codeInvalidArgs, false, err) } - var code *modelPackagePythonModel - if cfg.Model.Entry != nil { - code, err = collectEntryCode(ctx, cfg, opts.file) - if err != nil { - return withCode(codeInvalidArgs, false, err) - } - } - jsonOut := root.OutputType(cmd) == flags.OutputJSON if opts.dryRun { - return renderDeployDryRun(ctx, cfg, code, jsonOut) + return renderDeployDryRun(ctx, cfg, jsonOut) } w := cmdctx.WorkspaceClient(ctx) @@ -223,14 +234,14 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio result := deployResult{RegisteredModel: cfg.RegisteredModel, Endpoint: cfg.Endpoint} if cfg.registers() { // Fail on an endpoint air deploy won't change before copying any weights. - if err := checkEndpoint(ctx, w, cfg.Endpoint, cfg.RegisteredModel); err != nil { + if err := checkEndpoint(ctx, w, cfg.Endpoint, cfg.RegisteredModel, cfg.variant() == variantDedicated); err != nil { if stopped := deployStopped(ctx, err); stopped != nil { return fmt.Errorf("%w while checking endpoint %s, before registering anything", stopped, cfg.Endpoint) } return err } result.IdempotencyKey = effectiveIdempotencyToken(opts.idempotencyKey, cfg.IdempotencyToken) - resp, err := registerDeployment(ctx, w, cfg, opts, code, result.IdempotencyKey, pollInterval, !jsonOut) + resp, err := registerDeployment(ctx, w, cfg, opts, result.IdempotencyKey, pollInterval, !jsonOut) if err != nil { return err } @@ -243,10 +254,20 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio } } } else { - result.ModelVersion = *cfg.Model.Version + version, err := resolveModelVersion(ctx, w, cfg) + if err != nil { + return err + } + result.ModelVersion = version + if !jsonOut && cfg.ModelVersion == nil { + cmdio.LogString(ctx, fmt.Sprintf("Serving version %s, the newest READY version of %s.", version, cfg.RegisteredModel)) + } } - entity := cfg.servedEntity(result.ModelVersion) + entity, err := cfg.servedEntity(result.ModelVersion) + if err != nil { + return withCode(codeInvalidArgs, false, err) + } _, err = serveVersion(ctx, w, cfg.Endpoint, entity, !jsonOut) if err != nil { resume := deployResumeCommand(w.Config.Profile, opts, result.IdempotencyKey) @@ -269,36 +290,85 @@ func runDeploy(cmd *cobra.Command, opts deployOptions, pollInterval time.Duratio return nil } -// renderDeployDryRun reports a valid config and, for a checkpoint served by -// model.entry, the files a deployment would send. -func renderDeployDryRun(ctx context.Context, cfg *deployConfig, code *modelPackagePythonModel, jsonOut bool) error { - var files []string - if code != nil { - files = codeFilePaths(code) +// plan is the deployment a dry run reports. A dry run makes no request, so the version +// a new registration produces isn't known, and a registered model without +// model_version serves its newest READY version, picked when deploying. +func (c *deployConfig) plan() (deployPlan, error) { + p := deployPlan{Variant: c.variant()} + if c.registers() { + p.Checkpoint = *c.Checkpoint + } + if c.ModelVersion != nil { + p.ModelVersion = *c.ModelVersion + } + entity, err := c.servedEntity(p.ModelVersion) + if err != nil { + return deployPlan{}, err + } + if entity.Dedicated != nil { + p.ServedEntity = entity.dedicatedJSON() + } else { + p.ServedEntity = entity.servingInput() + } + return p, nil +} + +// renderDeployDryRun reports the plan of a valid config, without deploying. +func renderDeployDryRun(ctx context.Context, cfg *deployConfig, jsonOut bool) error { + plan, err := cfg.plan() + if err != nil { + return withCode(codeInvalidArgs, false, err) } if jsonOut { - return renderEnvelope(ctx, deployResult{Status: "DRY_RUN_OK", DryRun: true, CodeFiles: files}) + return renderEnvelope(ctx, deployResult{Status: "DRY_RUN_OK", DryRun: true, Plan: &plan}) } - if code != nil { - cmdio.LogString(ctx, fmt.Sprintf("Dry run: model.entry would send %d file(s):", len(files))) - for _, f := range files { - cmdio.LogString(ctx, " "+f) - } + cmdio.LogString(ctx, fmt.Sprintf("Dry run: %s.", cfg.readAs())) + switch { + case plan.Checkpoint != "": + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would register a new version of %s from %s, and serve it.", cfg.RegisteredModel, plan.Checkpoint)) + case plan.ModelVersion != "": + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would serve version %s of %s.", plan.ModelVersion, cfg.RegisteredModel)) + default: + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would serve the newest READY version of %s, picked when deploying.", cfg.RegisteredModel)) + } + served, err := json.MarshalIndent(plan.ServedEntity, " ", " ") + if err != nil { + return withCode(codeInternalError, false, err) } + cmdio.LogString(ctx, fmt.Sprintf("Dry run: would serve it on endpoint %s as:\n %s", cfg.Endpoint, served)) cmdio.LogString(ctx, fmt.Sprintf("Dry run: configuration for endpoint %q is valid; not deploying.", cfg.Endpoint)) return nil } +// resolveModelVersion returns the version of a registered model to serve: model_version +// when the config sets it, else the highest-numbered READY version of the model. +func resolveModelVersion(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig) (string, error) { + if cfg.ModelVersion != nil { + return *cfg.ModelVersion, nil + } + versions, err := w.ModelVersions.ListAll(ctx, catalog.ListModelVersionsRequest{FullName: cfg.RegisteredModel}) + if err != nil { + return "", fmt.Errorf("listing the versions of %s: %w", cfg.RegisteredModel, err) + } + newest := 0 + for _, v := range versions { + if v.Status == catalog.ModelVersionInfoStatusReady && v.Version > newest { + newest = v.Version + } + } + if newest == 0 { + return "", withCode(codeInvalidArgs, false, fmt.Errorf("%s has no READY version to serve; register and finalize a version, or set model_version", cfg.RegisteredModel)) + } + return strconv.Itoa(newest), nil +} + // registerDeployment registers the version and makes it ready to serve, showing // progress on a spinner when show is true. Its errors say how to continue. -func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, code *modelPackagePythonModel, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { +func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg *deployConfig, opts deployOptions, key string, pollInterval time.Duration, show bool) (*registerModelPackageResponse, error) { // Printed to stderr in every output mode, before the first request, so a run // that is killed can still be continued with this key. cmdio.LogString(ctx, fmt.Sprintf("Registering a version of %s with idempotency key %s", cfg.RegisteredModel, key)) - req, err := cfg.registrationRequest(key, code) - if err != nil { - return nil, withCode(codeInvalidArgs, false, err) - } + req := cfg.registrationRequest(key) req.RetryFailedStaging = opts.retryFailedStaging progress := func(*registerModelPackageResponse) {} @@ -315,7 +385,7 @@ func registerDeployment(ctx context.Context, w *databricks.WorkspaceClient, cfg } resume := deployResumeCommand(w.Config.Profile, opts, key) if errors.Is(err, errStagingFailed) { - return nil, withCode(codeStagingFailed, true, fmt.Errorf("copying the weights in %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", *cfg.Model.Path, stripControl(resp.ModelVersion), cfg.RegisteredModel, resume)) + return nil, withCode(codeStagingFailed, true, fmt.Errorf("copying the weights in %s into version %s of %s failed. Check that you can read every file, then run:\n %s --retry-failed-staging", *cfg.Checkpoint, stripControl(resp.ModelVersion), cfg.RegisteredModel, resume)) } if stopped := deployStopped(ctx, err); stopped != nil { continuing := "" @@ -380,21 +450,21 @@ func deployErrorCode(err error) (code string, retryable bool) { return codeInternalError, true } -// validateDeployFlags rejects flags the config's model source can't use. +// validateDeployFlags rejects flags the config's variant can't use. func validateDeployFlags(cfg *deployConfig, opts deployOptions) error { if opts.timeout <= 0 { return fmt.Errorf("--timeout must be positive, got %s", opts.timeout) } if opts.idempotencyKeySet { if !cfg.registers() { - return errors.New("--idempotency-key applies only when air deploy registers a version; model.type is registered") + return fmt.Errorf("--idempotency-key applies only when air deploy registers a version; %s", cfg.readAs()) } if err := validateDeployIdempotencyKey(opts.idempotencyKey, "--idempotency-key"); err != nil { return err } } - if opts.retryFailedStaging && cfg.Model.Type != modelTypeCheckpoint { - return fmt.Errorf("--retry-failed-staging applies only to a checkpoint; model.type is %s", cfg.Model.Type) + if opts.retryFailedStaging && !cfg.registers() { + return fmt.Errorf("--retry-failed-staging applies only to a checkpoint; %s", cfg.readAs()) } return nil } @@ -413,14 +483,14 @@ func registrationProgress(cfg *deployConfig, resp *registerModelPackageResponse) // serveVersion serves entity on the endpoint, showing the endpoint's state on a // spinner when show is true. -func serveVersion(ctx context.Context, w *databricks.WorkspaceClient, name string, entity serving.ServedEntityInput, show bool) (*serving.ServingEndpointDetailed, error) { +func serveVersion(ctx context.Context, w *databricks.WorkspaceClient, name string, entity servedEntity, show bool) (*servingEndpoint, error) { if !show { return serveOnEndpoint(ctx, w, name, entity, nil) } cmdio.LogString(ctx, fmt.Sprintf("Serving version %s of %s on endpoint %s", entity.EntityVersion, entity.EntityName, name)) sp := cmdio.NewSpinner(ctx) defer sp.Close() - return serveOnEndpoint(ctx, w, name, entity, func(endpoint *serving.ServingEndpointDetailed) { + return serveOnEndpoint(ctx, w, name, entity, func(endpoint *servingEndpoint) { if endpoint.State != nil { sp.Update(fmt.Sprintf("Waiting for endpoint %s (%s, %s)…", name, endpoint.State.Ready, endpoint.State.ConfigUpdate)) } diff --git a/cmd/air/deploy_test.go b/cmd/air/deploy_test.go index 57276665480..3c2af38a24a 100644 --- a/cmd/air/deploy_test.go +++ b/cmd/air/deploy_test.go @@ -7,9 +7,9 @@ import ( "errors" "fmt" "io" + "net/http" "net/http/httptest" "path/filepath" - "strconv" "strings" "testing" "time" @@ -27,6 +27,18 @@ import ( "github.com/stretchr/testify/require" ) +// registeredNoVersionConfig serves the newest READY version of a registered model. +const registeredNoVersionConfig = `registered_model_name: main.models.qwen +endpoint_name: qwen-chat +` + +// modelVersionsJSON lists the versions of the test model: 4 is the newest READY one. +const modelVersionsJSON = `{"model_versions": [ + {"version": 2, "status": "READY"}, + {"version": 5, "status": "PENDING_REGISTRATION"}, + {"version": 4, "status": "READY"} +]}` + // withDeadline returns the test's context with a deadline, which air deploy // always sets from --timeout. func withDeadline(t *testing.T) context.Context { @@ -45,6 +57,21 @@ func deployServer(t *testing.T, endpoints *fakeEndpoints, responses ...string) ( return fake.start(), fake } +// deployServerWithVersions is deployServer, also serving the model's versions, as +// the Unity Catalog API lists them, from versionsJSON. +func deployServerWithVersions(t *testing.T, endpoints *fakeEndpoints, versionsJSON string) (*httptest.Server, *fakeModelServing) { + t.Helper() + fake := newFakeModelServing(t) + fake.endpoints = func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/versions") { + _, _ = w.Write([]byte(versionsJSON)) + return + } + endpoints.ServeHTTP(w, r) + } + return fake.start(), fake +} + // runDeployCmd runs `air deploy` on config with the given flags. An empty srvURL // runs it without a workspace client, as --dry-run does. func runDeployCmd(t *testing.T, out flags.Output, buf *bytes.Buffer, srvURL, config string, flagValues ...string) error { @@ -69,12 +96,12 @@ func runDeployCmd(t *testing.T, out flags.Output, buf *bytes.Buffer, srvURL, con return cmd.RunE(cmd, nil) } -func TestDeployCheckpointJSON(t *testing.T) { +func TestDeployCommandJSON(t *testing.T) { endpoints := newFakeEndpoints(t, nil) srv, fake := deployServer(t, endpoints, stagingInProgress) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") require.NoError(t, err) var got struct { @@ -94,21 +121,23 @@ func TestDeployCheckpointJSON(t *testing.T) { bodies := fake.requests() require.Len(t, bodies, 1) assert.Equal(t, "key-1", bodies[0].IdempotencyKey) + assert.Equal(t, "/Volumes/main/models/weights/qwen3_8b", bodies[0].Checkpoint.VolumePath) assert.Equal(t, []finalizeModelPackageRequest{{RegisteredModel: "main.models.qwen", ModelVersion: "3"}}, fake.finalizeRequests) assert.Equal(t, []string{"create"}, endpoints.writes) served := endpoints.endpoint.Config.ServedEntities[0] assert.Equal(t, "3", served.EntityVersion) + assert.Equal(t, "GPU_MEDIUM", served.WorkloadType) assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, served.EnvironmentVars) // scale_to_zero_enabled: false is sent, not left to the server's default. require.Len(t, endpoints.rawWrites, 1) assert.Contains(t, endpoints.rawWrites[0], `"scale_to_zero_enabled":false`) } -func TestDeployCheckpointText(t *testing.T) { +func TestDeployCommandText(t *testing.T) { srv, _ := deployServer(t, newFakeEndpoints(t, nil), stagingReady) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig)) + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig)) out := buf.String() assert.Contains(t, out, "Registering a version of main.models.qwen with idempotency key ") assert.Contains(t, out, "Version 3 of main.models.qwen is ready.") @@ -118,12 +147,30 @@ func TestDeployCheckpointText(t *testing.T) { assert.NotContains(t, out, "Warning") } +func TestDeployDedicatedRegistersAndCreatesTheDedicatedEndpoint(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServer(t, endpoints, stagingReady) + + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, dedicatedDeployConfig, "idempotency-key", "key-1")) + + bodies := fake.requests() + require.Len(t, bodies, 1) + assert.Nil(t, bodies[0].Checkpoint) + assert.Equal(t, &dedicatedCheckpoint{VolumePath: "/Volumes/main/models/weights/qwen3_8b"}, bodies[0].DedicatedCheckpoint) + assert.Equal(t, []string{"create dedicated"}, endpoints.writes) + require.Len(t, endpoints.rawWrites, 1) + assert.Contains(t, endpoints.rawWrites[0], `"compute_profile":"COMPUTE_PROFILE_LOW_LATENCY"`) + assert.NotContains(t, endpoints.rawWrites[0], "scale_to_zero_enabled") + assert.Equal(t, "3", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) +} + func TestDeployWarnsWhenTheEnvironmentFailedToStage(t *testing.T) { environmentFailed := strings.Replace(stagingReady, `"environment_artifact": {"state": "SUCCEEDED"}`, `"environment_artifact": {"state": "FAILED"}`, 1) srv, _ := deployServer(t, newFakeEndpoints(t, nil), environmentFailed) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig)) + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig)) assert.Contains(t, buf.String(), "Warning: the version's serving environment couldn't be prepared ahead of time") } @@ -131,9 +178,9 @@ func TestDeployUsesTheConfigTokenThenANewKey(t *testing.T) { srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingReady) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig+"idempotency_token: from-config\n")) - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig+"idempotency_token: from-config\n", "idempotency-key", "from-flag")) - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig)) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig+"idempotency_token: from-config\n")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig+"idempotency_token: from-config\n", "idempotency-key", "from-flag")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig)) bodies := fake.requests() require.Len(t, bodies, 3) @@ -148,7 +195,7 @@ func TestDeployPrintsTheKeyBeforeRegisteringInJSONMode(t *testing.T) { var stdout, stderr bytes.Buffer cmd := withOutput(newDeployCommandWithPollInterval(time.Millisecond), flags.OutputJSON) - require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", checkpointDeployConfig))) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", commandDeployConfig))) require.NoError(t, cmd.Flags().Set("idempotency-key", "key-1")) ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &stdout, &stderr, "", "")) cmd.SetContext(cmdctx.SetWorkspaceClient(ctx, newTestWorkspaceClient(t, srv.URL))) @@ -161,7 +208,7 @@ func TestDeploySendsRetryFailedStaging(t *testing.T) { srv, fake := deployServer(t, newFakeEndpoints(t, nil), stagingInProgress) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "retry-failed-staging", "true")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig, "retry-failed-staging", "true")) bodies := fake.requests() require.Len(t, bodies, 1) assert.True(t, bodies[0].RetryFailedStaging) @@ -171,7 +218,7 @@ func TestDeployFailedStagingSaysHowToRetry(t *testing.T) { srv, _ := deployServer(t, newFakeEndpoints(t, nil), stagingFailed) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") require.Error(t, err) assert.Contains(t, err.Error(), "copying the weights in /Volumes/main/models/weights/qwen3_8b into version 3 of main.models.qwen failed") assert.Contains(t, err.Error(), " --idempotency-key key-1 --retry-failed-staging") @@ -181,7 +228,7 @@ func TestDeployReportsRegistrationErrors(t *testing.T) { srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") assert.ErrorContains(t, err, "registering a version of main.models.qwen with idempotency key key-1 failed: The idempotency key can't be used for this registration") } @@ -189,7 +236,7 @@ func TestDeployReportsErrorsAsJSON(t *testing.T) { srv, _ := deployServer(t, newFakeEndpoints(t, nil), conflictResponse) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig) + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig) require.ErrorIs(t, err, root.ErrAlreadyPrinted) var got errorEnvelope require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) @@ -198,11 +245,11 @@ func TestDeployReportsErrorsAsJSON(t *testing.T) { assert.False(t, got.Error.Retryable) buf.Reset() - err = runDeployCmd(t, flags.OutputJSON, &buf, "", "registered_model: x\n", "dry-run", "true") + err = runDeployCmd(t, flags.OutputJSON, &buf, "", "registered_model_name: x\n", "dry-run", "true") require.ErrorIs(t, err, root.ErrAlreadyPrinted) require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) assert.Equal(t, "INVALID_ARGS", got.Error.Code) - assert.Contains(t, got.Error.Message, "registered_model must be a Unity Catalog model name") + assert.Contains(t, got.Error.Message, "registered_model_name must be a Unity Catalog model name") } func TestDeployChecksTheEndpointBeforeRegistering(t *testing.T) { @@ -211,7 +258,7 @@ func TestDeployChecksTheEndpointBeforeRegistering(t *testing.T) { srv, fake := deployServer(t, newFakeEndpoints(t, endpointServing(other, readyState)), stagingReady) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig) + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig) assert.ErrorContains(t, err, "endpoint qwen-chat serves version 5 of main.models.llama") assert.Empty(t, fake.requests()) } @@ -222,7 +269,7 @@ func TestDeployReportsAnEndpointThatIsNotReady(t *testing.T) { srv, _ := deployServer(t, endpoints, stagingReady) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") require.ErrorIs(t, err, root.ErrAlreadyPrinted) var got errorEnvelope require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) @@ -240,7 +287,7 @@ func TestDeployTimesOutWhileCopyingTheWeights(t *testing.T) { // limiter, stops the wait. var buf bytes.Buffer cmd := withOutput(newDeployCommandWithPollInterval(100*time.Millisecond), flags.OutputJSON) - require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", checkpointDeployConfig))) + require.NoError(t, cmd.Flags().Set("file", writeConfigFile(t, "deploy.yaml", commandDeployConfig))) require.NoError(t, cmd.Flags().Set("idempotency-key", "key-1")) require.NoError(t, cmd.Flags().Set("timeout", "500ms")) ctx := cmdio.InContext(t.Context(), cmdio.NewIO(t.Context(), flags.OutputJSON, nil, &buf, io.Discard, "", "")) @@ -261,7 +308,7 @@ func TestDeployFinalizeFailureSaysHowToRetry(t *testing.T) { fake.finalizations = []string{notFinalizableResponse} var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") require.Error(t, err) assert.Contains(t, err.Error(), "finalizing version 3 of main.models.qwen failed: Version 3 of model main.models.qwen can't be finalized yet") assert.Contains(t, err.Error(), "To try again, run:\n databricks air deploy -f ") @@ -273,7 +320,7 @@ func TestDeployWarnsWhenTheFinalizedEnvironmentFailedToStage(t *testing.T) { fake.finalizations = []string{strings.Replace(finalizedResponse, `"environment_artifact": {"state": "IN_PROGRESS"}`, `"environment_artifact": {"state": "FAILED"}`, 1)} var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig)) + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig)) assert.Contains(t, buf.String(), "Warning: the version's serving environment couldn't be prepared ahead of time") } @@ -281,7 +328,7 @@ func TestDeployRetryableRegistrationFailureSaysHowToRetry(t *testing.T) { srv, fake := deployServer(t, newFakeEndpoints(t, nil), serverErrorResponse) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") require.Error(t, err) assert.Contains(t, err.Error(), "failed: Something went wrong.\n\nTo try again, run:\n databricks air deploy -f ") assert.Contains(t, err.Error(), " --idempotency-key key-1") @@ -294,7 +341,7 @@ func TestDeployEndpointFailureSaysHowToRetry(t *testing.T) { srv, _ := deployServer(t, endpoints, stagingReady) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, checkpointDeployConfig, "idempotency-key", "key-1") + err := runDeployCmd(t, flags.OutputText, &buf, srv.URL, commandDeployConfig, "idempotency-key", "key-1") require.Error(t, err) assert.Contains(t, err.Error(), "serving version 3 of main.models.qwen on endpoint qwen-chat failed: ") assert.Contains(t, err.Error(), "To try again without registering another version, run:\n databricks air deploy -f ") @@ -306,91 +353,129 @@ func TestDeployRegisteredVersionSkipsRegistration(t *testing.T) { srv, fake := deployServer(t, endpoints) var buf bytes.Buffer - err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, `registered_model: main.models.qwen -model: {type: registered, version: 7} -endpoint: qwen-chat -compute: {workload_type: GPU_MEDIUM, workload_size: Small} -`) - require.NoError(t, err) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, registeredDeployConfig)) assert.Empty(t, fake.requests()) assert.Equal(t, "7", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) + assert.Equal(t, "CPU", endpoints.endpoint.Config.ServedEntities[0].WorkloadType) assert.NotContains(t, buf.String(), "idempotency_key") } -func TestDeployEntrySendsThePythonModel(t *testing.T) { - cfg, _ := writeCodeTree(t, map[string]string{"agent.py": "import tools\n", "tools/__init__.py": "x = 1\n"}) - ready := `{"model_id": "m-2", "registered_model": "main.agents.support", "model_version": "1", "ready": true, - "staging_status": {"state": "SUCCEEDED", "model_artifact": {"state": "SUCCEEDED"}, "environment_artifact": {"state": "SUCCEEDED"}}}` - srv, fake := deployServer(t, newFakeEndpoints(t, nil), ready) +func TestDeployRegisteredPicksTheNewestReadyVersion(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, fake := deployServerWithVersions(t, endpoints, modelVersionsJSON) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, entryConfigAt(cfg.CodeSource.RootPath))) - bodies := fake.requests() - require.Len(t, bodies, 1) - assert.Equal(t, &modelPackageCheckpoint{ - VolumePath: "/Volumes/main/models/weights/support", - PythonModel: &modelPackagePythonModel{ - EntryFile: "agent.py", - EntrySource: "import tools\n", - Files: []modelPackageFile{{Path: "tools/__init__.py", Content: "x = 1\n"}}, - }, - Task: "agent/v1/responses", - Environment: modelPackageEnvironment{Runtime: "client.5-gpu", Dependencies: []string{"mlflow==3.5.0"}}, - }, bodies[0].Checkpoint) + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, srv.URL, registeredNoVersionConfig)) + assert.Empty(t, fake.requests()) + assert.Equal(t, "4", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) + assert.Contains(t, buf.String(), "Serving version 4, the newest READY version of main.models.qwen.") } -func TestDeployDryRunNeedsNoWorkspace(t *testing.T) { - cmd := newDeployCommand() - require.NoError(t, cmd.Flags().Set("dry-run", "true")) - assert.NoError(t, cmd.PreRunE(cmd, nil)) +func TestDeployRegisteredGivenVersionWinsOverTheNewestReady(t *testing.T) { + endpoints := newFakeEndpoints(t, nil) + srv, _ := deployServerWithVersions(t, endpoints, modelVersionsJSON) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", checkpointDeployConfig, "dry-run", "true")) - assert.Contains(t, buf.String(), `Dry run: configuration for endpoint "qwen-chat" is valid; not deploying.`) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, registeredDeployConfig)) + assert.Equal(t, "7", endpoints.endpoint.Config.ServedEntities[0].EntityVersion) +} - buf.Reset() - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", checkpointDeployConfig, "dry-run", "true")) - var got struct { - Data deployResult `json:"data"` - } +func TestDeployRegisteredWithoutAReadyVersionFails(t *testing.T) { + srv, _ := deployServerWithVersions(t, newFakeEndpoints(t, nil), `{"model_versions": [{"version": 1, "status": "PENDING_REGISTRATION"}]}`) + + var buf bytes.Buffer + err := runDeployCmd(t, flags.OutputJSON, &buf, srv.URL, registeredNoVersionConfig) + require.ErrorIs(t, err, root.ErrAlreadyPrinted) + var got errorEnvelope require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) - assert.Equal(t, deployResult{Status: "DRY_RUN_OK", DryRun: true}, got.Data) + assert.Equal(t, "INVALID_ARGS", got.Error.Code) + assert.Contains(t, got.Error.Message, "main.models.qwen has no READY version to serve; register and finalize a version, or set model_version") } -func TestDeployDryRunListsCodeFiles(t *testing.T) { - cfg, _ := writeCodeTree(t, map[string]string{"agent.py": "x\n", "tools/a.py": "a\n"}) +func TestDeployDryRunNeedsNoWorkspace(t *testing.T) { + cmd := newDeployCommand() + require.NoError(t, cmd.Flags().Set("dry-run", "true")) + assert.NoError(t, cmd.PreRunE(cmd, nil)) var buf bytes.Buffer - require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", entryConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) - assert.Contains(t, buf.String(), "Dry run: model.entry would send 2 file(s):\n agent.py\n tools/a.py\n") + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", commandDeployConfig, "dry-run", "true")) + out := buf.String() + assert.Contains(t, out, "Dry run: the file is read as a checkpoint served by command, because it sets checkpoint and command.") + assert.Contains(t, out, "Dry run: would register a new version of main.models.qwen from /Volumes/main/models/weights/qwen3_8b, and serve it.") + assert.Contains(t, out, `"workload_type": "GPU_MEDIUM"`) + assert.Contains(t, out, `Dry run: configuration for endpoint "qwen-chat" is valid; not deploying.`) buf.Reset() - require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", entryConfigAt(cfg.CodeSource.RootPath), "dry-run", "true")) + require.NoError(t, runDeployCmd(t, flags.OutputJSON, &buf, "", commandDeployConfig, "dry-run", "true")) var got struct { Data deployResult `json:"data"` } require.NoError(t, json.Unmarshal(buf.Bytes(), &got)) - assert.Equal(t, []string{"agent.py", "tools/a.py"}, got.Data.CodeFiles) + assert.Equal(t, "DRY_RUN_OK", got.Data.Status) + assert.True(t, got.Data.DryRun) + require.NotNil(t, got.Data.Plan) + assert.Equal(t, "command", got.Data.Plan.Variant) + assert.Equal(t, "/Volumes/main/models/weights/qwen3_8b", got.Data.Plan.Checkpoint) + served, ok := got.Data.Plan.ServedEntity.(map[string]any) + require.True(t, ok) + assert.Equal(t, "GPU_MEDIUM", served["workload_type"]) + assert.Equal(t, false, served["scale_to_zero_enabled"]) } -// entryConfigAt is entryDeployConfig with its code at root. -func entryConfigAt(root string) string { - return strings.Replace(entryDeployConfig, "root_path: .", "root_path: "+strconv.Quote(filepath.ToSlash(root)), 1) +func TestDeployDryRunShowsEachVariant(t *testing.T) { + tests := []struct { + name string + config string + want []string + }{ + { + "dedicated", + dedicatedDeployConfig, + []string{ + "Dry run: the file is read as a dedicated checkpoint, because it sets checkpoint but not command.", + "would register a new version of main.models.qwen from /Volumes/main/models/weights/qwen3_8b, and serve it.", + `"compute_profile": "COMPUTE_PROFILE_LOW_LATENCY"`, + `"provisioned_model_units": 600`, + }, + }, + { + "registered", + registeredDeployConfig, + []string{ + "Dry run: the file is read as a registered model, because it sets no checkpoint.", + "would serve version 7 of main.models.qwen.", + `"workload_type": "CPU"`, + }, + }, + { + "registered without a version", + registeredNoVersionConfig, + []string{"would serve the newest READY version of main.models.qwen, picked when deploying."}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, runDeployCmd(t, flags.OutputText, &buf, "", tt.config, "dry-run", "true")) + for _, want := range tt.want { + assert.Contains(t, buf.String(), want) + } + }) + } } func TestDeployFlagValidation(t *testing.T) { - registered := "registered_model: a.b.c\nmodel: {type: registered, version: 1}\nendpoint: e\ncompute: {workload_type: CPU, workload_size: Small}\n" tests := []struct { name string config string flagValues []string wantErr string }{ - {"key for a registered version", registered, []string{"idempotency-key", "k"}, "--idempotency-key applies only when air deploy registers a version"}, - {"bad key", checkpointDeployConfig, []string{"idempotency-key", "a/b"}, "--idempotency-key must be 1 to 128 letters"}, - {"retry for a registered version", registered, []string{"retry-failed-staging", "true"}, "--retry-failed-staging applies only to a checkpoint"}, - {"zero timeout", checkpointDeployConfig, []string{"timeout", "0s"}, "--timeout must be positive"}, - {"empty key", checkpointDeployConfig, []string{"idempotency-key", ""}, `--idempotency-key must be 1 to 128 letters, digits, '.', '_' or '-', got ""`}, + {"key for a registered version", registeredDeployConfig, []string{"idempotency-key", "k"}, "--idempotency-key applies only when air deploy registers a version; the file is read as a registered model, because it sets no checkpoint"}, + {"bad key", commandDeployConfig, []string{"idempotency-key", "a/b"}, "--idempotency-key must be 1 to 128 letters"}, + {"retry for a registered version", registeredDeployConfig, []string{"retry-failed-staging", "true"}, "--retry-failed-staging applies only to a checkpoint; the file is read as a registered model"}, + {"zero timeout", commandDeployConfig, []string{"timeout", "0s"}, "--timeout must be positive"}, + {"empty key", commandDeployConfig, []string{"idempotency-key", ""}, `--idempotency-key must be 1 to 128 letters, digits, '.', '_' or '-', got ""`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -404,9 +489,9 @@ func TestDeployFlagValidation(t *testing.T) { func TestDeployResumeCommand(t *testing.T) { abs, err := filepath.Abs("my deploy.yaml") require.NoError(t, err) - opts := deployOptions{file: "my deploy.yaml", overrides: []string{"compute.workload_size=Medium", "task=llm/v1/chat"}} + opts := deployOptions{file: "my deploy.yaml", overrides: []string{"compute.num_accelerators=4", "task=llm/v1/chat"}} assert.Equal(t, - "databricks air deploy -f "+shellquote.BashArg(filepath.ToSlash(abs))+" --override 'compute.workload_size=Medium' --override 'task=llm/v1/chat' --idempotency-key k-1 -p dev", + "databricks air deploy -f "+shellquote.BashArg(filepath.ToSlash(abs))+" --override 'compute.num_accelerators=4' --override 'task=llm/v1/chat' --idempotency-key k-1 -p dev", deployResumeCommand("dev", opts, "k-1")) abs, err = filepath.Abs("d.yaml") @@ -438,7 +523,7 @@ func TestDeployErrorCode(t *testing.T) { }{ {"interrupted", fmt.Errorf("x: %w", errDeployInterrupted), "INTERRUPTED", true}, {"timed out", fmt.Errorf("x: %w", errDeployTimedOut), "TIMEOUT", true}, - {"invalid input", withCode(codeInvalidArgs, false, errors.New("model: section is required")), "INVALID_ARGS", false}, + {"invalid input", withCode(codeInvalidArgs, false, errors.New("registered_model_name must be a Unity Catalog model name")), "INVALID_ARGS", false}, {"staging failed", withCode(codeStagingFailed, true, errors.New("copying failed")), "STAGING_FAILED", true}, {"server unavailable", fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "TEMPORARILY_UNAVAILABLE", StatusCode: 503}), "TEMPORARILY_UNAVAILABLE", true}, {"bad request", fmt.Errorf("x: %w", &apierr.APIError{ErrorCode: "INVALID_PARAMETER_VALUE", StatusCode: 400}), "INVALID_PARAMETER_VALUE", false}, @@ -457,10 +542,17 @@ func TestDeployHelpDocumentsAField(t *testing.T) { var out strings.Builder cmd := newDeployCommand() cmd.SetOut(&out) - cmd.SetArgs([]string{"-h", "config.model.type"}) + cmd.SetArgs([]string{"-h", "config.checkpoint"}) + require.NoError(t, cmd.Execute()) + assert.Contains(t, out.String(), "config.checkpoint\n Unity Catalog volume directory") + assert.Contains(t, out.String(), "Required: no") + + out.Reset() + cmd = newDeployCommand() + cmd.SetOut(&out) + cmd.SetArgs([]string{"-h", "config.compute.compute_profile"}) require.NoError(t, cmd.Execute()) - assert.Contains(t, out.String(), "config.model.type\n One of checkpoint") - assert.Contains(t, out.String(), "Required: yes") + assert.Contains(t, out.String(), "Required: when checkpoint is set without command") out.Reset() cmd = newDeployCommand() diff --git a/cmd/air/deploycode.go b/cmd/air/deploycode.go deleted file mode 100644 index 96fb72f129c..00000000000 --- a/cmd/air/deploycode.go +++ /dev/null @@ -1,229 +0,0 @@ -package aircmd - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "os" - "os/exec" - "path/filepath" - "regexp" - "slices" - "strings" - "sync" - "unicode/utf8" -) - -// Limits of the Python code of model.entry, as the server enforces them. -const ( - // maxCodePackageBytes caps the code files plus the dependency lines. The server - // also counts the small environment file it writes, so this check rejects only - // packages the server would reject too. - maxCodePackageBytes = 4 * 1024 * 1024 - // maxCodeFiles caps the files besides the entry file. - maxCodeFiles = 1000 - maxCodePathDepth = 16 -) - -// pythonCacheDir holds bytecode Python writes next to imported sources. -const pythonCacheDir = "__pycache__" - -// codePathSegmentRe matches a segment of a code file path the server can place. -var codePathSegmentRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*$`) }) - -// collectEntryCode reads the Python code that serves a checkpoint from -// code_source: model.entry and the other files under code_source.include_paths, -// as UTF-8 text, selected as a code_source snapshot selects them. Files git ignores aren't sent, and nor are -// hidden files and directories or __pycache__ directories, whose names the -// server can't place or which hold no source. configPath locates a relative -// code_source.root_path. -func collectEntryCode(ctx context.Context, cfg *deployConfig, configPath string) (*modelPackagePythonModel, error) { - root, err := resolveRootPath(ctx, cfg.CodeSource.RootPath, filepath.Dir(configPath)) - if err != nil { - return nil, err - } - isGitRepo := newGitRepo(root).isRepository(ctx) - files, err := snapshotFiles(ctx, root, cfg.CodeSource.IncludePaths, isGitRepo) - if err != nil { - return nil, err - } - files = slices.DeleteFunc(files, skippedCodeFile) - if isGitRepo { - ignored, err := ignoredTrackedFiles(ctx, root, files) - if err != nil { - return nil, err - } - if len(ignored) > 0 { - return nil, fmt.Errorf("code_source file %s matches a .gitignore rule but is tracked by git; air deploy doesn't send ignored files, so leave it out of code_source.include_paths or stop tracking it", ignored[0]) - } - } - - code := &modelPackagePythonModel{EntryFile: *cfg.Model.Entry} - var dependencies []string - if cfg.Environment != nil { - dependencies = cfg.Environment.Dependencies - } - // Check the listing before reading anything, so a root_path that holds large - // data fails fast instead of being read into memory. - if err := checkCodeFiles(files, dependencies); err != nil { - return nil, err - } - - foundEntry := false - for _, f := range files { - rel := filepath.ToSlash(f.rel) - content, err := readListedFile(filepath.Join(root, f.rel), f.size) - if errors.Is(err, errCodeFileChanged) { - return nil, fmt.Errorf("code_source file %s changed while air deploy was reading it; run air deploy again", rel) - } - if err != nil { - return nil, fmt.Errorf("failed to read code_source file %s: %w", rel, err) - } - if !utf8.Valid(content) { - return nil, fmt.Errorf("code_source file %s isn't UTF-8 text; air deploy sends source files only, so leave it out with code_source.include_paths", rel) - } - if rel == code.EntryFile { - code.EntrySource = string(content) - foundEntry = true - continue - } - code.Files = append(code.Files, modelPackageFile{Path: rel, Content: string(content)}) - } - if !foundEntry { - return nil, fmt.Errorf("model.entry %s isn't among the files code_source sends from %s; check that it exists, isn't ignored by git and is under code_source.include_paths", code.EntryFile, filepath.ToSlash(root)) - } - return code, nil -} - -// errCodeFileChanged reports a code file that isn't the one listed anymore. -var errCodeFileChanged = errors.New("code file changed since it was listed") - -// readListedFile reads the regular file of size bytes listed at path. It opens -// the file before checking it, so a file replaced after the listing, such as by a -// symlink to a file outside the project, is refused rather than sent. -func readListedFile(path string, size int64) ([]byte, error) { - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - opened, err := f.Stat() - if err != nil { - return nil, err - } - atPath, err := os.Lstat(path) - if err != nil { - return nil, err - } - if !atPath.Mode().IsRegular() || !os.SameFile(opened, atPath) || opened.Size() != size { - return nil, errCodeFileChanged - } - content, err := io.ReadAll(io.LimitReader(f, size+1)) - if err != nil { - return nil, err - } - if int64(len(content)) != size { - return nil, errCodeFileChanged - } - return content, nil -} - -// ignoredTrackedFiles returns the files that a .gitignore rule matches although -// git tracks them: ls-files lists tracked files whatever the rules say, and the -// rules are the user's statement of what not to share. -func ignoredTrackedFiles(ctx context.Context, root string, files []snapshotFile) ([]string, error) { - var paths bytes.Buffer - for _, f := range files { - paths.WriteString(filepath.ToSlash(f.rel)) - paths.WriteByte(0) - } - var stderr bytes.Buffer - cmd := exec.CommandContext(ctx, "git", "-C", root, "check-ignore", "--no-index", "-z", "--stdin") - cmd.Stdin = &paths - cmd.Stderr = &stderr - out, err := cmd.Output() - // check-ignore exits 1 when no path is ignored. - if exitErr, ok := errors.AsType[*exec.ExitError](err); ok && exitErr.ExitCode() == 1 { - return nil, nil - } - if err != nil { - return nil, fmt.Errorf("failed to evaluate git ignore rules: %w: %s", err, strings.TrimSpace(stderr.String())) - } - var ignored []string - for raw := range bytes.SplitSeq(out, []byte{0}) { - if len(raw) > 0 { - ignored = append(ignored, string(raw)) - } - } - return ignored, nil -} - -// checkCodeFiles checks the files to send against the server's limits, from -// their listing alone. -func checkCodeFiles(files []snapshotFile, dependencies []string) error { - // One of the files is the entry file, which the limit doesn't count. - if len(files) > maxCodeFiles+1 { - return fmt.Errorf("code_source sends %d files, but model.entry can bring at most %d other files; narrow it with code_source.include_paths", len(files), maxCodeFiles) - } - var size int64 - for _, dependency := range dependencies { - size += int64(len(dependency)) + 1 - } - for _, f := range files { - rel := filepath.ToSlash(f.rel) - if !os.FileMode(f.mode).IsRegular() { - return fmt.Errorf("code_source path %s isn't a regular file; air deploy sends the contents of files only, so replace it with a file or leave it out with code_source.include_paths", rel) - } - if err := validateCodePath(rel); err != nil { - return err - } - size += f.size - if size > maxCodePackageBytes { - return fmt.Errorf("code_source files and environment.dependencies total more than %d bytes, the most a checkpoint's Python code can hold; narrow code_source.include_paths", maxCodePackageBytes) - } - } - return nil -} - -// codeFilePaths lists the files of code, entry file first, as dry runs show them. -func codeFilePaths(code *modelPackagePythonModel) []string { - paths := []string{code.EntryFile} - for _, f := range code.Files { - paths = append(paths, f.Path) - } - return paths -} - -// sentCodePath reports whether a code_source file at rel (slash-separated) is -// sent: hidden files and directories and __pycache__ directories are skipped. -func sentCodePath(rel string) bool { - segments := strings.Split(rel, "/") - directories := segments[:len(segments)-1] - return !slices.ContainsFunc(segments, isHiddenName) && !slices.Contains(directories, pythonCacheDir) -} - -// skippedCodeFile reports whether a code_source file isn't sent. -func skippedCodeFile(f snapshotFile) bool { - return !sentCodePath(filepath.ToSlash(f.rel)) -} - -// isHiddenName reports whether a file or directory name is hidden by convention. -func isHiddenName(name string) bool { - return strings.HasPrefix(name, ".") -} - -// validateCodePath checks that the server can place a code file at rel. -func validateCodePath(rel string) error { - segments := strings.Split(rel, "/") - if len(segments) <= maxCodePathDepth && !slices.ContainsFunc(segments, invalidCodePathSegment) { - return nil - } - return fmt.Errorf("code_source file %s can't be sent: a path must have at most %d parts of letters, digits, '_', '-' and '.', each starting with a letter, digit or '_'; rename it or leave it out with code_source.include_paths", rel, maxCodePathDepth) -} - -// invalidCodePathSegment reports whether the server can't place a path segment. -func invalidCodePathSegment(segment string) bool { - return !codePathSegmentRe().MatchString(segment) -} diff --git a/cmd/air/deploycode_test.go b/cmd/air/deploycode_test.go deleted file mode 100644 index 04717cd132c..00000000000 --- a/cmd/air/deploycode_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package aircmd - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// writeCodeTree writes files (slash-separated path to content) into a new git -// repository and returns a deployment config whose code_source is its root. -func writeCodeTree(t *testing.T, files map[string]string) (*deployConfig, string) { - t.Helper() - return codeTreeConfig(t, newTestRepo(t), files) -} - -// codeTreeConfig writes files under dir and returns a deployment config whose -// code_source is dir. -func codeTreeConfig(t *testing.T, dir string, files map[string]string) (*deployConfig, string) { - t.Helper() - for rel, content := range files { - writeRepoFile(t, dir, filepath.FromSlash(rel), content) - } - entry, task, path := "agent.py", "agent/v1/responses", "/Volumes/main/models/weights/support" - cfg := &deployConfig{ - Model: &deployModelConfig{Type: modelTypeCheckpoint, Path: &path, Entry: &entry}, - Task: &task, - CodeSource: &deployCodeSourceConfig{RootPath: dir}, - Environment: &deployEnvironmentConfig{Dependencies: []string{"mlflow==3.5.0"}}, - Compute: &deployComputeConfig{WorkloadType: "GPU_SMALL", WorkloadSize: "Small"}, - } - return cfg, filepath.Join(dir, "deploy.yaml") -} - -func TestCollectModelCode(t *testing.T) { - cfg, configPath := writeCodeTree(t, map[string]string{ - "agent.py": "import tools.search\n", - "tools/__init__.py": "", - "tools/search.py": "def search(): ...\n", - "tools/__pycache__/search.pyc": "\x00\xff", - ".env": "TOKEN=x\n", - ".venv/lib/site.py": "x = 1\n", - "notes/.draft.py": "y = 2\n", - "deploy.yaml": "endpoint: e\n", - "configs/prompts/system.md": "You are helpful.\n", - "configs/prompts/__init__.py.md": "not hidden\n", - }) - - code, err := collectEntryCode(t.Context(), cfg, configPath) - require.NoError(t, err) - assert.Equal(t, "agent.py", code.EntryFile) - assert.Equal(t, "import tools.search\n", code.EntrySource) - - var paths []string - for _, f := range code.Files { - paths = append(paths, f.Path) - } - assert.ElementsMatch(t, []string{ - "configs/prompts/__init__.py.md", - "configs/prompts/system.md", - "deploy.yaml", - "tools/__init__.py", - "tools/search.py", - }, paths) -} - -func TestCollectModelCodeIsDeterministic(t *testing.T) { - cfg, configPath := writeCodeTree(t, map[string]string{ - "agent.py": "x = 1\n", "b.py": "b\n", "a/c.py": "c\n", "a.py": "a\n", - }) - first, err := collectEntryCode(t.Context(), cfg, configPath) - require.NoError(t, err) - second, err := collectEntryCode(t.Context(), cfg, configPath) - require.NoError(t, err) - assert.Equal(t, first, second) -} - -func TestCollectModelCodeIncludePaths(t *testing.T) { - cfg, configPath := writeCodeTree(t, map[string]string{ - "agent.py": "x = 1\n", "tools/a.py": "a\n", "data/big.csv": "1,2\n", - }) - cfg.CodeSource.IncludePaths = []string{"agent.py", "tools"} - code, err := collectEntryCode(t.Context(), cfg, configPath) - require.NoError(t, err) - require.Len(t, code.Files, 1) - assert.Equal(t, "tools/a.py", code.Files[0].Path) -} - -func TestCollectModelCodeErrors(t *testing.T) { - tests := []struct { - name string - files map[string]string - include []string - wantErr string - }{ - {"missing entry", map[string]string{"main.py": "x\n"}, nil, "model.entry agent.py isn't among the files code_source sends"}, - {"entry left out", map[string]string{"agent.py": "x\n", "tools/a.py": "a\n"}, []string{"tools"}, "model.entry agent.py isn't among the files code_source sends"}, - {"binary file", map[string]string{"agent.py": "x\n", "model.bin": "\xff\xfe\x00"}, nil, "code_source file model.bin isn't UTF-8 text"}, - {"unplaceable name", map[string]string{"agent.py": "x\n", "my tools/a.py": "a\n"}, nil, "code_source file my tools/a.py can't be sent"}, - {"too deep", map[string]string{"agent.py": "x\n", strings.Repeat("d/", maxCodePathDepth) + "a.py": "a\n"}, nil, "can't be sent: a path must have at most 16 parts"}, - {"too large", map[string]string{"agent.py": "x\n", "blob.txt": strings.Repeat("a", maxCodePackageBytes)}, nil, "total more than 4194304 bytes"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cfg, configPath := writeCodeTree(t, tt.files) - cfg.CodeSource.IncludePaths = tt.include - _, err := collectEntryCode(t.Context(), cfg, configPath) - assert.ErrorContains(t, err, tt.wantErr) - }) - } -} - -func TestCollectModelCodeLeavesOutIgnoredFiles(t *testing.T) { - cfg, configPath := writeCodeTree(t, map[string]string{ - "agent.py": "x\n", "credentials.json": "{}\n", "tools/a.py": "a\n", - }) - require.NoError(t, os.WriteFile(filepath.Join(cfg.CodeSource.RootPath, ".gitignore"), []byte("credentials.json\n"), 0o600)) - code, err := collectEntryCode(t.Context(), cfg, configPath) - require.NoError(t, err) - assert.Equal(t, []string{"agent.py", "tools/a.py"}, codeFilePaths(code)) -} - -func TestCollectModelCodeRefusesTrackedIgnoredFiles(t *testing.T) { - cfg, configPath := writeCodeTree(t, map[string]string{"agent.py": "x\n", "creds.json": "{}\n"}) - repo := cfg.CodeSource.RootPath - commitAll(t, repo, "init") - writeRepoFile(t, repo, ".gitignore", "creds.json\n") - cfg.CodeSource.IncludePaths = []string{"agent.py", "creds.json"} - - _, err := collectEntryCode(t.Context(), cfg, configPath) - assert.ErrorContains(t, err, "code_source file creds.json matches a .gitignore rule but is tracked by git") -} - -func TestCollectModelCodeOutsideGit(t *testing.T) { - cfg, configPath := codeTreeConfig(t, t.TempDir(), map[string]string{"agent.py": "x\n", "tools/a.py": "a\n", "credentials.json": "{}\n"}) - cfg.CodeSource.IncludePaths = []string{"agent.py", "tools"} - code, err := collectEntryCode(t.Context(), cfg, configPath) - require.NoError(t, err) - assert.Equal(t, []string{"agent.py", "tools/a.py"}, codeFilePaths(code)) -} - -func TestCollectModelCodeChecksSizesBeforeReading(t *testing.T) { - // Read in full, this file would fail as binary; the size check must reject it - // from its listing first. - cfg, configPath := writeCodeTree(t, map[string]string{ - "agent.py": "x\n", "data.bin": strings.Repeat("\xff", maxCodePackageBytes+1), - }) - _, err := collectEntryCode(t.Context(), cfg, configPath) - assert.ErrorContains(t, err, "total more than 4194304 bytes") -} - -func TestCollectModelCodeRejectsSymlink(t *testing.T) { - cfg, configPath := writeCodeTree(t, map[string]string{"agent.py": "x\n", "real.py": "r\n"}) - dir := filepath.Dir(configPath) - if err := os.Symlink("real.py", filepath.Join(dir, "link.py")); err != nil { - t.Skipf("symlinks unavailable: %v", err) - } - _, err := collectEntryCode(t.Context(), cfg, configPath) - assert.ErrorContains(t, err, "code_source path link.py isn't a regular file") -} - -func TestReadListedFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "model.py") - require.NoError(t, os.WriteFile(path, []byte("x = 1\n"), 0o600)) - content, err := readListedFile(path, 6) - require.NoError(t, err) - assert.Equal(t, "x = 1\n", string(content)) -} - -func TestReadListedFileRefusesAFileReplacedByASymlink(t *testing.T) { - dir := t.TempDir() - secret := filepath.Join(dir, "secret") - require.NoError(t, os.WriteFile(secret, []byte("token\n"), 0o600)) - path := filepath.Join(dir, "model.py") - if err := os.Symlink(secret, path); err != nil { - t.Skipf("symlinks unavailable: %v", err) - } - // Listed as a regular file of the secret's size, then replaced by the link. - _, err := readListedFile(path, 6) - assert.ErrorIs(t, err, errCodeFileChanged) -} - -func TestReadListedFileRefusesAFileThatChangedSize(t *testing.T) { - path := filepath.Join(t.TempDir(), "model.py") - require.NoError(t, os.WriteFile(path, []byte("x = 1\ny = 2\n"), 0o600)) - _, err := readListedFile(path, 6) - assert.ErrorIs(t, err, errCodeFileChanged) -} - -func TestCollectModelCodeTooManyFiles(t *testing.T) { - files := map[string]string{"agent.py": "x\n"} - for i := range maxCodeFiles + 1 { - files[fmt.Sprintf("pkg/m%d.py", i)] = "" - } - cfg, configPath := writeCodeTree(t, files) - _, err := collectEntryCode(t.Context(), cfg, configPath) - assert.ErrorContains(t, err, "code_source sends 1002 files, but model.entry can bring at most 1000 other files") -} - -func TestSentCodePath(t *testing.T) { - for rel, want := range map[string]bool{ - "agent.py": true, - "tools/search.py": true, - ".env": false, - "tools/.cache/x.py": false, - "__pycache__/a.pyc": false, - "tools/__pycache__/a.p": false, - "__pycache__": true, - } { - assert.Equal(t, want, sentCodePath(rel), rel) - } -} diff --git a/cmd/air/deployconfig.go b/cmd/air/deployconfig.go index 06e6a32064e..29449b4a688 100644 --- a/cmd/air/deployconfig.go +++ b/cmd/air/deployconfig.go @@ -3,7 +3,6 @@ package aircmd import ( "bytes" "context" - "encoding/json" "errors" "fmt" "maps" @@ -15,23 +14,40 @@ import ( "sync" "unicode/utf16" - "github.com/databricks/databricks-sdk-go/service/serving" "go.yaml.in/yaml/v3" ) -// This file defines the `air deploy` YAML schema and its structural validation. -// Fields shared with `air run` (environment, code_source, env_variables, secrets, -// idempotency_token) keep their meaning; the server validates what needs the -// workspace, such as task names, runtimes and dependencies. +// This file defines the `air deploy` YAML schema and its validation. The fields a +// file sets select its variant, and each variant reads its own fields. Fields shared +// with `air run` (environment, env_variables, secrets, idempotency_token) keep their +// meaning; the server validates what needs the workspace, such as task names, +// runtimes and dependencies. -// Model sources, as model.type. +// Variants of a deployment, named after what serves the checkpoint. const ( - modelTypeCheckpoint = "checkpoint" - modelTypeRegistered = "registered" + // variantCommand: a checkpoint served by the command the file sets. + variantCommand = "command" + // variantDedicated: a checkpoint Databricks serves with dedicated inference. + variantDedicated = "dedicated" + // variantRegistered: a model version already registered in Unity Catalog, which + // air deploy serves without registering another. + variantRegistered = "registered" ) -// modelTypes lists model.type values in the order the help and errors name them. -var modelTypes = []string{modelTypeCheckpoint, modelTypeRegistered} +// variantDescriptions and variantReasons say how a file was read, in errors about +// its fields: "the file is read as , because ". +var ( + variantDescriptions = map[string]string{ + variantCommand: "a checkpoint served by command", + variantDedicated: "a dedicated checkpoint", + variantRegistered: "a registered model", + } + variantReasons = map[string]string{ + variantCommand: "it sets checkpoint and command", + variantDedicated: "it sets checkpoint but not command", + variantRegistered: "it sets no checkpoint", + } +) // Audio tasks: their requests are multipart uploads of a recording, sent to the // endpoint's audio route rather than to invocations. @@ -43,9 +59,13 @@ const ( // deployCommandPath is the command whose -h documents deployConfig. const deployCommandPath = "databricks air deploy" -// defaultServingEnvironmentVersion is the serverless environment version a model -// is served on when environment.version is unset, as for `air run`. -const defaultServingEnvironmentVersion = 6 +// Serving compute a command or registered model gets when compute is omitted: one +// A10 for a command, CPU for a registered model. +const ( + defaultCommandWorkloadType = "GPU_MEDIUM" + defaultRegisteredWorkloadType = "CPU" + defaultWorkloadSize = "Small" +) // Limits the server enforces on a registration, checked here to fail early. const ( @@ -54,10 +74,6 @@ const ( maxDependencies = 1000 ) -// cpuWorkloadTypePrefix starts the name of every CPU serving workload type (CPU, -// CPU_MEDIUM, CPU_LARGE); the others are GPU workload types. -const cpuWorkloadTypePrefix = "CPU" - // registeredModelRe matches .. with the characters the // server accepts in each part. var registeredModelRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile("^[^\\s./`]+\\.[^\\s./`]+\\.[^\\s./`]+$") }) @@ -75,98 +91,141 @@ var endpointNameRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCo // workloadSizes are the concurrency tiers a served entity scales to. var workloadSizes = []string{"Small", "Medium", "Large"} -// entryFileRe matches the name of model.entry, the Python file the server places -// at the model's root. -var entryFileRe = sync.OnceValue(func() *regexp.Regexp { return regexp.MustCompile(`^[A-Za-z0-9_][A-Za-z0-9_.-]*\.py$`) }) +// servingShape is serving compute the server offers: an accelerator type, the number +// of accelerators, and the workload type that serves them. +type servingShape struct { + accelerator gpuType + count int + workloadType string +} -// deployConfig is the `air deploy` YAML schema: a model to register in Unity -// Catalog and the serving endpoint that serves it. -type deployConfig struct { - RegisteredModel string `yaml:"registered_model" help:"Unity Catalog model, as ... air deploy registers a new version of it, creating the model if it doesn't exist, unless model.type is registered." required:"yes" group:"Model"` - Model *deployModelConfig `yaml:"model" help:"What to serve: a checkpoint, or an existing version." required:"yes" group:"Model"` - Task *string `yaml:"task" help:"The model's task as MLflow names it. Required with command, and then one Model Serving serves a command for, such as llm/v1/chat, llm/v1/embeddings or llm/v1/audio/transcriptions. Optional with model.entry, e.g. llm/v1/chat or agent/v1/responses, where it selects MLflow's signature; a model without a task has no signature." required:"when command is set" group:"Model"` - Command *string `yaml:"command" help:"Shell command that starts a model server over a checkpoint's weights, at most 32,768 characters, serving task on port 8080. It runs from the directory where model.path appears under its own name. Set either command or model.entry." required:"when model.type is checkpoint and model.entry is not set" group:"Model"` - Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment to serve a checkpoint on and the packages to install." group:"Model"` - CodeSource *deployCodeSourceConfig `yaml:"code_source" help:"Local code of model.entry, sent with the registration and stored in the model version, where anyone who can read the model can read it. Only the files under include_paths are sent, leaving out files git ignores, hidden files and directories (names starting with '.') and __pycache__ directories; run with --dry-run to list them." required:"when model.entry is set" group:"Model"` - - Endpoint string `yaml:"endpoint" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead, with the compute, environment variables and secrets in this file replacing the ones it has; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` - Compute *deployComputeConfig `yaml:"compute" help:"Serving compute and how the endpoint scales." required:"yes" group:"Serving"` - EnvVariables map[string]string `yaml:"env_variables" help:"Plain environment variables of the served model, as NAME: value. A name here cannot also appear in secrets." group:"Serving"` - Secrets map[string]string `yaml:"secrets" help:"Environment variables of the served model sourced from secrets, as NAME: scope/key." group:"Serving"` - - IdempotencyToken *string `yaml:"idempotency_token" help:"Key of the registration: 1 to 128 letters, digits, '.', '_' or '-'. Running air deploy again with the same key and unchanged inputs continues that registration instead of registering another version. Defaults to a new key, which air deploy prints." group:"Registration"` - MLflowExperimentPath *string `yaml:"mlflow_experiment_path" help:"Workspace path of the MLflow experiment that records the registered model (default: /Users//air-deployments)." group:"Registration"` +// servingShapes lists the serving compute a command or registered model can use, in +// the order errors name them. +var servingShapes = []servingShape{ + {gpuType1xA10, 1, "GPU_MEDIUM"}, + {gpuType1xA10, 4, "MULTIGPU_MEDIUM"}, + {gpuType1xA10, 8, "GPU_MEDIUM_8"}, + {gpuType1xH100, 1, "GPU_XLARGE"}, + {gpuType8xH100, 8, "GPU_XLARGE_8"}, + {gpuType1xRTX6000, 1, "GPU_LARGE_RTX"}, } -// deployModelConfig is the `model` block: where the served model comes from. -type deployModelConfig struct { - Type string `yaml:"type" help:"One of checkpoint (weights in a volume, served by command or by the Python file model.entry) or registered (an existing version of registered_model)." required:"yes"` - Path *string `yaml:"path" help:"Unity Catalog volume directory, as /Volumes////, holding a checkpoint's weights. Model Serving copies it, with its subdirectories, into the version, where it appears under its own name: command finds it there, and model.entry reads its path as context.artifacts[\"\"]." required:"when type is checkpoint"` - Entry *string `yaml:"entry" help:"Python file directly under code_source.root_path that builds the model and calls mlflow.models.set_model, e.g. model.py. Model Serving's model server loads it and calls its predict. Set either entry or command." required:"when type is checkpoint and command is not set"` - ModelConfig map[string]any `yaml:"model_config" help:"Settings model.entry reads as context.model_config, as MLflow's log_model(model_config=...) records them."` - Streamable *bool `yaml:"streamable" help:"Whether model.entry answers streaming requests with predict_stream. MLflow detects this from the class when it logs a model; here it is declared. Defaults to true for a chat or agent task and false otherwise."` - Version *string `yaml:"version" help:"Version of registered_model to serve, e.g. 3." required:"when type is registered"` +// computeProfiles are the dedicated compute profiles, lowercase as the file names +// them. The API takes each as COMPUTE_PROFILE_ followed by its uppercase form. +var computeProfiles = []string{"low_latency", "fast", "throughput", "minimal_cost"} + +// deployConfig is the `air deploy` YAML schema: a model to serve on an endpoint, +// and the checkpoint to register as a version of it, if any. +type deployConfig struct { + RegisteredModel string `yaml:"registered_model_name" help:"Unity Catalog model, as ... A checkpoint is registered as a new version of it, creating the model if it doesn't exist. Without a checkpoint, the model must already have a version to serve." required:"yes" group:"Model"` + Checkpoint *string `yaml:"checkpoint" help:"Unity Catalog volume directory holding a checkpoint's weights, as /Volumes////. Model Serving copies it, with its subdirectories, into the version air deploy registers. Leave it out to serve a version that is already registered." group:"Model"` + Command *string `yaml:"command" help:"Shell command that starts a model server over the checkpoint's weights on port 8080, at most 32,768 characters. It runs from the directory where the checkpoint appears under its own name. Set it to serve the checkpoint with your own server; without it, Databricks serves the checkpoint with dedicated inference." group:"Model"` + Task *string `yaml:"task" help:"The task Model Serving routes to the command, such as llm/v1/chat, llm/v1/embeddings or llm/v1/audio/transcriptions." required:"when command is set" group:"Model"` + Environment *deployEnvironmentConfig `yaml:"environment" help:"Serverless environment to serve the command on, and the packages to install. The model uses its Python version." required:"when command is set" group:"Model"` + ModelVersion *string `yaml:"model_version" help:"Version of registered_model to serve, e.g. 3. Without it, the newest READY version is served." group:"Model"` + + Endpoint string `yaml:"endpoint_name" help:"Serving endpoint to serve the model on. It is created if it doesn't exist. An endpoint that serves one version of registered_model is updated to serve this version instead, with the compute, environment variables and secrets in this file replacing the ones it has; air deploy doesn't change an endpoint that serves anything else." required:"yes" group:"Serving"` + Compute *deployComputeConfig `yaml:"compute" help:"Compute to serve on. A command or registered model takes serving compute, and a dedicated checkpoint takes dedicated compute. Without compute, a command is served on one GPU_1xA10 with workload size Small, and a registered model on CPU with workload size Small." required:"when checkpoint is set without command" group:"Serving"` + EnvVariables map[string]string `yaml:"env_variables" help:"Plain environment variables of the served model, as NAME: value. A name here cannot also appear in secrets. A command or registered model only." group:"Serving"` + Secrets map[string]string `yaml:"secrets" help:"Environment variables of the served model sourced from secrets, as NAME: scope/key. A command or registered model only." group:"Serving"` + + IdempotencyToken *string `yaml:"idempotency_token" help:"Key of the registration: 1 to 128 letters, digits, '.', '_' or '-'. Running air deploy again with the same key and unchanged inputs continues that registration instead of registering another version. Defaults to a new key, which air deploy prints. A checkpoint only." group:"Registration"` + MLflowExperimentPath *string `yaml:"mlflow_experiment_path" help:"Workspace path of the MLflow experiment that records the registered model (default: /Users//air-deployments). A checkpoint only." group:"Registration"` } -// deployEnvironmentConfig is the `environment` block of a deployment. +// deployEnvironmentConfig is the `environment` block of a command. type deployEnvironmentConfig struct { - Version *stringOrInt `yaml:"version" help:"Serverless environment to serve on: 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment (default: 6), as for air run. The model uses its Python version. Standard includes CUDA but not PyTorch; the Databricks AI environment adds PyTorch, Transformers and vLLM, and needs GPU compute. A command that doesn't install SGLang runs on the Databricks AI environment either way."` + Version *stringOrInt `yaml:"version" required:"yes" help:"Serverless environment to serve on: 6 for the Standard environment or databricks_ai_v6 for the Databricks AI environment, as for air run. The model uses its Python version. Standard includes CUDA but not PyTorch; the Databricks AI environment adds PyTorch, Transformers and vLLM, and needs GPU compute. A command that doesn't install SGLang runs on the Databricks AI environment either way."` Dependencies []string `yaml:"dependencies" help:"pip requirement lines to install, e.g. einops==0.8.0, at most 1,000."` } -// deployCodeSourceConfig is the `code_source` block: the local code of model.entry. -type deployCodeSourceConfig struct { - RootPath string `yaml:"root_path" help:"Directory holding the model's code. A relative path resolves against the YAML file's directory." required:"yes"` - IncludePaths []string `yaml:"include_paths" help:"Files and directories to send, relative to root_path and without \"..\", e.g. [agent.py, tools]." required:"yes"` +// deployComputeConfig is the `compute` block. A command or registered model reads +// the serving fields, and a dedicated checkpoint the dedicated fields. +type deployComputeConfig struct { + AcceleratorType *string `yaml:"accelerator_type" help:"Accelerator to serve on, as in air run: GPU_1xA10, GPU_1xH100, GPU_8xH100 or GPU_1xRTX6000. Each takes the num_accelerators listed for it: GPU_1xA10 takes 1, 4 or 8; GPU_1xH100 takes 1; GPU_8xH100 takes 8; GPU_1xRTX6000 takes 1." required:"when compute is set" group:"Serving compute"` + NumAccelerators *int `yaml:"num_accelerators" help:"Number of accelerators, from the pairs listed under accelerator_type." required:"when compute is set" group:"Serving compute"` + WorkloadSize *string `yaml:"workload_size" help:"How far the endpoint scales: Small, Medium or Large." required:"when compute is set" group:"Serving compute"` + + ComputeProfile *string `yaml:"compute_profile" help:"Dedicated compute profile: low_latency, fast, throughput or minimal_cost, case-insensitive." required:"when checkpoint is set without command" group:"Dedicated compute"` + ProvisionedModelUnits *int `yaml:"provisioned_model_units" help:"Model units to provision, above 0." required:"when checkpoint is set without command" group:"Dedicated compute"` + ReservationTerm *string `yaml:"reservation_term" help:"Commitment term of the provisioned units, such as ONE_MONTH. Not supported yet: dedicated endpoints don't accept a commitment term, so a file that sets it is refused before anything is registered." group:"Dedicated compute"` } -// deployComputeConfig is the `compute` block: the served entity's workload. -type deployComputeConfig struct { - WorkloadType string `yaml:"workload_type" help:"Serving compute, e.g. CPU, GPU_SMALL, GPU_MEDIUM or GPU_LARGE. A checkpoint is served on a GPU workload type." required:"yes"` - WorkloadSize string `yaml:"workload_size" help:"How far the endpoint scales: Small, Medium or Large." required:"yes"` - ScaleToZeroEnabled bool `yaml:"scale_to_zero_enabled" help:"Whether the endpoint scales to zero when idle (default: false)."` +// variantField is a field that only some variants use. +type variantField struct { + name string + set bool + users []string + // applies says what the field is for, as the error that rejects it names it. + applies string +} + +// variantFields lists the optional fields with the variants that use them, in the +// order their errors are reported. +func (c *deployConfig) variantFields() []variantField { + var compute deployComputeConfig + if c.Compute != nil { + compute = *c.Compute + } + servingVariants := []string{variantCommand, variantRegistered} + registeringVariants := []string{variantCommand, variantDedicated} + return []variantField{ + {"command", c.Command != nil, []string{variantCommand}, "a checkpoint served by command"}, + {"task", c.Task != nil, []string{variantCommand}, "a checkpoint served by command"}, + {"environment", c.Environment != nil, []string{variantCommand}, "a checkpoint served by command"}, + {"model_version", c.ModelVersion != nil, []string{variantRegistered}, "a registered model"}, + {"env_variables", c.EnvVariables != nil, servingVariants, "a command or registered model"}, + {"secrets", c.Secrets != nil, servingVariants, "a command or registered model"}, + {"idempotency_token", c.IdempotencyToken != nil, registeringVariants, "a checkpoint, which air deploy registers"}, + {"mlflow_experiment_path", c.MLflowExperimentPath != nil, registeringVariants, "a checkpoint, which air deploy registers"}, + {"compute.accelerator_type", compute.AcceleratorType != nil, servingVariants, "a command or registered model"}, + {"compute.num_accelerators", compute.NumAccelerators != nil, servingVariants, "a command or registered model"}, + {"compute.workload_size", compute.WorkloadSize != nil, servingVariants, "a command or registered model"}, + {"compute.compute_profile", compute.ComputeProfile != nil, []string{variantDedicated}, "a dedicated checkpoint"}, + {"compute.provisioned_model_units", compute.ProvisionedModelUnits != nil, []string{variantDedicated}, "a dedicated checkpoint"}, + {"compute.reservation_term", compute.ReservationTerm != nil, []string{variantDedicated}, "a dedicated checkpoint"}, + } } -// sourceFields lists, per model.type, the optional fields that apply to it, in -// addition to the required ones in requiredSourceFields. Every other source field -// is rejected, so a field the source can't use is never silently dropped. -var sourceFields = map[string][]string{ - modelTypeCheckpoint: {"model.entry", "model.model_config", "model.streamable", "task", "command", "code_source", "environment", "idempotency_token", "mlflow_experiment_path"}, - modelTypeRegistered: {}, +// variant is the deployment the file describes, inferred from the fields it sets: a +// checkpoint with a command is served by that command, a checkpoint without one is +// served with dedicated inference, and no checkpoint means a registered version. +func (c *deployConfig) variant() string { + switch { + case c.Checkpoint == nil: + return variantRegistered + case c.Command != nil: + return variantCommand + default: + return variantDedicated + } } -// requiredSourceFields lists the fields each model.type requires. -var requiredSourceFields = map[string][]string{ - modelTypeCheckpoint: {"model.path"}, - modelTypeRegistered: {"model.version"}, +// readAs says how the file was read and why, for errors about its fields. +func (c *deployConfig) readAs() string { + v := c.variant() + return fmt.Sprintf("the file is read as %s, because %s", variantDescriptions[v], variantReasons[v]) +} + +// registers reports whether air deploy registers a new version from the checkpoint. +func (c *deployConfig) registers() bool { + return c.Checkpoint != nil } // validate runs structural validation over the whole config, returning the // first failure. Fields are checked in declaration order to keep errors stable. func (c *deployConfig) validate() error { if !registeredModelRe().MatchString(c.RegisteredModel) { - return fmt.Errorf("registered_model must be a Unity Catalog model name as .., got %q", c.RegisteredModel) - } - if c.Model == nil { - return errors.New("model: section is required") + return fmt.Errorf("registered_model_name must be a Unity Catalog model name as .., got %q", c.RegisteredModel) } if err := c.validateSource(); err != nil { return err } if !endpointNameRe().MatchString(c.Endpoint) { - return fmt.Errorf("endpoint must be a serving endpoint name of letters, digits, '-' and '_', got %q", c.Endpoint) - } - if c.Compute == nil { - return errors.New("compute: section is required") + return fmt.Errorf("endpoint_name must be a serving endpoint name of letters, digits, '-' and '_', got %q", c.Endpoint) } - if workloadTypes := servingWorkloadTypes(); !slices.Contains(workloadTypes, c.Compute.WorkloadType) { - return fmt.Errorf("compute.workload_type must be one of %s, got %q", strings.Join(workloadTypes, ", "), c.Compute.WorkloadType) - } - if !slices.Contains(workloadSizes, c.Compute.WorkloadSize) { - return fmt.Errorf("compute.workload_size must be one of %s, got %q", strings.Join(workloadSizes, ", "), c.Compute.WorkloadSize) - } - if c.Model.Type == modelTypeCheckpoint && !c.servesOnGPU() { - return fmt.Errorf("a checkpoint is served on GPU; set compute.workload_type to a GPU workload type such as GPU_MEDIUM, got %q", c.Compute.WorkloadType) + if err := c.validateCompute(); err != nil { + return err } if err := validateEnvironmentVariables(c.EnvVariables, c.Secrets); err != nil { return err @@ -182,128 +241,148 @@ func (c *deployConfig) validate() error { return nil } -// validateSource checks model.type and the fields that describe the source. +// validateSource checks that the fields the file sets fit its variant, and that the +// variant's own fields are valid. func (c *deployConfig) validateSource() error { - m := c.Model - if !slices.Contains(modelTypes, m.Type) { - return fmt.Errorf("model.type must be one of %s, got %q", strings.Join(modelTypes, ", "), m.Type) - } - - set := map[string]bool{ - "model.path": m.Path != nil, - "model.entry": m.Entry != nil, - "model.model_config": m.ModelConfig != nil, - "model.streamable": m.Streamable != nil, - "model.version": m.Version != nil, - "task": c.Task != nil, - "command": c.Command != nil, - "environment": c.Environment != nil, - "code_source": c.CodeSource != nil, - "idempotency_token": c.IdempotencyToken != nil, - "mlflow_experiment_path": c.MLflowExperimentPath != nil, - } - required := requiredSourceFields[m.Type] - for _, name := range required { - if !set[name] { - return fmt.Errorf("%s is required when model.type is %s", name, m.Type) + if err := c.checkVariantFields(); err != nil { + return err + } + if c.Checkpoint != nil { + if err := validateVolumeDirectory("checkpoint", *c.Checkpoint); err != nil { + return err } } - for _, name := range slices.Sorted(maps.Keys(set)) { - if set[name] && !slices.Contains(required, name) && !slices.Contains(sourceFields[m.Type], name) { - return fmt.Errorf("%s can't be set when model.type is %s", name, m.Type) + switch c.variant() { + case variantCommand: + return c.validateCommandSource() + case variantRegistered: + if c.ModelVersion != nil && !modelVersionRe().MatchString(*c.ModelVersion) { + return fmt.Errorf("model_version must be a model version number such as 3, got %q", *c.ModelVersion) } } + return nil +} - if m.Type == modelTypeCheckpoint { - if err := c.validateCheckpoint(); err != nil { - return err +// checkVariantFields rejects a field the file's variant doesn't use, naming the +// field, the variant and why the file is read as that variant. +func (c *deployConfig) checkVariantFields() error { + variant := c.variant() + for _, f := range c.variantFields() { + if f.set && !slices.Contains(f.users, variant) { + return fmt.Errorf("%s can't be set: %s, and %s applies only to %s", f.name, c.readAs(), f.name, f.applies) } } - if m.Path != nil { - if err := validateVolumeDirectory("model.path", *m.Path); err != nil { - return err - } + return nil +} + +// validateCommandSource checks the fields only a checkpoint served by command has. +func (c *deployConfig) validateCommandSource() error { + if c.Task == nil { + return errors.New("task is required for a checkpoint served by command: the task Model Serving routes to the command, such as llm/v1/chat") } - if c.CodeSource != nil && m.Entry == nil { - return errors.New("code_source can't be set without model.entry, the file in it that builds the model") + if strings.TrimSpace(*c.Task) == "" { + return errors.New("task cannot be empty") } - if m.Entry != nil && !entryFileRe().MatchString(*m.Entry) { - return fmt.Errorf("model.entry must name a .py file directly under code_source.root_path, e.g. agent.py, got %q", *m.Entry) + if strings.TrimSpace(*c.Command) == "" { + return errors.New("command cannot be empty") } - if m.Version != nil && !modelVersionRe().MatchString(*m.Version) { - return fmt.Errorf("model.version must be a model version number such as 3, got %q", *m.Version) + if n := serverLength(*c.Command); n > maxServingCommandChars { + return fmt.Errorf("command must be at most %d characters, got %d", maxServingCommandChars, n) } - if c.Task != nil && strings.TrimSpace(*c.Task) == "" { - return errors.New("task cannot be empty") + if c.Environment == nil { + return errors.New("environment is required for a checkpoint served by command: the serverless environment and packages to serve it on") } - if c.Command != nil { - if err := validateCommand(*c.Command); err != nil { - return err - } - if n := serverLength(*c.Command); n > maxServingCommandChars { - return fmt.Errorf("command must be at most %d characters, got %d", maxServingCommandChars, n) - } + return c.Environment.validate() +} + +// validateCompute checks the compute the variant reads. +func (c *deployConfig) validateCompute() error { + if c.variant() == variantDedicated { + _, err := c.resolveDedicatedCompute() + return err } - if c.Environment != nil { - if err := c.Environment.validate(); err != nil { - return err + _, _, err := c.resolveServingCompute() + return err +} + +// resolveServingCompute returns the workload type and size a command or registered +// model is served on. Without compute, the defaults apply; with it, all three fields +// are required and the accelerator and count must be a pair the server offers. +func (c *deployConfig) resolveServingCompute() (string, string, error) { + if c.Compute == nil { + if c.variant() == variantRegistered { + return defaultRegisteredWorkloadType, defaultWorkloadSize, nil } + return defaultCommandWorkloadType, defaultWorkloadSize, nil } - if c.CodeSource != nil { - // The files are stored in the model version for anyone who can read the - // model, so the user names them rather than sending a whole directory. - if c.CodeSource.IncludePaths == nil { - return errors.New("code_source.include_paths is required: list the files and directories to send, e.g. include_paths: [agent.py, tools]") - } - // git reads a leading ':' as pathspec magic, such as ":!x" for everything - // but x, and expands globs, either of which would send files the list - // doesn't name; padding would match nothing. - for _, p := range c.CodeSource.IncludePaths { - if strings.HasPrefix(p, ":") || strings.ContainsAny(p, "*?[") || p != strings.TrimSpace(p) { - return fmt.Errorf("code_source.include_paths must name files and directories, without patterns or surrounding spaces, got %q", p) - } - } - snapshot := snapshotSourceConfig{RootPath: c.CodeSource.RootPath, IncludePaths: c.CodeSource.IncludePaths} - if err := snapshot.validate("code_source"); err != nil { - return err + compute := c.Compute + switch { + case compute.AcceleratorType == nil: + return "", "", errors.New("compute.accelerator_type is required when compute is set, e.g. GPU_1xA10") + case compute.NumAccelerators == nil: + return "", "", errors.New("compute.num_accelerators is required when compute is set, e.g. 1") + case compute.WorkloadSize == nil: + return "", "", errors.New("compute.workload_size is required when compute is set: Small, Medium or Large") + } + if !slices.Contains(workloadSizes, *compute.WorkloadSize) { + return "", "", fmt.Errorf("compute.workload_size must be one of %s, got %q", strings.Join(workloadSizes, ", "), *compute.WorkloadSize) + } + accelerator, err := parseGPUType(*compute.AcceleratorType) + if err != nil { + return "", "", fmt.Errorf("compute.accelerator_type: %w; the supported pairs are %s", err, servingPairs()) + } + for _, shape := range servingShapes { + if shape.accelerator == accelerator && shape.count == *compute.NumAccelerators { + return shape.workloadType, *compute.WorkloadSize, nil } } - return nil + return "", "", fmt.Errorf("compute.accelerator_type and compute.num_accelerators must be a supported pair, got %s with %d; the supported pairs are %s", *compute.AcceleratorType, *compute.NumAccelerators, servingPairs()) } -// validateCheckpoint checks the fields of a checkpoint: weights to copy, and -// exactly one way to serve them. -func (c *deployConfig) validateCheckpoint() error { - m := c.Model - if (c.Command != nil) == (m.Entry != nil) { - return errors.New("exactly one of command and model.entry is required when model.type is checkpoint: a command that serves the weights, or a Python file Model Serving's model server loads") +// servingPairs lists the accelerator and count pairs serving compute supports. +func servingPairs() string { + pairs := make([]string, 0, len(servingShapes)) + for _, shape := range servingShapes { + pairs = append(pairs, fmt.Sprintf("%s x%d", shape.accelerator, shape.count)) } - if c.Command != nil && c.Task == nil { - return errors.New("task is required when command is set: the task Model Serving routes to the command, such as llm/v1/chat") + return strings.Join(pairs, ", ") +} + +// dedicatedCompute is the dedicated compute a dedicated served entity gets. +type dedicatedCompute struct { + ComputeProfile string + ProvisionedModelUnits int64 +} + +// resolveDedicatedCompute returns the dedicated compute of a dedicated checkpoint. The +// server has no default for it, so compute, its profile and its units are required. +func (c *deployConfig) resolveDedicatedCompute() (*dedicatedCompute, error) { + if c.Compute == nil { + return nil, errors.New("compute is required for a dedicated checkpoint: set compute.compute_profile and compute.provisioned_model_units") } - if m.Entry != nil && c.CodeSource == nil { - return errors.New("code_source is required when model.entry is set: the directory that holds the file") + compute := c.Compute + if compute.ComputeProfile == nil { + return nil, errors.New("compute.compute_profile is required for a dedicated checkpoint: one of low_latency, fast, throughput or minimal_cost") } - if m.Entry == nil && (m.ModelConfig != nil || m.Streamable != nil) { - return errors.New("model.model_config and model.streamable apply to model.entry, not to command") + profile := strings.ToLower(*compute.ComputeProfile) + if !slices.Contains(computeProfiles, profile) { + return nil, fmt.Errorf("compute.compute_profile must be one of %s, got %q", strings.Join(computeProfiles, ", "), *compute.ComputeProfile) } - if m.ModelConfig != nil { - if _, err := modelConfigJSON(m.ModelConfig); err != nil { - return err - } + if compute.ProvisionedModelUnits == nil { + return nil, errors.New("compute.provisioned_model_units is required for a dedicated checkpoint: the model units to provision, above 0") } - return nil -} - -// modelConfigJSON encodes model_config as the JSON object the server reads. -// YAML can hold numbers JSON can't, .nan and .inf, and those are rejected rather -// than dropped. -func modelConfigJSON(config map[string]any) (string, error) { - raw, err := json.Marshal(config) - if err != nil { - return "", fmt.Errorf("model.model_config must hold only values JSON can, so no .nan or .inf: %w", err) + if *compute.ProvisionedModelUnits <= 0 { + return nil, fmt.Errorf("compute.provisioned_model_units must be above 0, got %d", *compute.ProvisionedModelUnits) + } + // The server refuses a commitment term for a dedicated endpoint, which it would + // only do after the weights are copied. + if compute.ReservationTerm != nil { + return nil, errors.New("compute.reservation_term isn't supported yet: dedicated endpoints don't accept a commitment term; leave it out") } - return string(raw), nil + return &dedicatedCompute{ + ComputeProfile: "COMPUTE_PROFILE_" + strings.ToUpper(profile), + ProvisionedModelUnits: int64(*compute.ProvisionedModelUnits), + }, nil } // validateVolumeDirectory checks a Unity Catalog volume directory the server @@ -317,10 +396,11 @@ func validateVolumeDirectory(field, v string) error { // validate checks the environment version and dependency lines. func (e *deployEnvironmentConfig) validate() error { - if e.Version != nil { - if _, _, err := e.runtimeVersion(); err != nil { - return err - } + if e.Version == nil { + return errors.New("environment.version is required: the serverless environment to serve the command on, such as 6 or databricks_ai_v6") + } + if _, _, err := e.runtimeVersion(); err != nil { + return err } if len(e.Dependencies) > maxDependencies { return fmt.Errorf("environment.dependencies can list at most %d packages, got %d", maxDependencies, len(e.Dependencies)) @@ -352,33 +432,11 @@ func serverLength(s string) int { return n } -// servingWorkloadTypes lists the served entity workload types, sorted. -func servingWorkloadTypes() []string { - var workloadType serving.ServingModelWorkloadType - var types []string - for _, t := range workloadType.Values() { - types = append(types, string(t)) - } - slices.Sort(types) - return types -} - -// registers reports whether air deploy registers a new version, rather than -// serving an existing one. -func (c *deployConfig) registers() bool { - return c.Model.Type != modelTypeRegistered -} - -// servesOnGPU reports whether compute.workload_type is a GPU workload type. -func (c *deployConfig) servesOnGPU() bool { - return !strings.HasPrefix(c.Compute.WorkloadType, cpuWorkloadTypePrefix) -} - // runtimeVersion is environment.version as a version number, and whether it names // the Databricks AI environment, as air run reads the same field. func (e *deployEnvironmentConfig) runtimeVersion() (int, bool, error) { - if e == nil || e.Version == nil { - return defaultServingEnvironmentVersion, false, nil + if e.Version == nil { + return 0, false, errors.New("environment.version is required") } normalized, err := validateRuntimeVersion(e.Version.raw, "environment.version") if err != nil { @@ -392,30 +450,6 @@ func (e *deployEnvironmentConfig) runtimeVersion() (int, bool, error) { return major, ai, nil } -// servingRuntime is the serverless runtime the model is served on, as the server -// names it: client., with a -gpu suffix on GPU compute. -func (c *deployConfig) servingRuntime() string { - version, _, _ := c.Environment.runtimeVersion() - runtime := "client." + strconv.Itoa(version) - if c.servesOnGPU() { - runtime += "-gpu" - } - return runtime -} - -// servingEnvironment is the registration's runtime, dependencies and base -// environment. -func (c *deployConfig) servingEnvironment() modelPackageEnvironment { - environment := modelPackageEnvironment{Runtime: c.servingRuntime()} - if _, ai, _ := c.Environment.runtimeVersion(); ai { - environment.BaseEnvironment = baseEnvironmentDatabricksAI - } - if c.Environment != nil { - environment.Dependencies = c.Environment.Dependencies - } - return environment -} - // servedEnvironmentVars merges env_variables and secrets into the served // entity's environment variables, with each secret as {{secrets/scope/key}}. func (c *deployConfig) servedEnvironmentVars() map[string]string { @@ -439,35 +473,34 @@ func deploySchema() configField { path: configHelpRoot, help: `Deployment YAML schema. For details on a field, pass its path to -h, for example: - databricks air deploy -h config.model.type + databricks air deploy -h config.checkpoint - Minimal example, a checkpoint served by vLLM: - registered_model: main.models.qwen - model: - type: checkpoint - path: /Volumes/main/models/weights/qwen3_8b + A checkpoint served by a command of your own: + registered_model_name: main.serving.qwen30b_sglang + endpoint_name: qwen3-30b-sglang + checkpoint: /Volumes/main/weights/qwen3_30b_a3b + command: python -m sglang.launch_server --model-path qwen3_30b_a3b --attention-backend triton --sampling-backend pytorch --host 0.0.0.0 --port 8080 task: llm/v1/chat - command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --served-model-name base --host 0.0.0.0 --port 8080 - endpoint: qwen-chat + environment: + version: 6 + dependencies: [sglang==0.5.23] compute: - workload_type: GPU_MEDIUM + accelerator_type: GPU_1xH100 + num_accelerators: 1 workload_size: Small - A checkpoint served by Python code, model.py, which calls mlflow.models.set_model: - registered_model: main.models.whisper - model: - type: checkpoint - path: /Volumes/main/models/weights/whisper - entry: model.py - code_source: - root_path: . - include_paths: [model.py] - environment: - version: databricks_ai_v6 - endpoint: whisper + A checkpoint served with dedicated inference: + registered_model_name: main.serving.gemma4 + endpoint_name: gemma4 + checkpoint: /Volumes/main/weights/gemma4_e4b compute: - workload_type: GPU_MEDIUM - workload_size: Small`, + compute_profile: low_latency + provisioned_model_units: 600 + + A version already registered in Unity Catalog: + registered_model_name: main.ml.churn + endpoint_name: churn + model_version: 3`, children: describeStruct(reflect.TypeFor[deployConfig](), configHelpRoot), } } diff --git a/cmd/air/deployconfig_test.go b/cmd/air/deployconfig_test.go index 8ea4542f99b..c147a9f8920 100644 --- a/cmd/air/deployconfig_test.go +++ b/cmd/air/deployconfig_test.go @@ -8,17 +8,18 @@ import ( "github.com/stretchr/testify/require" ) -const checkpointDeployConfig = `registered_model: main.models.qwen -model: - type: checkpoint - path: /Volumes/main/models/weights/qwen3_8b -task: llm/v1/chat +// commandDeployConfig is a checkpoint served by a command. +const commandDeployConfig = `registered_model_name: main.models.qwen +checkpoint: /Volumes/main/models/weights/qwen3_8b command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080 +task: llm/v1/chat environment: + version: 6 dependencies: [einops==0.8.0] -endpoint: qwen-chat +endpoint_name: qwen-chat compute: - workload_type: GPU_MEDIUM + accelerator_type: GPU_1xA10 + num_accelerators: 1 workload_size: Small secrets: HF_TOKEN: llm/hf_token @@ -26,24 +27,19 @@ env_variables: LOG_LEVEL: info ` -// entryDeployConfig is a checkpoint served by Python code, model.entry. -const entryDeployConfig = `registered_model: main.agents.support -model: - type: checkpoint - path: /Volumes/main/models/weights/support - entry: agent.py -task: agent/v1/responses -code_source: - root_path: . - include_paths: [agent.py, tools] -environment: - version: 5 - dependencies: [mlflow==3.5.0] -endpoint: support-agent +// dedicatedDeployConfig is a checkpoint served with dedicated inference. +const dedicatedDeployConfig = `registered_model_name: main.models.qwen +checkpoint: /Volumes/main/models/weights/qwen3_8b +endpoint_name: qwen-chat compute: - workload_type: GPU_SMALL - workload_size: Small - scale_to_zero_enabled: true + compute_profile: low_latency + provisioned_model_units: 600 +` + +// registeredDeployConfig serves a version that is already registered. +const registeredDeployConfig = `registered_model_name: main.models.qwen +model_version: 7 +endpoint_name: qwen-chat ` func loadDeployConfigText(t *testing.T, text string, overrides ...string) (*deployConfig, error) { @@ -51,16 +47,30 @@ func loadDeployConfigText(t *testing.T, text string, overrides ...string) (*depl return loadDeployConfig(t.Context(), writeConfigFile(t, "deploy.yaml", text), overrides) } -func TestLoadDeployConfigCheckpoint(t *testing.T) { - cfg, err := loadDeployConfigText(t, checkpointDeployConfig) - require.NoError(t, err) - assert.True(t, cfg.registers()) - assert.Equal(t, "client.6-gpu", cfg.servingRuntime()) - assert.Equal(t, modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}}, cfg.servingEnvironment()) - assert.Equal(t, map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, cfg.servedEnvironmentVars()) +func TestLoadDeployConfigVariantIsInferredFromTheFields(t *testing.T) { + for _, tt := range []struct { + name string + text string + variant string + }{ + {"checkpoint and command", commandDeployConfig, variantCommand}, + {"checkpoint without command", dedicatedDeployConfig, variantDedicated}, + {"no checkpoint", registeredDeployConfig, variantRegistered}, + } { + t.Run(tt.name, func(t *testing.T) { + cfg, err := loadDeployConfigText(t, tt.text) + require.NoError(t, err) + assert.Equal(t, tt.variant, cfg.variant()) + assert.Equal(t, tt.variant != variantRegistered, cfg.registers()) + }) + } +} - req, err := cfg.registrationRequest("key-1", nil) +func TestLoadDeployConfigCommand(t *testing.T) { + cfg, err := loadDeployConfigText(t, commandDeployConfig) require.NoError(t, err) + + req := cfg.registrationRequest("key-1") assert.Equal(t, registerModelPackageRequest{ RegisteredModel: "main.models.qwen", IdempotencyKey: "key-1", @@ -71,158 +81,263 @@ func TestLoadDeployConfigCheckpoint(t *testing.T) { Environment: modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}}, }, }, req) -} -func TestLoadDeployConfigAudioCheckpoint(t *testing.T) { - for _, task := range []string{taskAudioTranscriptions, taskAudioTranslations} { - t.Run(task, func(t *testing.T) { - cfg, err := loadDeployConfigText(t, strings.Replace(checkpointDeployConfig, "task: llm/v1/chat", "task: "+task, 1)) - require.NoError(t, err) - req, err := cfg.registrationRequest("key-1", nil) - require.NoError(t, err) - assert.Equal(t, task, req.Checkpoint.Task) - }) - } + entity, err := cfg.servedEntity("3") + require.NoError(t, err) + assert.Equal(t, servedEntity{ + EntityName: "main.models.qwen", + EntityVersion: "3", + WorkloadType: "GPU_MEDIUM", + WorkloadSize: "Small", + EnvironmentVars: map[string]string{"LOG_LEVEL": "info", "HF_TOKEN": "{{secrets/llm/hf_token}}"}, + }, entity) } -const pythonCheckpointDeployConfig = `registered_model: main.ml.whisper -model: - type: checkpoint - path: /Volumes/main/weights/whisper - entry: model.py - model_config: {language: en, beams: 4} - streamable: false -task: llm/v1/chat -code_source: - root_path: . - include_paths: [model.py] -environment: - version: databricks_ai_v6 - dependencies: [soundfile==0.13.1] -endpoint: whisper -compute: - workload_type: GPU_MEDIUM - workload_size: Small -` - -func TestLoadDeployConfigPythonCheckpoint(t *testing.T) { - cfg, err := loadDeployConfigText(t, pythonCheckpointDeployConfig) +func TestLoadDeployConfigCommandOnDatabricksAI(t *testing.T) { + cfg, err := loadDeployConfigText(t, strings.Replace(commandDeployConfig, " version: 6\n", " version: databricks_ai_v6\n", 1)) require.NoError(t, err) - assert.Equal(t, "client.6-gpu", cfg.servingRuntime()) + req := cfg.registrationRequest("key-1") + assert.Equal(t, modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"einops==0.8.0"}, BaseEnvironment: "databricks-ai"}, req.Checkpoint.Environment) +} - streamable := false - code := &modelPackagePythonModel{EntryFile: "model.py", EntrySource: "import mlflow\n", Files: []modelPackageFile{{Path: "util/a.py", Content: "x = 1\n"}}} - req, err := cfg.registrationRequest("key-1", code) +func TestLoadDeployConfigDedicated(t *testing.T) { + cfg, err := loadDeployConfigText(t, dedicatedDeployConfig+"mlflow_experiment_path: /Shared/qwen\n") require.NoError(t, err) + + req := cfg.registrationRequest("key-1") assert.Equal(t, registerModelPackageRequest{ - RegisteredModel: "main.ml.whisper", - IdempotencyKey: "key-1", - Checkpoint: &modelPackageCheckpoint{ - VolumePath: "/Volumes/main/weights/whisper", - PythonModel: &modelPackagePythonModel{ - EntryFile: "model.py", - EntrySource: "import mlflow\n", - Files: []modelPackageFile{{Path: "util/a.py", Content: "x = 1\n"}}, - ModelConfig: `{"beams":4,"language":"en"}`, - Streamable: &streamable, - }, - Task: "llm/v1/chat", - Environment: modelPackageEnvironment{Runtime: "client.6-gpu", Dependencies: []string{"soundfile==0.13.1"}, BaseEnvironment: "databricks-ai"}, - }, + RegisteredModel: "main.models.qwen", + IdempotencyKey: "key-1", + DedicatedCheckpoint: &dedicatedCheckpoint{VolumePath: "/Volumes/main/models/weights/qwen3_8b"}, + ExperimentPath: "/Shared/qwen", }, req) -} -func TestLoadDeployConfigPythonCheckpointWithoutTask(t *testing.T) { - text := strings.Replace(pythonCheckpointDeployConfig, "task: llm/v1/chat\n", "", 1) - text = strings.Replace(text, " model_config: {language: en, beams: 4}\n streamable: false\n", "", 1) - cfg, err := loadDeployConfigText(t, text) + dedicated, err := cfg.resolveDedicatedCompute() require.NoError(t, err) - req, err := cfg.registrationRequest("k", &modelPackagePythonModel{EntryFile: "model.py"}) - require.NoError(t, err) - assert.Empty(t, req.Checkpoint.Task) - assert.Nil(t, req.Checkpoint.PythonModel.Streamable) - assert.Empty(t, req.Checkpoint.PythonModel.ModelConfig) + assert.Equal(t, &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_LOW_LATENCY", ProvisionedModelUnits: 600}, dedicated) } -func TestLoadDeployConfigEntryCheckpointServedEntity(t *testing.T) { - cfg, err := loadDeployConfigText(t, entryDeployConfig) +func TestLoadDeployConfigRegisteredDefaultsToCPU(t *testing.T) { + cfg, err := loadDeployConfigText(t, registeredDeployConfig) require.NoError(t, err) - assert.Equal(t, "client.5-gpu", cfg.servingRuntime()) - assert.Nil(t, cfg.servedEnvironmentVars()) - - entity := cfg.servedEntity("4") - assert.Equal(t, "main.agents.support", entity.EntityName) - assert.Equal(t, "4", entity.EntityVersion) - assert.True(t, entity.ScaleToZeroEnabled) - assert.Contains(t, entity.ForceSendFields, "ScaleToZeroEnabled") -} + assert.False(t, cfg.registers()) -func TestLoadDeployConfigRegisteredVersion(t *testing.T) { - cfg, err := loadDeployConfigText(t, `registered_model: main.ml.churn -model: {type: registered, version: 7} -endpoint: churn -compute: {workload_type: CPU, workload_size: Small} -`) + entity, err := cfg.servedEntity("7") require.NoError(t, err) - assert.False(t, cfg.registers()) - assert.Equal(t, "7", *cfg.Model.Version) + assert.Equal(t, "CPU", entity.WorkloadType) + assert.Equal(t, "Small", entity.WorkloadSize) + assert.Nil(t, entity.EnvironmentVars) } -func TestLoadDeployConfigRejectsUnsupportedModelTypes(t *testing.T) { - for _, modelType := range []string{"code", "saved_model_dir"} { - t.Run(modelType, func(t *testing.T) { - _, err := loadDeployConfigText(t, `registered_model: main.ml.churn -model: {type: `+modelType+`, path: /Volumes/main/ml/runs/model} -endpoint: churn -compute: {workload_type: GPU_SMALL, workload_size: Small} -`) - assert.ErrorContains(t, err, `model.type must be one of checkpoint, registered, got "`+modelType+`"`) +func TestDeployConfigCrossVariantFields(t *testing.T) { + tests := []struct { + name string + config string + wantErr string + }{ + { + "task without a checkpoint", + registeredDeployConfig + "task: llm/v1/chat\n", + "task can't be set: the file is read as a registered model, because it sets no checkpoint, and task applies only to a checkpoint served by command", + }, + { + "command without a checkpoint", + registeredDeployConfig + "command: python serve.py\n", + "command can't be set: the file is read as a registered model, because it sets no checkpoint, and command applies only to a checkpoint served by command", + }, + { + "environment on a dedicated checkpoint", + dedicatedDeployConfig + "environment: {version: 6}\n", + "environment can't be set: the file is read as a dedicated checkpoint, because it sets checkpoint but not command, and environment applies only to a checkpoint served by command", + }, + { + "model_version with a checkpoint", + commandDeployConfig + "model_version: 3\n", + "model_version can't be set: the file is read as a checkpoint served by command, because it sets checkpoint and command, and model_version applies only to a registered model", + }, + { + "model_version on a dedicated checkpoint", + dedicatedDeployConfig + "model_version: 3\n", + "model_version can't be set: the file is read as a dedicated checkpoint", + }, + { + "secrets on a dedicated checkpoint", + dedicatedDeployConfig + "secrets: {HF_TOKEN: llm/hf_token}\n", + "secrets can't be set: the file is read as a dedicated checkpoint, because it sets checkpoint but not command, and secrets applies only to a command or registered model", + }, + { + "env_variables on a dedicated checkpoint", + dedicatedDeployConfig + "env_variables: {LOG_LEVEL: info}\n", + "env_variables can't be set: the file is read as a dedicated checkpoint", + }, + { + "idempotency_token without a checkpoint", + registeredDeployConfig + "idempotency_token: key-1\n", + "idempotency_token can't be set: the file is read as a registered model, because it sets no checkpoint, and idempotency_token applies only to a checkpoint, which air deploy registers", + }, + { + "mlflow_experiment_path without a checkpoint", + registeredDeployConfig + "mlflow_experiment_path: /Shared/qwen\n", + "mlflow_experiment_path can't be set: the file is read as a registered model", + }, + { + "serving compute on a dedicated checkpoint", + strings.Replace(dedicatedDeployConfig, " compute_profile: low_latency\n", " compute_profile: low_latency\n accelerator_type: GPU_1xA10\n", 1), + "compute.accelerator_type can't be set: the file is read as a dedicated checkpoint, because it sets checkpoint but not command, and compute.accelerator_type applies only to a command or registered model", + }, + { + "dedicated compute on a command", + strings.Replace(commandDeployConfig, " workload_size: Small\n", " workload_size: Small\n compute_profile: low_latency\n", 1), + "compute.compute_profile can't be set: the file is read as a checkpoint served by command, because it sets checkpoint and command, and compute.compute_profile applies only to a dedicated checkpoint", + }, + { + "reservation term on a command", + strings.Replace(commandDeployConfig, " workload_size: Small\n", " workload_size: Small\n reservation_term: ONE_MONTH\n", 1), + "compute.reservation_term can't be set: the file is read as a checkpoint served by command", + }, + { + "dedicated compute on a registered model", + registeredDeployConfig + "compute: {workload_size: Small, provisioned_model_units: 10}\n", + "compute.provisioned_model_units can't be set: the file is read as a registered model", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, tt.config) + assert.ErrorContains(t, err, tt.wantErr) }) } } -func TestLoadDeployConfigExperimentPath(t *testing.T) { - cfg, err := loadDeployConfigText(t, checkpointDeployConfig+"mlflow_experiment_path: /Shared/qwen\n") - require.NoError(t, err) - req, err := cfg.registrationRequest("k", nil) - require.NoError(t, err) - assert.Equal(t, "/Shared/qwen", req.ExperimentPath) +func TestDeployConfigRequiredFieldsPerVariant(t *testing.T) { + tests := []struct { + name string + config string + wantErr string + }{ + {"command without a task", dropLine("task:")(commandDeployConfig), "task is required for a checkpoint served by command"}, + {"command without an environment", dropBlock("environment:")(commandDeployConfig), "environment is required for a checkpoint served by command"}, + {"dedicated without compute", dropBlock("compute:")(dedicatedDeployConfig), "compute is required for a dedicated checkpoint"}, + {"dedicated without a profile", dropLine(" compute_profile:")(dedicatedDeployConfig), "compute.compute_profile is required for a dedicated checkpoint"}, + {"dedicated without units", dropLine(" provisioned_model_units:")(dedicatedDeployConfig), "compute.provisioned_model_units is required for a dedicated checkpoint"}, + {"command compute without num_accelerators", dropLine(" num_accelerators:")(commandDeployConfig), "compute.num_accelerators is required when compute is set"}, + {"registered compute without accelerator", registeredDeployConfig + "compute: {workload_size: Small}\n", "compute.accelerator_type is required when compute is set"}, + {"no endpoint", dropLine("endpoint_name:")(commandDeployConfig), `endpoint_name must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, + {"no registered model", dropLine("registered_model_name:")(commandDeployConfig), `registered_model_name must be a Unity Catalog model name as .., got ""`}, + {"registered with a bad version", replaceFirst("model_version: 7", "model_version: v7")(registeredDeployConfig), "model_version must be a model version number such as 3"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := loadDeployConfigText(t, tt.config) + assert.ErrorContains(t, err, tt.wantErr) + }) + } } -func TestLoadDeployConfigRejectsModelConfigJSONCantHold(t *testing.T) { - for name, value := range map[string]string{ - "NaN": "{ratio: .nan}", - "infinity": "{limit: .inf}", - "negative infinity": "{weights: [1, -.inf]}", +// servingCompute is a config's serving compute for one accelerator, count and size. +func servingCompute(accelerator string, count int, size string) *deployComputeConfig { + return &deployComputeConfig{AcceleratorType: &accelerator, NumAccelerators: &count, WorkloadSize: &size} +} + +func TestDeployServingComputeTable(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve")} + for _, tt := range []struct { + accelerator string + count int + workload string + }{ + {"GPU_1xA10", 1, "GPU_MEDIUM"}, + {"GPU_1xA10", 4, "MULTIGPU_MEDIUM"}, + {"GPU_1xA10", 8, "GPU_MEDIUM_8"}, + {"GPU_1xH100", 1, "GPU_XLARGE"}, + {"GPU_8xH100", 8, "GPU_XLARGE_8"}, + {"GPU_1xRTX6000", 1, "GPU_LARGE_RTX"}, } { - t.Run(name, func(t *testing.T) { - text := strings.Replace(pythonCheckpointDeployConfig, "{language: en, beams: 4}", value, 1) - _, err := loadDeployConfigText(t, text) - assert.ErrorContains(t, err, "model.model_config must hold only values JSON can, so no .nan or .inf") - }) + cfg.Compute = servingCompute(tt.accelerator, tt.count, "Medium") + workload, size, err := cfg.resolveServingCompute() + require.NoError(t, err, tt.accelerator) + assert.Equal(t, tt.workload, workload, "%s x%d", tt.accelerator, tt.count) + assert.Equal(t, "Medium", size) } } -func TestLoadDeployConfigOverride(t *testing.T) { - cfg, err := loadDeployConfigText(t, checkpointDeployConfig, "compute.workload_size=Medium", "environment.version=5") +func TestDeployServingComputeRejectsUnsupportedPairs(t *testing.T) { + const pairs = "GPU_1xA10 x1, GPU_1xA10 x4, GPU_1xA10 x8, GPU_1xH100 x1, GPU_8xH100 x8, GPU_1xRTX6000 x1" + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve")} + for _, tt := range []struct { + accelerator string + count int + }{ + {"GPU_1xA10", 2}, + {"GPU_1xH100", 8}, + {"GPU_8xH100", 1}, + {"GPU_8xB300", 8}, + {"GPU_1xRTX6000", 2}, + } { + cfg.Compute = servingCompute(tt.accelerator, tt.count, "Small") + _, _, err := cfg.resolveServingCompute() + assert.ErrorContains(t, err, "must be a supported pair, got "+tt.accelerator+" with ", tt.accelerator) + assert.ErrorContains(t, err, "the supported pairs are "+pairs, tt.accelerator) + } + + cfg.Compute = servingCompute("GPU_9xA10", 1, "Small") + _, _, err := cfg.resolveServingCompute() + assert.ErrorContains(t, err, `compute.accelerator_type: "GPU_9xA10" is not a recognized accelerator type; the supported pairs are `+pairs) +} + +func TestDeployServingComputeDefaults(t *testing.T) { + command := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve")} + workload, size, err := command.resolveServingCompute() + require.NoError(t, err) + assert.Equal(t, "GPU_MEDIUM", workload) + assert.Equal(t, "Small", size) + + registered := deployConfig{} + workload, size, err = registered.resolveServingCompute() require.NoError(t, err) - assert.Equal(t, "Medium", cfg.Compute.WorkloadSize) - assert.Equal(t, "client.5-gpu", cfg.servingRuntime()) + assert.Equal(t, "CPU", workload) + assert.Equal(t, "Small", size) } -func TestLoadDeployConfigRejectsUnknownOverride(t *testing.T) { - _, err := loadDeployConfigText(t, checkpointDeployConfig, "compute.accelerator_type=GPU_1xH100") - assert.ErrorContains(t, err, `invalid --override "compute.accelerator_type": "accelerator_type" is not a known field; available fields are: scale_to_zero_enabled, workload_size, workload_type`) +func TestDeployServingComputeRejectsBadSize(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w"), Command: new("serve"), Compute: servingCompute("GPU_1xA10", 1, "Huge")} + _, _, err := cfg.resolveServingCompute() + assert.ErrorContains(t, err, `compute.workload_size must be one of Small, Medium, Large, got "Huge"`) } -func TestLoadDeployConfigRejectsUnknownField(t *testing.T) { - _, err := loadDeployConfigText(t, checkpointDeployConfig+"experiment_name: x\n") - assert.ErrorContains(t, err, "field experiment_name not found") +func TestDeployDedicatedComputeMapping(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w")} + for profile, want := range map[string]string{ + "low_latency": "COMPUTE_PROFILE_LOW_LATENCY", + "FAST": "COMPUTE_PROFILE_FAST", + "Throughput": "COMPUTE_PROFILE_THROUGHPUT", + "minimal_cost": "COMPUTE_PROFILE_MINIMAL_COST", + } { + cfg.Compute = &deployComputeConfig{ComputeProfile: new(profile), ProvisionedModelUnits: new(600)} + dedicated, err := cfg.resolveDedicatedCompute() + require.NoError(t, err, profile) + assert.Equal(t, want, dedicated.ComputeProfile, profile) + } } -func TestLoadDeployConfigRejectsEmptyFile(t *testing.T) { - _, err := loadDeployConfigText(t, "") - assert.ErrorContains(t, err, "is empty") +func TestDeployDedicatedComputeRejects(t *testing.T) { + cfg := deployConfig{Checkpoint: new("/Volumes/a/b/c/w")} + for _, tt := range []struct { + name string + compute deployComputeConfig + wantErr string + }{ + {"unknown profile", deployComputeConfig{ComputeProfile: new("fastest"), ProvisionedModelUnits: new(600)}, `compute.compute_profile must be one of low_latency, fast, throughput, minimal_cost, got "fastest"`}, + {"zero units", deployComputeConfig{ComputeProfile: new("fast"), ProvisionedModelUnits: new(0)}, "compute.provisioned_model_units must be above 0, got 0"}, + {"negative units", deployComputeConfig{ComputeProfile: new("fast"), ProvisionedModelUnits: new(-1)}, "compute.provisioned_model_units must be above 0, got -1"}, + {"a reservation term", deployComputeConfig{ComputeProfile: new("fast"), ProvisionedModelUnits: new(600), ReservationTerm: new("ONE_MONTH")}, "compute.reservation_term isn't supported yet"}, + } { + t.Run(tt.name, func(t *testing.T) { + cfg.Compute = &tt.compute + _, err := cfg.resolveDedicatedCompute() + assert.ErrorContains(t, err, tt.wantErr) + }) + } } func TestDeployConfigValidation(t *testing.T) { @@ -231,74 +346,55 @@ func TestDeployConfigValidation(t *testing.T) { edit func(string) string wantErr string }{ - {"bad model name", replaceFirst("main.models.qwen", "qwen"), "registered_model must be a Unity Catalog model name"}, - {"backticked model name", replaceFirst("main.models.qwen", "main.models.`qwen`"), "registered_model must be a Unity Catalog model name"}, - {"unknown type", replaceFirst("type: checkpoint", "type: pickle"), `model.type must be one of checkpoint, registered, got "pickle"`}, - {"checkpoint without task", replaceFirst("task: llm/v1/chat\n", ""), "task is required when command is set"}, - {"checkpoint without command", dropLine("command:"), "exactly one of command and model.entry is required when model.type is checkpoint"}, - {"checkpoint with command and entry", replaceFirst(" type: checkpoint\n", " type: checkpoint\n entry: a.py\n"), "exactly one of command and model.entry is required when model.type is checkpoint"}, - {"checkpoint with code_source but no entry", appendText("code_source: {root_path: .}\n"), "code_source can't be set without model.entry"}, - {"checkpoint without weights", dropLine(" path:"), "model.path is required when model.type is checkpoint"}, - {"checkpoint with artifacts", replaceFirst(" type: checkpoint\n", " type: checkpoint\n artifacts: {tok: /Volumes/a/b/c/d}\n"), "field artifacts not found"}, - {"model_config with a command", replaceFirst(" type: checkpoint\n", " type: checkpoint\n model_config: {k: v}\n"), "model.model_config and model.streamable apply to model.entry, not to command"}, - {"AI environment before version 5", replaceFirst(" dependencies: [einops==0.8.0]", " version: databricks_ai_v4"), "databricks_ai_v in environment.version requires AI Runtime version 5 or higher"}, - {"bad environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: six"), "unsupported client image version"}, - {"path outside volumes", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/dbfs/weights"), "model.path must be a Unity Catalog volume directory"}, - {"path too long", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("a", 1100)), "model.path must be a Unity Catalog volume directory"}, + {"bad model name", replaceFirst("main.models.qwen", "qwen"), "registered_model_name must be a Unity Catalog model name"}, + {"backticked model name", replaceFirst("main.models.qwen", "main.models.`qwen`"), "registered_model_name must be a Unity Catalog model name"}, + {"command without task", replaceFirst("task: llm/v1/chat\n", ""), "task is required for a checkpoint served by command"}, + {"empty task", replaceFirst("task: llm/v1/chat", `task: " "`), "task cannot be empty"}, + {"path outside volumes", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/dbfs/weights"), "checkpoint must be a Unity Catalog volume directory"}, + {"path too long", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("a", 1100)), "checkpoint must be a Unity Catalog volume directory"}, + {"path too long in characters", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1020)), "checkpoint must be a Unity Catalog volume directory"}, {"empty command", replaceFirst("command: python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", `command: " "`), "command cannot be empty"}, {"command too long", replaceFirst("command: python", "command: python "+strings.Repeat("x", maxServingCommandChars)), "command must be at most 32768 characters"}, - {"path too long in characters", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1020)), "model.path must be a Unity Catalog volume directory"}, - {"checkpoint on CPU", replaceFirst("GPU_MEDIUM", "CPU"), "a checkpoint is served on GPU"}, - {"unknown workload type", replaceFirst("GPU_MEDIUM", "GPU_MEDUIM"), `compute.workload_type must be one of CPU, CPU_LARGE, CPU_MEDIUM, GPU_LARGE, GPU_MEDIUM, GPU_SMALL, GPU_XLARGE, GPU_XLARGE_8, MULTIGPU_MEDIUM, got "GPU_MEDUIM"`}, - {"lowercase workload type", replaceFirst("GPU_MEDIUM", "gpu_medium"), "compute.workload_type must be one of"}, - {"zero environment version", replaceFirst(" dependencies: [einops==0.8.0]", " version: 0"), "environment.version must be a serverless environment version such as 6 or databricks_ai_v6"}, + {"AI environment before version 5", replaceFirst(" version: 6\n", " version: databricks_ai_v4\n"), "databricks_ai_v in environment.version requires AI Runtime version 5 or higher"}, + {"bad environment version", replaceFirst(" version: 6\n", " version: six\n"), "unsupported client image version"}, + {"zero environment version", replaceFirst(" version: 6\n", " version: 0\n"), "environment.version must be a serverless environment version such as 6 or databricks_ai_v6"}, + {"no environment version", replaceFirst(" version: 6\n", ""), "environment.version is required"}, {"empty dependency", replaceFirst("[einops==0.8.0]", `[""]`), "environment.dependencies[0] cannot be empty"}, - {"no endpoint", dropLine("endpoint:"), `endpoint must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, - {"endpoint with a path", replaceFirst("endpoint: qwen-chat", "endpoint: x/../../jobs"), "endpoint must be a serving endpoint name"}, - {"no compute", dropBlock("compute:"), "compute: section is required"}, - {"no workload size", dropLine(" workload_size:"), `compute.workload_size must be one of Small, Medium, Large, got ""`}, + {"no environment", dropBlock("environment:"), "environment is required for a checkpoint served by command"}, + {"no endpoint", dropLine("endpoint_name:"), `endpoint_name must be a serving endpoint name of letters, digits, '-' and '_', got ""`}, + {"endpoint with a path", replaceFirst("endpoint_name: qwen-chat", "endpoint_name: x/../../jobs"), "endpoint_name must be a serving endpoint name"}, + {"no workload size", dropLine(" workload_size:"), "compute.workload_size is required when compute is set"}, {"lowercase workload size", replaceFirst("workload_size: Small", "workload_size: medium"), "compute.workload_size must be one of Small, Medium, Large"}, + {"unsupported accelerator pair", replaceFirst("num_accelerators: 1", "num_accelerators: 2"), "must be a supported pair, got GPU_1xA10 with 2"}, {"secret without key", replaceFirst("llm/hf_token", "llm"), "expected format 'scope/key'"}, {"env var and secret", replaceFirst("LOG_LEVEL: info", "HF_TOKEN: x"), `"HF_TOKEN" is set in both env_variables and secrets`}, {"bad idempotency token", appendText("idempotency_token: a b\n"), "idempotency_token must be 1 to 128 letters"}, {"long idempotency token", appendText("idempotency_token: " + strings.Repeat("k", 129) + "\n"), "idempotency_token must be 1 to 128 letters"}, {"relative experiment", appendText("mlflow_experiment_path: Users/me/x\n"), "mlflow_experiment_path must be an absolute workspace path"}, - {"no model", dropBlock("model:"), "model: section is required"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := loadDeployConfigText(t, tt.edit(checkpointDeployConfig)) + _, err := loadDeployConfigText(t, tt.edit(commandDeployConfig)) assert.ErrorContains(t, err, tt.wantErr) }) } } -func TestDeployConfigSourceFields(t *testing.T) { - base := "endpoint: e\ncompute: {workload_type: GPU_SMALL, workload_size: Small}\nregistered_model: a.b.c\n" +func TestDeployConfigDedicatedValidation(t *testing.T) { tests := []struct { name string - source string + edit func(string) string wantErr string }{ - {"checkpoint entry without code_source", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\n", "code_source is required when model.entry is set"}, - {"checkpoint entry in a directory", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: src/agent.py}\ncode_source: {root_path: ., include_paths: [src]}\n", "model.entry must name a .py file directly under code_source.root_path"}, - {"checkpoint entry not python", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.ipynb}\ncode_source: {root_path: ., include_paths: [agent.ipynb]}\n", "model.entry must name a .py file"}, - {"checkpoint entry without include_paths", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: .}\n", "code_source.include_paths is required: list the files and directories to send"}, - {"checkpoint include pathspec magic", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\":!creds.json\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got ":!creds.json"`}, - {"checkpoint include glob", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\"*\"]}\n", `code_source.include_paths must name files and directories, without patterns or surrounding spaces, got "*"`}, - {"checkpoint include padded", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [\" agent.py\"]}\n", `got " agent.py"`}, - {"checkpoint include traversal", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [../x]}\n", "code_source.include_paths cannot contain '..' traversal"}, - {"checkpoint entry with command", "model: {type: checkpoint, path: /Volumes/a/b/c/m, entry: agent.py}\ncode_source: {root_path: ., include_paths: [agent.py]}\ntask: llm/v1/chat\ncommand: x\n", "exactly one of command and model.entry is required"}, - {"registered with entry", "model: {type: registered, version: 3, entry: agent.py}\n", "model.entry can't be set when model.type is registered"}, - {"registered without version", "model: {type: registered}\n", "model.version is required when model.type is registered"}, - {"registered with path", "model: {type: registered, version: 3, path: /Volumes/a/b/c/m}\n", "model.path can't be set when model.type is registered"}, - {"registered with token", "model: {type: registered, version: 3}\nidempotency_token: k\n", "idempotency_token can't be set when model.type is registered"}, - {"registered bad version", "model: {type: registered, version: v3}\n", "model.version must be a model version number"}, - {"registered version zero", "model: {type: registered, version: 0}\n", "model.version must be a model version number"}, + {"bad path", replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/dbfs/weights"), "checkpoint must be a Unity Catalog volume directory"}, + {"bad endpoint", replaceFirst("endpoint_name: qwen-chat", "endpoint_name: a/b"), "endpoint_name must be a serving endpoint name"}, + {"bad token", appendText("idempotency_token: a b\n"), "idempotency_token must be 1 to 128 letters"}, + {"unknown profile", replaceFirst("low_latency", "fastest"), "compute.compute_profile must be one of"}, + {"zero units", replaceFirst("provisioned_model_units: 600", "provisioned_model_units: 0"), "compute.provisioned_model_units must be above 0"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := loadDeployConfigText(t, base+tt.source) + _, err := loadDeployConfigText(t, tt.edit(dedicatedDeployConfig)) assert.ErrorContains(t, err, tt.wantErr) }) } @@ -306,20 +402,42 @@ func TestDeployConfigSourceFields(t *testing.T) { func TestDeployConfigCountsCharactersNotBytes(t *testing.T) { // 1,000 two-byte characters fit the 1,024-character cap on a volume path. - _, err := loadDeployConfigText(t, replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1000))(checkpointDeployConfig)) + _, err := loadDeployConfigText(t, replaceFirst("/Volumes/main/models/weights/qwen3_8b", "/Volumes/"+strings.Repeat("é", 1000))(commandDeployConfig)) assert.NoError(t, err) } -func TestDeployConfigServingRuntimeByWorkload(t *testing.T) { - for workloadType, want := range map[string]string{ - "CPU": "client.6", - "CPU_LARGE": "client.6", - "GPU_SMALL": "client.6-gpu", - "MULTIGPU_MEDIUM": "client.6-gpu", - } { - cfg := deployConfig{Compute: &deployComputeConfig{WorkloadType: workloadType}} - assert.Equal(t, want, cfg.servingRuntime(), workloadType) - } +func TestLoadDeployConfigOverride(t *testing.T) { + cfg, err := loadDeployConfigText(t, commandDeployConfig, "compute.num_accelerators=4", "environment.version=5") + require.NoError(t, err) + entity, err := cfg.servedEntity("3") + require.NoError(t, err) + assert.Equal(t, "MULTIGPU_MEDIUM", entity.WorkloadType) + assert.Equal(t, "client.5-gpu", cfg.registrationRequest("k").Checkpoint.Environment.Runtime) +} + +func TestLoadDeployConfigOverrideIsValidatedAgainstTheVariant(t *testing.T) { + _, err := loadDeployConfigText(t, commandDeployConfig, "compute.compute_profile=low_latency") + assert.ErrorContains(t, err, "compute.compute_profile can't be set: the file is read as a checkpoint served by command") +} + +func TestLoadDeployConfigRejectsUnknownOverride(t *testing.T) { + _, err := loadDeployConfigText(t, commandDeployConfig, "compute.workers=2") + assert.ErrorContains(t, err, `"workers" is not a known field`) +} + +func TestLoadDeployConfigRejectsUnknownField(t *testing.T) { + _, err := loadDeployConfigText(t, commandDeployConfig+"experiment_name: x\n") + assert.ErrorContains(t, err, "field experiment_name not found") +} + +func TestLoadDeployConfigRejectsOldShape(t *testing.T) { + _, err := loadDeployConfigText(t, "registered_model: main.models.qwen\nendpoint: qwen-chat\n") + assert.ErrorContains(t, err, "field registered_model not found") +} + +func TestLoadDeployConfigRejectsEmptyFile(t *testing.T) { + _, err := loadDeployConfigText(t, "") + assert.ErrorContains(t, err, "is empty") } func TestDeploySchemaHelp(t *testing.T) { @@ -328,37 +446,30 @@ func TestDeploySchemaHelp(t *testing.T) { var out strings.Builder renderConfigField(&out, deployCommandPath, root, nil) help := out.String() - assert.Contains(t, help, " databricks air deploy -h config.model.type\n") + assert.Contains(t, help, " databricks air deploy -h config.checkpoint\n") assert.Contains(t, help, " Model:\n") assert.Contains(t, help, " Serving:\n") assert.Contains(t, help, " Registration:\n") assert.Contains(t, help, `For details on a field, run "databricks air deploy -h config.".`) - field, err := resolveSchemaField(deploySchema(), "model.type") + field, err := resolveSchemaField(deploySchema(), "compute.compute_profile") require.NoError(t, err) - assert.Equal(t, "yes", field.required) + assert.Equal(t, "when checkpoint is set without command", field.required) - _, err = resolveSchemaField(deploySchema(), "model.typ") - assert.ErrorContains(t, err, `did you mean "type"?`) + _, err = resolveSchemaField(deploySchema(), "checkpont") + assert.ErrorContains(t, err, `did you mean "checkpoint"?`) _, err = resolveSchemaField(deploySchema(), "secrets.HF_TOKEN") assert.ErrorContains(t, err, "holds user-defined keys") } -// Every source field named in sourceFields and requiredSourceFields must be a -// schema path, so the per-source checks can't drift from the YAML. -func TestDeploySourceFieldsAreSchemaPaths(t *testing.T) { - for _, table := range []map[string][]string{sourceFields, requiredSourceFields} { - for _, fields := range table { - for _, f := range fields { - _, err := resolveSchemaField(deploySchema(), f) - assert.NoError(t, err, f) - } - } - } - for _, modelType := range modelTypes { - assert.Contains(t, requiredSourceFields, modelType) - assert.Contains(t, sourceFields, modelType) +// Every field a variant names must be a schema path, so the variant checks can't +// drift from the YAML. +func TestDeployVariantFieldsAreSchemaPaths(t *testing.T) { + var cfg deployConfig + for _, f := range cfg.variantFields() { + _, err := resolveSchemaField(deploySchema(), f.name) + assert.NoError(t, err, f.name) } } diff --git a/cmd/air/deployendpoint.go b/cmd/air/deployendpoint.go index a0481d40a57..7c64bcc1cc5 100644 --- a/cmd/air/deployendpoint.go +++ b/cmd/air/deployendpoint.go @@ -5,83 +5,241 @@ import ( "errors" "fmt" "maps" + "net/http" + "slices" "strings" "time" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/retry" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/retries" "github.com/databricks/databricks-sdk-go/service/serving" ) -// servedEntity is the one served entity air deploy puts on the endpoint. -func (c *deployConfig) servedEntity(version string) serving.ServedEntityInput { +// servingEndpointsPath is the serving endpoints API. Dedicated endpoints are created +// and configured under dedicatedEndpointsPath. +const ( + servingEndpointsPath = "/api/2.0/serving-endpoints" + dedicatedEndpointsPath = servingEndpointsPath + "/pt" +) + +// servedEntity is the one entity air deploy serves on the endpoint: a version of the +// model on serving compute, or on dedicated compute when Dedicated is set. +type servedEntity struct { + EntityName string + EntityVersion string + // WorkloadType and WorkloadSize are the serving compute. + WorkloadType string + WorkloadSize string + // EnvironmentVars are the served model's environment variables. + EnvironmentVars map[string]string + // Dedicated is the dedicated compute, nil for serving compute. + Dedicated *dedicatedCompute +} + +// servedEntity is the entity the endpoint serves for version: the version the +// registration produced, or the one the config names. +func (c *deployConfig) servedEntity(version string) (servedEntity, error) { + entity := servedEntity{EntityName: c.RegisteredModel, EntityVersion: version, EnvironmentVars: c.servedEnvironmentVars()} + if c.variant() == variantDedicated { + dedicated, err := c.resolveDedicatedCompute() + if err != nil { + return servedEntity{}, err + } + entity.Dedicated = dedicated + return entity, nil + } + workloadType, workloadSize, err := c.resolveServingCompute() + if err != nil { + return servedEntity{}, err + } + entity.WorkloadType, entity.WorkloadSize = workloadType, workloadSize + return entity, nil +} + +// servingInput is the entity as the serving endpoints API takes it. scale_to_zero_enabled +// is sent even though it's false, since the server's default isn't known to be false. +func (e servedEntity) servingInput() serving.ServedEntityInput { return serving.ServedEntityInput{ - EntityName: c.RegisteredModel, - EntityVersion: version, - WorkloadType: serving.ServingModelWorkloadType(c.Compute.WorkloadType), - WorkloadSize: c.Compute.WorkloadSize, - ScaleToZeroEnabled: c.Compute.ScaleToZeroEnabled, - EnvironmentVars: c.servedEnvironmentVars(), - // scale_to_zero_enabled: false is the user's choice, so send it rather than leave it - // to the server's default. - ForceSendFields: []string{"ScaleToZeroEnabled"}, - } -} - -// getEndpoint returns the endpoint named name, or nil if it doesn't exist. -func getEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string) (*serving.ServingEndpointDetailed, error) { - endpoint, err := retry.With(ctx, retry.IsTransientRead, func() (*serving.ServingEndpointDetailed, error) { - return w.ServingEndpoints.GetByName(ctx, name) + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + WorkloadType: serving.ServingModelWorkloadType(e.WorkloadType), + WorkloadSize: e.WorkloadSize, + ScaleToZeroEnabled: false, + EnvironmentVars: e.EnvironmentVars, + ForceSendFields: []string{"ScaleToZeroEnabled"}, + } +} + +// dedicatedServedEntity is a dedicated served entity as the dedicated endpoints API +// takes it. The SDK has no compute_profile, so these requests are sent as JSON. The +// entity_version is empty in a dry run, before the registration produces one. +type dedicatedServedEntity struct { + EntityName string `json:"entity_name"` + EntityVersion string `json:"entity_version,omitempty"` + ProvisionedModelUnits int64 `json:"provisioned_model_units"` + ComputeProfile string `json:"compute_profile"` +} + +// dedicatedEndpointConfig is the config of a dedicated endpoint. +type dedicatedEndpointConfig struct { + ServedEntities []dedicatedServedEntity `json:"served_entities"` +} + +// dedicatedEndpointUpdate replaces the config of a dedicated endpoint. +type dedicatedEndpointUpdate struct { + Config dedicatedEndpointConfig `json:"config"` +} + +// dedicatedEndpointRequest creates a dedicated endpoint. +type dedicatedEndpointRequest struct { + Name string `json:"name"` + Config dedicatedEndpointConfig `json:"config"` +} + +// dedicatedJSON is the entity as the dedicated endpoints API takes it. +func (e servedEntity) dedicatedJSON() dedicatedServedEntity { + return dedicatedServedEntity{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + ProvisionedModelUnits: e.Dedicated.ProvisionedModelUnits, + ComputeProfile: e.Dedicated.ComputeProfile, + } +} + +// servingEndpoint is a serving endpoint as GET /api/2.0/serving-endpoints/{name} +// reports it, with the fields air deploy reads. It replaces the SDK's type, which has +// no compute_profile for a dedicated served entity. +type servingEndpoint struct { + Name string `json:"name"` + State *serving.EndpointState `json:"state,omitempty"` + Config *endpointConfig `json:"config,omitempty"` + PendingConfig *endpointConfig `json:"pending_config,omitempty"` +} + +// endpointConfig is the served entities of an endpoint's current or pending config. +type endpointConfig struct { + ServedEntities []observedEntity `json:"served_entities,omitempty"` +} + +// observedEntity is a served entity as the endpoint reports it: a dedicated one by +// its units and profile, and any other by its workload. +type observedEntity struct { + EntityName string `json:"entity_name"` + EntityVersion string `json:"entity_version"` + WorkloadType string `json:"workload_type"` + WorkloadSize string `json:"workload_size"` + ScaleToZeroEnabled bool `json:"scale_to_zero_enabled"` + EnvironmentVars map[string]string `json:"environment_vars"` + ProvisionedModelUnits int64 `json:"provisioned_model_units"` + ComputeProfile string `json:"compute_profile"` +} + +// endpointAPI calls the serving endpoints API for air deploy. Endpoints are read as +// JSON, and dedicated endpoints written so; serving compute is written through the SDK. +type endpointAPI struct { + w *databricks.WorkspaceClient + api *client.DatabricksClient + headers map[string]string +} + +func newEndpointAPI(w *databricks.WorkspaceClient) (*endpointAPI, error) { + api, err := client.New(w.Config) + if err != nil { + return nil, withCode(codeInternalError, false, fmt.Errorf("failed to create API client: %w", err)) + } + return &endpointAPI{w: w, api: api, headers: auth.WorkspaceIDHeaders(w.Config)}, nil +} + +// read returns the endpoint named name, and the error the API gives when it doesn't +// exist. A transient failure is repeated. +func (e *endpointAPI) read(ctx context.Context, name string) (*servingEndpoint, error) { + return retry.With(ctx, retry.IsTransientRead, func() (*servingEndpoint, error) { + endpoint := &servingEndpoint{} + err := e.api.Do(ctx, http.MethodGet, servingEndpointsPath+"/"+name, e.headers, nil, nil, endpoint) + return endpoint, err }) +} + +// get returns the endpoint named name, or nil if it doesn't exist. +func (e *endpointAPI) get(ctx context.Context, name string) (*servingEndpoint, error) { + endpoint, err := e.read(ctx, name) if errors.Is(err, apierr.ErrResourceDoesNotExist) { return nil, nil } return endpoint, err } -// waitNotUpdating waits until the endpoint has no update in progress, calling -// progress on every poll. It polls as the SDK's WaitGetServingEndpointNotUpdating -// does, except that each poll repeats a transient failure, which ends the SDK's -// wait: an endpoint can take many minutes to start. Creates and config updates -// aren't repeated this way, since a repeat after an applied write would fail. -func waitNotUpdating(ctx context.Context, w *databricks.WorkspaceClient, name string, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { - return retries.Poll(ctx, untilDeadline(ctx), func() (*serving.ServingEndpointDetailed, *retries.Err) { - endpoint, err := retry.With(ctx, retry.IsTransientRead, func() (*serving.ServingEndpointDetailed, error) { - return w.ServingEndpoints.GetByName(ctx, name) - }) - if err != nil { - return nil, retries.Halt(err) - } - if progress != nil { - progress(endpoint) - } - var state serving.EndpointStateConfigUpdate - if endpoint.State != nil { - state = endpoint.State.ConfigUpdate - } - switch state { - case serving.EndpointStateConfigUpdateNotUpdating: - return endpoint, nil - case serving.EndpointStateConfigUpdateUpdateFailed, serving.EndpointStateConfigUpdateUpdateCanceled: - return nil, retries.Halt(fmt.Errorf("the endpoint's update ended in %s; check its events and logs in the Serving UI", state)) - default: - return nil, retries.Continues(string(state)) +// create creates the endpoint named name with entity. It returns once the API accepts +// the request, not once the endpoint is up. +func (e *endpointAPI) create(ctx context.Context, name string, entity servedEntity) error { + if entity.Dedicated != nil { + request := dedicatedEndpointRequest{ + Name: name, + Config: dedicatedEndpointConfig{ServedEntities: []dedicatedServedEntity{entity.dedicatedJSON()}}, } + return e.api.Do(ctx, http.MethodPost, dedicatedEndpointsPath, e.headers, nil, request, nil) + } + _, err := e.w.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ + Name: name, + Config: &serving.EndpointCoreConfigInput{ServedEntities: []serving.ServedEntityInput{entity.servingInput()}}, + }) + return err +} + +// update replaces the endpoint named name's served entity with entity. It returns once +// the API accepts the update. +func (e *endpointAPI) update(ctx context.Context, name string, entity servedEntity) error { + if entity.Dedicated != nil { + update := dedicatedEndpointUpdate{Config: dedicatedEndpointConfig{ServedEntities: []dedicatedServedEntity{entity.dedicatedJSON()}}} + return e.api.Do(ctx, http.MethodPut, dedicatedEndpointsPath+"/"+name+"/config", e.headers, nil, update, nil) + } + _, err := e.w.ServingEndpoints.UpdateConfig(ctx, serving.EndpointCoreConfigInput{ + Name: name, + ServedEntities: []serving.ServedEntityInput{entity.servingInput()}, }) + return err } // checkEndpoint fails when the endpoint named name, if it exists, serves -// something air deploy won't replace with a version of model. air deploy checks -// it before registering, so a wrong endpoint name fails before any weights are -// copied. -func checkEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name, model string) error { - endpoint, err := getEndpoint(ctx, w, name) +// something air deploy won't replace with a version of model, or is of the other +// kind than dedicated says. air deploy checks it before registering, so a wrong +// endpoint name fails before any weights are copied. +func checkEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name, model string, dedicated bool) error { + e, err := newEndpointAPI(w) + if err != nil { + return err + } + endpoint, err := e.get(ctx, name) if err != nil || endpoint == nil { return err } - return checkReplaceable(endpoint, model) + if err := checkReplaceable(endpoint, model); err != nil { + return err + } + return checkKind(endpoint, dedicated) +} + +// checkKind fails when the endpoint serves on dedicated compute and dedicated is +// false, or the reverse: the server converts neither kind of endpoint to the other. +// An endpoint that reports no served entity is taken as either kind. +func checkKind(endpoint *servingEndpoint, dedicated bool) error { + served := append(currentEntities(endpoint), pendingEntities(endpoint)...) + if len(served) == 0 { + return nil + } + isDedicated := slices.ContainsFunc(served, func(e observedEntity) bool { return e.ComputeProfile != "" }) + if isDedicated == dedicated { + return nil + } + kind, want := "serving compute", "dedicated compute" + if isDedicated { + kind, want = want, kind + } + return withCode(codeInvalidArgs, false, fmt.Errorf("endpoint %s serves on %s, and this file serves on %s; an endpoint can't change between the two, so set endpoint_name to another name", stripControl(endpoint.Name), kind, want)) } // checkReplaceable fails unless the endpoint serves at most one entity, a version @@ -89,10 +247,10 @@ func checkEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name, mod // remove another model or a traffic split the user didn't ask to change. The // endpoint lists every served model under served_entities too (served_models // mirrors it), so served_entities covers both. -func checkReplaceable(endpoint *serving.ServingEndpointDetailed, model string) error { - for _, served := range [][]serving.ServedEntityOutput{currentEntities(endpoint), pendingEntities(endpoint)} { +func checkReplaceable(endpoint *servingEndpoint, model string) error { + for _, served := range [][]observedEntity{currentEntities(endpoint), pendingEntities(endpoint)} { if len(served) > 1 || (len(served) == 1 && !strings.EqualFold(served[0].EntityName, model)) { - return withCode(codeInvalidArgs, false, fmt.Errorf("endpoint %s serves %s, and air deploy replaces only a single version of %s; set endpoint to another name, or change this endpoint with databricks serving-endpoints update-config", stripControl(endpoint.Name), describeEntities(served), model)) + return withCode(codeInvalidArgs, false, fmt.Errorf("endpoint %s serves %s, and air deploy replaces only a single version of %s; set endpoint_name to another name, or change this endpoint with databricks serving-endpoints update-config", stripControl(endpoint.Name), describeEntities(served), model)) } } return nil @@ -102,27 +260,27 @@ func checkReplaceable(endpoint *serving.ServingEndpointDetailed, model string) e // it is ready, calling progress on every poll. It creates the endpoint if it // doesn't exist. An endpoint that already serves entity isn't updated, and an // update to entity that an interrupted air deploy started is waited for. -func serveOnEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string, entity serving.ServedEntityInput, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { - endpoint, err := getEndpoint(ctx, w, name) +func serveOnEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name string, entity servedEntity, progress func(*servingEndpoint)) (*servingEndpoint, error) { + e, err := newEndpointAPI(w) + if err != nil { + return nil, err + } + endpoint, err := e.get(ctx, name) if err != nil { return nil, err } if endpoint == nil { - _, err := w.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ - Name: name, - Config: &serving.EndpointCoreConfigInput{ServedEntities: []serving.ServedEntityInput{entity}}, - }) - if err != nil { + if err := e.create(ctx, name, entity); err != nil { return nil, err } - return waitServes(ctx, w, name, entity, progress) + return waitServes(ctx, e, name, entity, progress) } if updating(endpoint) { if servesOnly(pendingEntities(endpoint), entity) { - return waitServes(ctx, w, name, entity, progress) + return waitServes(ctx, e, name, entity, progress) } // The endpoint takes no other update until this one finishes. - endpoint, err = waitNotUpdating(ctx, w, name, progress) + endpoint, err = waitNotUpdating(ctx, e, name, progress) if err != nil { return nil, err } @@ -130,23 +288,50 @@ func serveOnEndpoint(ctx context.Context, w *databricks.WorkspaceClient, name st if err := checkReplaceable(endpoint, entity.EntityName); err != nil { return nil, err } + if err := checkKind(endpoint, entity.Dedicated != nil); err != nil { + return nil, err + } if servesOnly(currentEntities(endpoint), entity) { return checkServes(endpoint, entity) } - _, err = w.ServingEndpoints.UpdateConfig(ctx, serving.EndpointCoreConfigInput{ - Name: name, - ServedEntities: []serving.ServedEntityInput{entity}, - }) - if err != nil { + if err := e.update(ctx, name, entity); err != nil { return nil, err } - return waitServes(ctx, w, name, entity, progress) + return waitServes(ctx, e, name, entity, progress) +} + +// waitNotUpdating waits until the endpoint has no update in progress, calling +// progress on every poll. Each poll repeats a transient failure, which ends the SDK's +// own wait: an endpoint can take many minutes to start. Creates and config updates +// aren't repeated this way, since a repeat after an applied write would fail. +func waitNotUpdating(ctx context.Context, e *endpointAPI, name string, progress func(*servingEndpoint)) (*servingEndpoint, error) { + return retries.Poll(ctx, untilDeadline(ctx), func() (*servingEndpoint, *retries.Err) { + endpoint, err := e.read(ctx, name) + if err != nil { + return nil, retries.Halt(err) + } + if progress != nil { + progress(endpoint) + } + var state serving.EndpointStateConfigUpdate + if endpoint.State != nil { + state = endpoint.State.ConfigUpdate + } + switch state { + case serving.EndpointStateConfigUpdateNotUpdating: + return endpoint, nil + case serving.EndpointStateConfigUpdateUpdateFailed, serving.EndpointStateConfigUpdateUpdateCanceled: + return nil, retries.Halt(fmt.Errorf("the endpoint's update ended in %s; check its events and logs in the Serving UI", state)) + default: + return nil, retries.Continues(string(state)) + } + }) } // waitServes waits until the endpoint's update finishes, then checks that it // serves entity. -func waitServes(ctx context.Context, w *databricks.WorkspaceClient, name string, entity serving.ServedEntityInput, progress func(*serving.ServingEndpointDetailed)) (*serving.ServingEndpointDetailed, error) { - endpoint, err := waitNotUpdating(ctx, w, name, progress) +func waitServes(ctx context.Context, e *endpointAPI, name string, entity servedEntity, progress func(*servingEndpoint)) (*servingEndpoint, error) { + endpoint, err := waitNotUpdating(ctx, e, name, progress) if err != nil { return nil, err } @@ -159,7 +344,7 @@ func waitServes(ctx context.Context, w *databricks.WorkspaceClient, name string, var errEndpointNotReady = errors.New("endpoint isn't ready") // checkServes returns endpoint if it serves only entity and is ready to serve. -func checkServes(endpoint *serving.ServingEndpointDetailed, entity serving.ServedEntityInput) (*serving.ServingEndpointDetailed, error) { +func checkServes(endpoint *servingEndpoint, entity servedEntity) (*servingEndpoint, error) { name := stripControl(endpoint.Name) if !servesOnly(currentEntities(endpoint), entity) { return nil, fmt.Errorf("endpoint %s was changed to serve %s while air deploy waited for it", name, describeEntities(currentEntities(endpoint))) @@ -170,18 +355,18 @@ func checkServes(endpoint *serving.ServingEndpointDetailed, entity serving.Serve return endpoint, nil } -func updating(endpoint *serving.ServingEndpointDetailed) bool { +func updating(endpoint *servingEndpoint) bool { return endpoint.State != nil && endpoint.State.ConfigUpdate == serving.EndpointStateConfigUpdateInProgress } -func currentEntities(endpoint *serving.ServingEndpointDetailed) []serving.ServedEntityOutput { +func currentEntities(endpoint *servingEndpoint) []observedEntity { if endpoint.Config == nil { return nil } return endpoint.Config.ServedEntities } -func pendingEntities(endpoint *serving.ServingEndpointDetailed) []serving.ServedEntityOutput { +func pendingEntities(endpoint *servingEndpoint) []observedEntity { if endpoint.PendingConfig == nil { return nil } @@ -189,7 +374,7 @@ func pendingEntities(endpoint *serving.ServingEndpointDetailed) []serving.Served } // describeEntities names served entities for an error, e.g. "version 2 of a.b.c". -func describeEntities(served []serving.ServedEntityOutput) string { +func describeEntities(served []observedEntity) string { if len(served) == 0 { return "nothing" } @@ -205,20 +390,25 @@ func describeEntities(served []serving.ServedEntityOutput) string { return strings.Join(names, " and ") } -// servesOnly reports whether served is exactly entity, with the same workload and -// environment variables. -func servesOnly(served []serving.ServedEntityOutput, entity serving.ServedEntityInput) bool { +// servesOnly reports whether served is exactly entity. A dedicated entity must match +// its units and compute profile; any other must match its workload, scale-to-zero +// setting and environment variables. +func servesOnly(served []observedEntity, entity servedEntity) bool { if len(served) != 1 { return false } s := served[0] // Unity Catalog names are case-insensitive, and the endpoint reports them as // Unity Catalog stores them. - return strings.EqualFold(s.EntityName, entity.EntityName) && - s.EntityVersion == entity.EntityVersion && - s.WorkloadType == entity.WorkloadType && + if !strings.EqualFold(s.EntityName, entity.EntityName) || s.EntityVersion != entity.EntityVersion { + return false + } + if entity.Dedicated != nil { + return s.ProvisionedModelUnits == entity.Dedicated.ProvisionedModelUnits && s.ComputeProfile == entity.Dedicated.ComputeProfile + } + return s.WorkloadType == entity.WorkloadType && s.WorkloadSize == entity.WorkloadSize && - s.ScaleToZeroEnabled == entity.ScaleToZeroEnabled && + !s.ScaleToZeroEnabled && maps.Equal(s.EnvironmentVars, entity.EnvironmentVars) } diff --git a/cmd/air/deployendpoint_test.go b/cmd/air/deployendpoint_test.go index 0a4dc57e019..177d9b43234 100644 --- a/cmd/air/deployendpoint_test.go +++ b/cmd/air/deployendpoint_test.go @@ -14,17 +14,17 @@ import ( "github.com/stretchr/testify/require" ) -const servingEndpointsPath = "/api/2.0/serving-endpoints" - // fakeEndpoints is a stateful fake of the serving endpoints API for one endpoint. // A create or config update is applied at once, and the endpoint then reports // finalState (READY and NOT_UPDATING unless a test sets otherwise). type fakeEndpoints struct { t *testing.T mu sync.Mutex - endpoint *serving.ServingEndpointDetailed + endpoint *servingEndpoint finalState serving.EndpointState - writes []string + // writes names every create and config update: "create" and "update" for serving + // compute, "create dedicated" and "update dedicated" for dedicated compute. + writes []string // rawWrites holds the body of every create and config update, as sent. rawWrites []string // gets counts the GETs of an existing endpoint. Those numbered in failOnGets @@ -34,7 +34,7 @@ type fakeEndpoints struct { readyAfterGets int } -func newFakeEndpoints(t *testing.T, existing *serving.ServingEndpointDetailed) *fakeEndpoints { +func newFakeEndpoints(t *testing.T, existing *servingEndpoint) *fakeEndpoints { return &fakeEndpoints{ t: t, endpoint: existing, @@ -52,20 +52,49 @@ func (f *fakeEndpoints) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } f.writes = append(f.writes, "create") - f.endpoint = &serving.ServingEndpointDetailed{Name: req.Name, Config: f.output(req.Config.ServedEntities)} + var entities []observedEntity + for _, e := range req.Config.ServedEntities { + entities = append(entities, observeServing(e)) + } + f.endpoint = &servingEndpoint{Name: req.Name, Config: &endpointConfig{ServedEntities: entities}, State: &f.finalState} + case r.Method == http.MethodPost && r.URL.Path == dedicatedEndpointsPath: + var req dedicatedEndpointRequest + if !f.decode(w, r, &req) { + return + } + f.writes = append(f.writes, "create dedicated") + f.endpoint = &servingEndpoint{Name: req.Name, Config: &endpointConfig{ServedEntities: observeDedicated(req.Config.ServedEntities)}, State: &f.finalState} + case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, dedicatedEndpointsPath+"/"): + var req dedicatedEndpointUpdate + if !f.decode(w, r, &req) { + return + } + // As the server does, which reads the entities from the request's config. + if len(req.Config.ServedEntities) == 0 { + http.Error(w, `{"error_code": "INVALID_PARAMETER_VALUE", "message": "config is required for dedicated inference"}`, http.StatusBadRequest) + return + } + if f.updateConflicts(w) { + return + } + f.writes = append(f.writes, "update dedicated") + f.endpoint.Config = &endpointConfig{ServedEntities: observeDedicated(req.Config.ServedEntities)} + f.endpoint.PendingConfig = nil f.endpoint.State = &f.finalState case r.Method == http.MethodPut && strings.HasSuffix(r.URL.Path, "/config"): var req serving.EndpointCoreConfigInput if !f.decode(w, r, &req) { return } - if updating(f.endpoint) { - w.WriteHeader(http.StatusConflict) - _, _ = w.Write([]byte(`{"error_code": "RESOURCE_CONFLICT", "message": "The endpoint is being updated."}`)) + if f.updateConflicts(w) { return } f.writes = append(f.writes, "update") - f.endpoint.Config = f.output(req.ServedEntities) + var entities []observedEntity + for _, e := range req.ServedEntities { + entities = append(entities, observeServing(e)) + } + f.endpoint.Config = &endpointConfig{ServedEntities: entities} f.endpoint.PendingConfig = nil f.endpoint.State = &f.finalState case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, servingEndpointsPath+"/"): @@ -92,6 +121,16 @@ func (f *fakeEndpoints) ServeHTTP(w http.ResponseWriter, r *http.Request) { assert.NoError(f.t, json.NewEncoder(w).Encode(f.endpoint)) } +// updateConflicts answers a config update that arrives while the endpoint is updating. +func (f *fakeEndpoints) updateConflicts(w http.ResponseWriter) bool { + if !updating(f.endpoint) { + return false + } + w.WriteHeader(http.StatusConflict) + _, _ = w.Write([]byte(`{"error_code": "RESOURCE_CONFLICT", "message": "The endpoint is being updated."}`)) + return true +} + // decode reads a request body into v, failing the request (not the handler // goroutine) when it can't. func (f *fakeEndpoints) decode(w http.ResponseWriter, r *http.Request, v any) bool { @@ -104,22 +143,47 @@ func (f *fakeEndpoints) decode(w http.ResponseWriter, r *http.Request, v any) bo return true } -func (f *fakeEndpoints) output(entities []serving.ServedEntityInput) *serving.EndpointCoreConfigOutput { - out := &serving.EndpointCoreConfigOutput{ConfigVersion: 1} +// observeServing is a served entity with serving compute, as the endpoint reports it. +func observeServing(e serving.ServedEntityInput) observedEntity { + return observedEntity{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + WorkloadType: string(e.WorkloadType), + WorkloadSize: e.WorkloadSize, + EnvironmentVars: e.EnvironmentVars, + } +} + +// observeDedicated are dedicated served entities, as the endpoint reports them. +func observeDedicated(entities []dedicatedServedEntity) []observedEntity { + var out []observedEntity for _, e := range entities { - out.ServedEntities = append(out.ServedEntities, servedOutput(e)) + out = append(out, observedEntity{ + EntityName: e.EntityName, + EntityVersion: e.EntityVersion, + ProvisionedModelUnits: e.ProvisionedModelUnits, + ComputeProfile: e.ComputeProfile, + }) } return out } -func servedOutput(e serving.ServedEntityInput) serving.ServedEntityOutput { - return serving.ServedEntityOutput{ - EntityName: e.EntityName, - EntityVersion: e.EntityVersion, - WorkloadType: e.WorkloadType, - WorkloadSize: e.WorkloadSize, - ScaleToZeroEnabled: e.ScaleToZeroEnabled, - EnvironmentVars: e.EnvironmentVars, +// observe is entity as the endpoint reports it once it serves it. +func observe(entity servedEntity) observedEntity { + if entity.Dedicated != nil { + return observedEntity{ + EntityName: entity.EntityName, + EntityVersion: entity.EntityVersion, + ProvisionedModelUnits: entity.Dedicated.ProvisionedModelUnits, + ComputeProfile: entity.Dedicated.ComputeProfile, + } + } + return observedEntity{ + EntityName: entity.EntityName, + EntityVersion: entity.EntityVersion, + WorkloadType: entity.WorkloadType, + WorkloadSize: entity.WorkloadSize, + EnvironmentVars: entity.EnvironmentVars, } } @@ -129,20 +193,30 @@ func (f *fakeEndpoints) start() *httptest.Server { return srv } -func testEntity(version string) serving.ServedEntityInput { - return serving.ServedEntityInput{ +// testEntity is a version of the test model on serving compute: a command's GPU_MEDIUM. +func testEntity(version string) servedEntity { + return servedEntity{ EntityName: "main.models.qwen", EntityVersion: version, - WorkloadType: serving.ServingModelWorkloadTypeGpuMedium, + WorkloadType: "GPU_MEDIUM", WorkloadSize: "Small", EnvironmentVars: map[string]string{"HF_TOKEN": "{{secrets/llm/hf_token}}"}, } } -func endpointServing(entity serving.ServedEntityInput, state serving.EndpointState) *serving.ServingEndpointDetailed { - return &serving.ServingEndpointDetailed{ +// testDedicatedEntity is a version of the test model on dedicated compute. +func testDedicatedEntity(version string) servedEntity { + return servedEntity{ + EntityName: "main.models.qwen", + EntityVersion: version, + Dedicated: &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_LOW_LATENCY", ProvisionedModelUnits: 600}, + } +} + +func endpointServing(entity servedEntity, state serving.EndpointState) *servingEndpoint { + return &servingEndpoint{ Name: "qwen-chat", - Config: &serving.EndpointCoreConfigOutput{ServedEntities: []serving.ServedEntityOutput{servedOutput(entity)}}, + Config: &endpointConfig{ServedEntities: []observedEntity{observe(entity)}}, State: &state, } } @@ -191,14 +265,128 @@ func TestServeOnEndpointLeavesAServedVersion(t *testing.T) { assert.Empty(t, fake.writes) } +func TestServeOnEndpointCreatesADedicatedEndpoint(t *testing.T) { + fake := newFakeEndpoints(t, nil) + srv := fake.start() + + endpoint, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testDedicatedEntity("3"), nil) + require.NoError(t, err) + assert.Equal(t, []string{"create dedicated"}, fake.writes) + assert.Equal(t, "COMPUTE_PROFILE_LOW_LATENCY", endpoint.Config.ServedEntities[0].ComputeProfile) + require.Len(t, fake.rawWrites, 1) + assert.JSONEq(t, `{ + "name": "qwen-chat", + "config": {"served_entities": [{ + "entity_name": "main.models.qwen", + "entity_version": "3", + "provisioned_model_units": 600, + "compute_profile": "COMPUTE_PROFILE_LOW_LATENCY" + }]} + }`, fake.rawWrites[0]) +} + +func TestServeOnEndpointUpdatesADedicatedEndpointWhenItsCapacityChanges(t *testing.T) { + tests := []struct { + name string + entity servedEntity + wantBody string + }{ + { + "units", + servedEntity{EntityName: "main.models.qwen", EntityVersion: "3", Dedicated: &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_LOW_LATENCY", ProvisionedModelUnits: 900}}, + `{"config": {"served_entities": [{"entity_name": "main.models.qwen", "entity_version": "3", "provisioned_model_units": 900, "compute_profile": "COMPUTE_PROFILE_LOW_LATENCY"}]}}`, + }, + { + "profile", + servedEntity{EntityName: "main.models.qwen", EntityVersion: "3", Dedicated: &dedicatedCompute{ComputeProfile: "COMPUTE_PROFILE_FAST", ProvisionedModelUnits: 600}}, + `{"config": {"served_entities": [{"entity_name": "main.models.qwen", "entity_version": "3", "provisioned_model_units": 600, "compute_profile": "COMPUTE_PROFILE_FAST"}]}}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testDedicatedEntity("3"), readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", tt.entity, nil) + require.NoError(t, err) + assert.Equal(t, []string{"update dedicated"}, fake.writes) + require.Len(t, fake.rawWrites, 1) + assert.JSONEq(t, tt.wantBody, fake.rawWrites[0]) + }) + } +} + +func TestServeOnEndpointLeavesADedicatedVersion(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(testDedicatedEntity("3"), readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testDedicatedEntity("3"), nil) + require.NoError(t, err) + assert.Empty(t, fake.writes) +} + +func TestServeOnEndpointRefusesToChangeAnEndpointsKind(t *testing.T) { + tests := []struct { + name string + existing servedEntity + entity servedEntity + wantErr string + }{ + {"serving compute to dedicated", testEntity("3"), testDedicatedEntity("3"), "endpoint qwen-chat serves on serving compute, and this file serves on dedicated compute"}, + {"dedicated to serving compute", testDedicatedEntity("3"), testEntity("3"), "endpoint qwen-chat serves on dedicated compute, and this file serves on serving compute"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fake := newFakeEndpoints(t, endpointServing(tt.existing, readyState)) + srv := fake.start() + + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", tt.entity, nil) + assert.ErrorContains(t, err, tt.wantErr) + assert.Empty(t, fake.writes) + }) + } +} + +func TestCheckEndpointRefusesAnEndpointOfTheOtherKind(t *testing.T) { + tests := []struct { + name string + existing servedEntity + dedicated bool + wantErr string + }{ + {"dedicated file, serving endpoint", testEntity("2"), true, "serves on serving compute, and this file serves on dedicated compute"}, + {"serving file, dedicated endpoint", testDedicatedEntity("2"), false, "serves on dedicated compute, and this file serves on serving compute"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + srv := newFakeEndpoints(t, endpointServing(tt.existing, readyState)).start() + err := checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen", tt.dedicated) + assert.ErrorContains(t, err, tt.wantErr) + }) + } +} + +func TestCheckEndpointAllowsAnEndpointOfTheSameKind(t *testing.T) { + for name, tt := range map[string]struct { + existing servedEntity + dedicated bool + }{ + "serving": {testEntity("2"), false}, + "dedicated": {testDedicatedEntity("2"), true}, + } { + srv := newFakeEndpoints(t, endpointServing(tt.existing, readyState)).start() + assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen", tt.dedicated), name) + } +} + func TestServeOnEndpointWaitsForItsOwnPendingUpdate(t *testing.T) { existing := endpointServing(testEntity("2"), serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress}) - existing.PendingConfig = &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{servedOutput(testEntity("3"))}} + existing.PendingConfig = &endpointConfig{ServedEntities: []observedEntity{observe(testEntity("3"))}} fake := newFakeEndpoints(t, existing) srv := fake.start() polls := 0 - _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(*serving.ServingEndpointDetailed) { + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(*servingEndpoint) { // The pending update finishes after the first poll. polls++ fake.mu.Lock() @@ -212,11 +400,11 @@ func TestServeOnEndpointWaitsForItsOwnPendingUpdate(t *testing.T) { func TestServeOnEndpointWaitsForAnotherUpdateBeforeUpdating(t *testing.T) { existing := endpointServing(testEntity("1"), serving.EndpointState{Ready: serving.EndpointStateReadyReady, ConfigUpdate: serving.EndpointStateConfigUpdateInProgress}) - existing.PendingConfig = &serving.EndpointPendingConfig{ServedEntities: []serving.ServedEntityOutput{servedOutput(testEntity("2"))}} + existing.PendingConfig = &endpointConfig{ServedEntities: []observedEntity{observe(testEntity("2"))}} fake := newFakeEndpoints(t, existing) srv := fake.start() - _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(endpoint *serving.ServingEndpointDetailed) { + _, err := serveOnEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", testEntity("3"), func(endpoint *servingEndpoint) { // The other update finishes after the first poll. if updating(endpoint) { fake.mu.Lock() @@ -233,10 +421,10 @@ func TestServeOnEndpointRefusesToReplaceOtherEntities(t *testing.T) { other := testEntity("5") other.EntityName = "main.models.llama" split := endpointServing(testEntity("2"), readyState) - split.Config.ServedEntities = append(split.Config.ServedEntities, servedOutput(testEntity("1"))) + split.Config.ServedEntities = append(split.Config.ServedEntities, observe(testEntity("1"))) tests := []struct { name string - existing *serving.ServingEndpointDetailed + existing *servingEndpoint wantErr string }{ {"another model", endpointServing(other, readyState), "endpoint qwen-chat serves version 5 of main.models.llama, and air deploy replaces only a single version of main.models.qwen"}, @@ -247,7 +435,7 @@ func TestServeOnEndpointRefusesToReplaceOtherEntities(t *testing.T) { fake := newFakeEndpoints(t, tt.existing) w := newTestWorkspaceClient(t, fake.start().URL) - assert.ErrorContains(t, checkEndpoint(withDeadline(t), w, "qwen-chat", "main.models.qwen"), tt.wantErr) + assert.ErrorContains(t, checkEndpoint(withDeadline(t), w, "qwen-chat", "main.models.qwen", false), tt.wantErr) _, err := serveOnEndpoint(withDeadline(t), w, "qwen-chat", testEntity("3"), nil) assert.ErrorContains(t, err, tt.wantErr) assert.Empty(t, fake.writes) @@ -257,7 +445,7 @@ func TestServeOnEndpointRefusesToReplaceOtherEntities(t *testing.T) { func TestCheckEndpointAllowsAMissingEndpoint(t *testing.T) { srv := newFakeEndpoints(t, nil).start() - assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen")) + assert.NoError(t, checkEndpoint(withDeadline(t), newTestWorkspaceClient(t, srv.URL), "qwen-chat", "main.models.qwen", false)) } func TestServeOnEndpointWaitsThroughTransientFailures(t *testing.T) { @@ -292,17 +480,31 @@ func TestServeOnEndpointReportsAnEndpointThatIsNotReady(t *testing.T) { assert.ErrorContains(t, err, "endpoint qwen-chat serves version 3 of main.models.qwen but isn't ready") } +func TestDedicatedEntityJSON(t *testing.T) { + entity := testDedicatedEntity("3") + raw, err := json.Marshal(entity.dedicatedJSON()) + require.NoError(t, err) + assert.JSONEq(t, `{"entity_name": "main.models.qwen", "entity_version": "3", "provisioned_model_units": 600, "compute_profile": "COMPUTE_PROFILE_LOW_LATENCY"}`, string(raw)) +} + func TestServesOnly(t *testing.T) { entity := testEntity("3") - assert.True(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity)}, entity)) + assert.True(t, servesOnly([]observedEntity{observe(entity)}, entity)) assert.False(t, servesOnly(nil, entity)) - assert.False(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity), servedOutput(testEntity("2"))}, entity)) + assert.False(t, servesOnly([]observedEntity{observe(entity), observe(testEntity("2"))}, entity)) changed := entity changed.EnvironmentVars = map[string]string{"HF_TOKEN": "{{secrets/llm/other}}"} - assert.False(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity)}, changed)) + assert.False(t, servesOnly([]observedEntity{observe(entity)}, changed)) - scaled := entity + scaled := observe(entity) scaled.ScaleToZeroEnabled = true - assert.False(t, servesOnly([]serving.ServedEntityOutput{servedOutput(entity)}, scaled)) + assert.False(t, servesOnly([]observedEntity{scaled}, entity)) +} + +func TestServesOnlyDedicated(t *testing.T) { + entity := testDedicatedEntity("3") + assert.True(t, servesOnly([]observedEntity{observe(entity)}, entity)) + assert.False(t, servesOnly([]observedEntity{observe(testEntity("3"))}, entity), "serving compute isn't a dedicated entity") + assert.False(t, servesOnly([]observedEntity{observe(entity)}, testEntity("3")), "a dedicated entity isn't serving compute") } diff --git a/cmd/air/deployregister.go b/cmd/air/deployregister.go index b1d4207c64c..d455ae47960 100644 --- a/cmd/air/deployregister.go +++ b/cmd/air/deployregister.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "slices" + "strconv" "time" "github.com/databricks/cli/libs/auth" @@ -45,44 +46,36 @@ const ( stagingStateFailed = "FAILED" ) -// registerModelPackageRequest is the RegisterModelPackageRequest body. +// registerModelPackageRequest is the RegisterModelPackageRequest body. It sets +// exactly one of Checkpoint, a checkpoint served by command, and DedicatedCheckpoint, +// a checkpoint served with dedicated inference. type registerModelPackageRequest struct { - RegisteredModel string `json:"registered_model"` - IdempotencyKey string `json:"idempotency_key"` - Checkpoint *modelPackageCheckpoint `json:"checkpoint,omitempty"` - ExperimentPath string `json:"experiment_path,omitempty"` - RetryFailedStaging bool `json:"retry_failed_staging,omitempty"` + RegisteredModel string `json:"registered_model"` + IdempotencyKey string `json:"idempotency_key"` + Checkpoint *modelPackageCheckpoint `json:"checkpoint,omitempty"` + DedicatedCheckpoint *dedicatedCheckpoint `json:"dedicated_checkpoint,omitempty"` + ExperimentPath string `json:"experiment_path,omitempty"` + RetryFailedStaging bool `json:"retry_failed_staging,omitempty"` } -// modelPackageCheckpoint is a checkpoint's weights in a volume and what serves -// them: exactly one of ServingCommand and PythonModel is set. +// modelPackageCheckpoint is a checkpoint's weights in a volume and the command that +// serves them. type modelPackageCheckpoint struct { - VolumePath string `json:"volume_path"` - ServingCommand string `json:"serving_command,omitempty"` - PythonModel *modelPackagePythonModel `json:"python_model,omitempty"` - Task string `json:"task,omitempty"` - Environment modelPackageEnvironment `json:"environment"` + VolumePath string `json:"volume_path"` + ServingCommand string `json:"serving_command"` + Task string `json:"task"` + Environment modelPackageEnvironment `json:"environment"` } -// modelPackagePythonModel is the Python code that serves a checkpoint's weights. -type modelPackagePythonModel struct { - EntryFile string `json:"entry_file"` - EntrySource string `json:"entry_source"` - Files []modelPackageFile `json:"files,omitempty"` - ModelConfig string `json:"model_config,omitempty"` - Streamable *bool `json:"streamable,omitempty"` +// dedicatedCheckpoint is a checkpoint's weights in a volume, served with dedicated +// inference. +type dedicatedCheckpoint struct { + VolumePath string `json:"volume_path"` } // baseEnvironmentDatabricksAI names the Databricks AI environment to the server. const baseEnvironmentDatabricksAI = "databricks-ai" -// modelPackageFile is a source file the entry file imports, by path relative to -// code_source.root_path. -type modelPackageFile struct { - Path string `json:"path"` - Content string `json:"content"` -} - type modelPackageEnvironment struct { Runtime string `json:"runtime"` Dependencies []string `json:"dependencies,omitempty"` @@ -163,9 +156,7 @@ func (e *finalizeError) Error() string { return e.err.Error() } func (e *finalizeError) Unwrap() error { return e.err } // registrationRequest builds the request that registers cfg's checkpoint under key. -// code is the collected code of model.entry, and is nil when a command serves the -// checkpoint. -func (c *deployConfig) registrationRequest(key string, code *modelPackagePythonModel) (registerModelPackageRequest, error) { +func (c *deployConfig) registrationRequest(key string) registerModelPackageRequest { req := registerModelPackageRequest{ RegisteredModel: c.RegisteredModel, IdempotencyKey: key, @@ -173,29 +164,33 @@ func (c *deployConfig) registrationRequest(key string, code *modelPackagePythonM if c.MLflowExperimentPath != nil { req.ExperimentPath = *c.MLflowExperimentPath } - checkpoint := &modelPackageCheckpoint{VolumePath: *c.Model.Path, Environment: c.servingEnvironment()} - if c.Task != nil { - checkpoint.Task = *c.Task + if c.variant() == variantDedicated { + req.DedicatedCheckpoint = &dedicatedCheckpoint{VolumePath: *c.Checkpoint} + return req } - if c.Command != nil { - checkpoint.ServingCommand = *c.Command - } else { - checkpoint.PythonModel = &modelPackagePythonModel{ - EntryFile: code.EntryFile, - EntrySource: code.EntrySource, - Files: code.Files, - Streamable: c.Model.Streamable, - } - if c.Model.ModelConfig != nil { - raw, err := modelConfigJSON(c.Model.ModelConfig) - if err != nil { - return registerModelPackageRequest{}, err - } - checkpoint.PythonModel.ModelConfig = raw - } + req.Checkpoint = &modelPackageCheckpoint{ + VolumePath: *c.Checkpoint, + ServingCommand: *c.Command, + Task: *c.Task, + Environment: c.servingEnvironment(), + } + return req +} + +// servingEnvironment is the environment a command's checkpoint is registered with: +// its runtime, dependencies and base environment. A command is served on +// accelerators, so the runtime is the GPU one. The version was validated when the +// config loaded. +func (c *deployConfig) servingEnvironment() modelPackageEnvironment { + version, ai, _ := c.Environment.runtimeVersion() + environment := modelPackageEnvironment{ + Runtime: "client." + strconv.Itoa(version) + "-gpu", + Dependencies: c.Environment.Dependencies, + } + if ai { + environment.BaseEnvironment = baseEnvironmentDatabricksAI } - req.Checkpoint = checkpoint - return req, nil + return environment } // registerAndFinalize registers req's version and makes it ready to serve, diff --git a/cmd/air/deployregister_test.go b/cmd/air/deployregister_test.go index 0e9840a9fc1..98dbd6aff74 100644 --- a/cmd/air/deployregister_test.go +++ b/cmd/air/deployregister_test.go @@ -394,26 +394,33 @@ func TestRegisterAndFinalizeStopsWhenCanceled(t *testing.T) { assert.Empty(t, fake.finalizeRequests) } -func TestRegisterPythonCheckpointRequestJSON(t *testing.T) { - streamable := true - raw, err := json.Marshal(registerModelPackageRequest{ - RegisteredModel: "a.b.c", - IdempotencyKey: "k", - Checkpoint: &modelPackageCheckpoint{ - VolumePath: "/Volumes/a/b/c/w", - PythonModel: &modelPackagePythonModel{EntryFile: "model.py", EntrySource: "x = 1\n", ModelConfig: `{"k":"v"}`, Streamable: &streamable}, - Environment: modelPackageEnvironment{Runtime: "client.6-gpu", BaseEnvironment: "databricks-ai"}, - }, - }) +func TestRegistrationRequestForCommandJSON(t *testing.T) { + cfg, err := loadDeployConfigText(t, commandDeployConfig+"mlflow_experiment_path: /Shared/qwen\n") + require.NoError(t, err) + raw, err := json.Marshal(cfg.registrationRequest("k")) require.NoError(t, err) assert.JSONEq(t, `{ - "registered_model": "a.b.c", + "registered_model": "main.models.qwen", "idempotency_key": "k", "checkpoint": { - "volume_path": "/Volumes/a/b/c/w", - "python_model": {"entry_file": "model.py", "entry_source": "x = 1\n", "model_config": "{\"k\":\"v\"}", "streamable": true}, - "environment": {"runtime": "client.6-gpu", "base_environment": "databricks-ai"} - } + "volume_path": "/Volumes/main/models/weights/qwen3_8b", + "serving_command": "python -m vllm.entrypoints.openai.api_server --model qwen3_8b --port 8080", + "task": "llm/v1/chat", + "environment": {"runtime": "client.6-gpu", "dependencies": ["einops==0.8.0"]} + }, + "experiment_path": "/Shared/qwen" + }`, string(raw)) +} + +func TestRegistrationRequestForDedicatedJSON(t *testing.T) { + cfg, err := loadDeployConfigText(t, dedicatedDeployConfig) + require.NoError(t, err) + raw, err := json.Marshal(cfg.registrationRequest("k")) + require.NoError(t, err) + assert.JSONEq(t, `{ + "registered_model": "main.models.qwen", + "idempotency_key": "k", + "dedicated_checkpoint": {"volume_path": "/Volumes/main/models/weights/qwen3_8b"} }`, string(raw)) }