Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions acceptance/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,10 @@ func testAccept(t *testing.T, inprocessMode bool, selectedTests []string, skipTo
// from the host shell would move the venv the localenv tests expect.
os.Unsetenv("UV_PROJECT_ENVIRONMENT") //nolint:usetesting // t.Setenv cannot unset

// Agent skills set DATABRICKS_SKILLS_INVOKED (experimental skill tracking),
// so a host value would add skill/ pairs to recorded user agents.
os.Unsetenv("DATABRICKS_SKILLS_INVOKED") //nolint:usetesting // t.Setenv cannot unset

if !requirePrerequisites(t) {
// Don't run the suite against a stale toolchain; the failed subtest
// has already marked the parent test as failed.
Expand Down
2 changes: 2 additions & 0 deletions acceptance/cmd/skills-invoked/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
bundle:
name: skills-invoked
1 change: 1 addition & 0 deletions acceptance/cmd/skills-invoked/out.test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Cloud = false
36 changes: 36 additions & 0 deletions acceptance/cmd/skills-invoked/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@

=== Env var adds the skills to the user agent on a workspace GET

>>> [CLI] current-user me

>>> print_requests.py --get //scim/v2/Me

=== Env var adds the skills on databricks api

>>> [CLI] api get /api/2.1/clusters/list

>>> print_requests.py --get //clusters/list

=== Env var adds the skills from bundle commands

>>> [CLI] bundle validate

>>> print_requests.py --get //scim/v2/Me

=== Invalid entries are dropped and the rest is normalized

>>> [CLI] current-user me

>>> print_requests.py --get //scim/v2/Me

=== No skills when unset

>>> [CLI] current-user me

>>> print_requests.py --get //scim/v2/Me

=== No skills when telemetry is disabled

>>> [CLI] current-user me

>>> print_requests.py --get //scim/v2/Me
33 changes: 33 additions & 0 deletions acceptance/cmd/skills-invoked/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Experimental skill-usage tracking: DATABRICKS_SKILLS_INVOKED adds skill/<name> pairs to the user agent.
# The rest of the user agent varies by machine, so each request dump goes to a LOG.* file (kept for
# diagnosis, excluded from the diff) and only the pairs are asserted.

title "Env var adds the skills to the user agent on a workspace GET"
DATABRICKS_SKILLS_INVOKED=databricks-core,databricks-jobs trace $CLI current-user me > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.me
contains.py " skill/databricks-core skill/databricks-jobs" < LOG.me > /dev/null

title "Env var adds the skills on databricks api"
DATABRICKS_SKILLS_INVOKED=databricks-core MSYS_NO_PATHCONV=1 trace $CLI api get /api/2.1/clusters/list > /dev/null
trace print_requests.py --get //clusters/list > LOG.api
contains.py " skill/databricks-core" < LOG.api > /dev/null

title "Env var adds the skills from bundle commands"
DATABRICKS_SKILLS_INVOKED=databricks-dabs trace $CLI bundle validate > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.bundle
contains.py " skill/databricks-dabs" < LOG.bundle > /dev/null

title "Invalid entries are dropped and the rest is normalized"
DATABRICKS_SKILLS_INVOKED=' Databricks-Core , bad name!,,databricks-core,databricks:databricks-jobs ' trace $CLI current-user me > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.normalized
contains.py " skill/databricks-core" "!skill/databricks-core skill/" "!bad" "!databricks:" < LOG.normalized > /dev/null

title "No skills when unset"
trace $CLI current-user me > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.unset
contains.py "!skill/" < LOG.unset > /dev/null

title "No skills when telemetry is disabled"
DATABRICKS_SKILLS_INVOKED=databricks-core DATABRICKS_CLI_DISABLE_TELEMETRY=1 trace $CLI current-user me > /dev/null
trace print_requests.py --get //scim/v2/Me > LOG.disabled
contains.py "!skill/" < LOG.disabled > /dev/null
5 changes: 5 additions & 0 deletions acceptance/cmd/skills-invoked/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
RecordRequests = true

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nice to have] With DATABRICKS_CLI_DISABLE_TELEMETRY=1 exported in the invoking shell, this fixture inherits the opt-out through auth.ProcessEnv. The CLI correctly omits the skills, so the first positive contains.py assertion fails. Set Env.DATABRICKS_CLI_DISABLE_TELEMETRY = "" in this fixture; the final scenario can still set it to 1 inline.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in d6888e2: the fixture sets Env.DATABRICKS_CLI_DISABLE_TELEMETRY = "", and the last scenario still sets it to 1 inline. I checked it passes with the opt-out exported in my shell.

IncludeRequestHeaders = ["User-Agent"]
Ignore = [".databricks"]
# A host opt-out would otherwise remove the skills the positive cases assert.
Env.DATABRICKS_CLI_DISABLE_TELEMETRY = ""
8 changes: 8 additions & 0 deletions cmd/root/export_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@ import (
"github.com/spf13/cobra"
)

// Experimental skill-usage tracking, see user_agent_skills.go.
var (
ParseSkills = parseSkills
WithSkillsInUserAgent = withSkillsInUserAgent
SkillsInvokedEnvVar = skillsInvokedEnvVar
MaxSkillsUserAgentLength = maxSkillsUserAgentLength
)

// BuildAll builds every command in t and returns the root command, for tests
// that walk the whole tree.
func BuildAll(ctx context.Context, t *Tree) *cobra.Command {
Expand Down
1 change: 1 addition & 0 deletions cmd/root/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ func newRootCommand(ctx context.Context) *cobra.Command {
ctx = installer.WithAiDevKitInUserAgent(ctx)
ctx = installer.WithCallingAgentToolingInUserAgent(ctx)
ctx = InjectTestPidToUserAgent(ctx)
ctx = withSkillsInUserAgent(ctx)
cmd.SetContext(ctx)

// Recommend installing Databricks AI tooling to the calling coding agent
Expand Down
83 changes: 83 additions & 0 deletions cmd/root/user_agent_skills.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package root

import (
"context"
"regexp"
"slices"
"strings"
"sync"

"github.com/databricks/cli/libs/env"
"github.com/databricks/cli/libs/log"
"github.com/databricks/cli/libs/telemetry"
"github.com/databricks/databricks-sdk-go/useragent"
)

// Experimental skill-usage tracking: the databricks-core agent skill
// (https://github.com/databricks/databricks-agent-skills/tree/main/skills/databricks-core)
// tells agents to prefix CLI calls with this env var, set to the comma-separated
// bare names of the skills they loaded. It may change or be removed without
// notice. It is an env var rather than a flag so removing it doesn't change the
// CLI's interface, and because older CLIs ignore unknown env vars but reject
// unknown flags.
const skillsInvokedEnvVar = "DATABRICKS_SKILLS_INVOKED"

// skillUserAgentKey is repeated once per skill: "skill/databricks-core skill/databricks-jobs".
const skillUserAgentKey = "skill"

const (
maxSkillLength = 64

// maxSkillsUserAgentLength bounds all " skill/<name>" pairs together. It
// fits five of the longest current skill names (45-byte pairs). The usage
// logs that feed skill analytics cap the user agent at 512 characters, and
// the rest of the CLI's user agent (auth/, agent/ and other pairs, some
// appended after these) takes about 200, so the skills must leave room for it.
maxSkillsUserAgentLength = 225
)

// skillNamePattern is a subset of what the SDK accepts in a user agent value,
// so a valid name can never make useragent.InContext panic.
var skillNamePattern = sync.OnceValue(func() *regexp.Regexp {
return regexp.MustCompile(`^[a-z0-9][a-z0-9_.-]*$`)
})

// parseSkills normalizes a comma-separated list of skill names: entries are
// trimmed, lowercased and deduplicated, invalid ones are dropped, and names are
// kept in order while their pairs fit in maxSkillsUserAgentLength. Invalid input is only logged at debug level because this
// is telemetry and must never fail or warn a command.
func parseSkills(ctx context.Context, raw string) []string {
var skills []string
size := 0
for entry := range strings.SplitSeq(raw, ",") {
name := strings.ToLower(strings.TrimSpace(entry))
pairSize := len(" "+skillUserAgentKey+"/") + len(name)
switch {
case name == "":
case len(name) > maxSkillLength || !skillNamePattern().MatchString(name):
log.Debugf(ctx, "dropping invalid %s entry %q", skillsInvokedEnvVar, entry)
case slices.Contains(skills, name):
case size+pairSize > maxSkillsUserAgentLength:
log.Debugf(ctx, "dropping %s entry %q: skills exceed %d user agent bytes", skillsInvokedEnvVar, name, maxSkillsUserAgentLength)
default:
skills = append(skills, name)
size += pairSize
}
}
return skills
}

// withSkillsInUserAgent adds one skill/<name> pair per skill in
// skillsInvokedEnvVar (experimental tracking), unless the user opted out of
// telemetry.
func withSkillsInUserAgent(ctx context.Context) context.Context {
if telemetry.Disabled(ctx) {
return ctx
}
for _, skill := range parseSkills(ctx, env.Get(ctx, skillsInvokedEnvVar)) {
// Each skill adds its own pair to the same context; this is accumulation,
// not the accidental nesting fatcontext flags.
ctx = useragent.InContext(ctx, skillUserAgentKey, skill) //nolint:fatcontext
}
return ctx
}
96 changes: 96 additions & 0 deletions cmd/root/user_agent_skills_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package root_test

import (
"fmt"
"strings"
"testing"

"github.com/databricks/cli/cmd/root"
"github.com/databricks/cli/libs/env"
"github.com/databricks/databricks-sdk-go/useragent"
"github.com/stretchr/testify/assert"
)

// skillNames returns n distinct skill names of exactly length bytes.
func skillNames(n, length int) []string {
names := make([]string, 0, n)
for i := range n {
prefix := fmt.Sprintf("s%02d", i)
names = append(names, prefix+strings.Repeat("x", length-len(prefix)))
}
return names
}

// pairsSize is the length of " skill/<name>" for every name.
func pairsSize(names []string) int {
size := 0
for _, name := range names {
size += len(" skill/") + len(name)
}
return size
}

func TestParseSkills(t *testing.T) {
// 38-byte names (the longest current skill) make 45-byte pairs: five fill
// 225 bytes exactly and a sixth doesn't fit.
fiveFit := skillNames(5, 38)
sixTooMany := skillNames(6, 38)
// 64-, 64-, 64- and 22-byte pairs make 214 bytes, so " skill/abcd" (11)
// lands exactly on 225 and nothing more fits.
exact := []string{strings.Repeat("a", 57), strings.Repeat("b", 57), strings.Repeat("c", 57), "databricks-core", "abcd"}

tests := []struct {
name string
raw string
want []string
}{
{"empty", "", nil},
{"single", "databricks-core", []string{"databricks-core"}},
{"multiple", "a,b,c", []string{"a", "b", "c"}},
{"trims and lowercases", " Databricks-Core , JOBS ", []string{"databricks-core", "jobs"}},
{"drops empty entries", ",,a,,", []string{"a"}},
{"allows underscore and dot", "a_b.c-d", []string{"a_b.c-d"}},
{"drops disallowed skill-name characters", "ok,bad name,a:b,a/b,a+b", []string{"ok"}},
{"drops leading punctuation", "-a,.b,_c", nil},
{"dedupes preserving first occurrence", "b,a,b,a,c", []string{"b", "a", "c"}},
{"dedupes after normalization", "Core,core, CORE", []string{"core"}},
{"drops entry longer than 64 bytes", strings.Repeat("a", 65) + ",ok", []string{"ok"}},
{"keeps entry of exactly 64 bytes", strings.Repeat("a", 64), []string{strings.Repeat("a", 64)}},
{"keeps five of the longest skill names", strings.Join(fiveFit, ","), fiveFit},
{"drops pairs over the budget", strings.Join(sixTooMany, ","), sixTooMany[:5]},
{"fills the budget exactly", strings.Join(exact, ",") + ",d", exact},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := root.ParseSkills(t.Context(), tt.raw)
assert.Equal(t, tt.want, got)
assert.LessOrEqual(t, pairsSize(got), root.MaxSkillsUserAgentLength)
})
}
}

func TestWithSkillsInUserAgent(t *testing.T) {
tests := []struct {
name string
skills string
disableTelemetry string
want string
}{
{"unset", "", "", ""},
{"set", "databricks-core,databricks-jobs", "", "skill/databricks-core skill/databricks-jobs"},
{"telemetry disabled", "databricks-core", "1", ""},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Always set both: an empty value in the context overrides the host environment.
ctx := env.Set(t.Context(), root.SkillsInvokedEnvVar, tt.skills)
ctx = env.Set(ctx, "DATABRICKS_CLI_DISABLE_TELEMETRY", tt.disableTelemetry)
ua := useragent.FromContext(root.WithSkillsInUserAgent(ctx))
if tt.want == "" {
assert.NotContains(t, ua, "skill/")
} else {
assert.Contains(t, ua, tt.want)
}
})
}
}
Loading