Skip to content

feat(aitools): record install error categories from CLI JSON output #2188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rclarey wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat(aitools): record install error categories from CLI JSON output #2188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b72bf51
feat(aitools): record install error categories from CLI JSON output
rclarey Sep 10, 2026
839fc72
Merge branch 'main' into aitools-install-error-categories
rclarey Sep 10, 2026
e2ed1a1
Merge branch 'main' into aitools-install-error-categories
rclarey Sep 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
242 changes: 236 additions & 6 deletions packages/databricks-vscode/src/aitools/AiToolsManager.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {commands, Uri} from "vscode";
import path from "path";
import {
AiToolsAgent,
AiToolsInstallOutput,
AiToolsListResult,
CliWrapper,
ProcessError,
Expand Down Expand Up @@ -67,6 +68,17 @@ function listResult(
};
}

const installSuccessOutput: AiToolsInstallOutput = {
scope: "global",
agents: [
{
name: "claude-code",
delivery: "plugin",
status: "installed",
},
],
};

// The project root the WorkspaceFolderManager mock reports; the global root is
// the real home dir (the manager derives it from getHomedir()). The injected
// loader maps a state-file path back to its scope by the projectDir prefix.
Expand Down Expand Up @@ -421,6 +433,7 @@ describe(__filename, () => {
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenCall(async () => {
loaders.global = loadSuccess;
return {output: installSuccessOutput, error: undefined};
});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
Expand All @@ -441,6 +454,223 @@ describe(__filename, () => {
assert.strictEqual(manager.model.state.updateStatus, "upToDate");
});

it("records a clean success from the CLI's JSON result", async () => {
const loaders: ScopeLoaders = {};
const {manager, mockCli, stubTelemetry} = setup(loaders);
when(
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenCall(async () => {
loaders.global = loadSuccess;
return {output: installSuccessOutput, error: undefined};
});
when(mockCli.aitoolsList(anything())).thenResolve(listResult([]));

await manager.install("global", "sidePane", ["claude-code"]);

const [installEvent] = stubTelemetry.eventsOfType(
Events.AITOOLS_INSTALL
);
assert.strictEqual(installEvent.props.result, "success");
// No error fields on a clean install.
assert.strictEqual(installEvent.props.globalErrorCategory, undefined);
assert.strictEqual(installEvent.props.agentErrors, undefined);
});

it("records per-agent error categories and surfaces a JSON-reported failure", async () => {
const loaders: ScopeLoaders = {};
const {manager, mockCli, stubTelemetry} = setup(loaders);
// The CLI exits non-zero on failure but still prints the structured
// result, so aitoolsInstall resolves with it rather than throwing.
when(
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenResolve({
output: {
scope: "global",
agents: [
{
name: "claude-code",
delivery: "plugin",
status: "installed",
},
{
name: "copilot",
delivery: "plugin",
status: "failed",
error_category: "PLUGIN_INSTALL_FAILED",
message: "gh CLI is not on PATH",
},
],
},
error: new Error("Command failed"),
});
when(mockCli.aitoolsList(anything())).thenResolve(listResult([]));

// A reported failure is surfaced to the command layer as a ProcessError
// whose (local-only) message folds in the failing agent's CLI message, so
// "Show Logs" has real detail even though --output json leaves stderr empty.
await assert.rejects(
() =>
manager.install("global", "sidePane", [
"claude-code",
"copilot",
]),
(e: unknown) =>
e instanceof ProcessError &&
e.message.includes("copilot: gh CLI is not on PATH")
);

const [installEvent] = stubTelemetry.eventsOfType(
Events.AITOOLS_INSTALL
);
assert.strictEqual(installEvent.props.result, "error");
// Only the failing agent's category is recorded, keyed by agent id, with
// no free-form message.
assert.deepStrictEqual(installEvent.props.agentErrors, {
copilot: "PLUGIN_INSTALL_FAILED",
});
assert.strictEqual(installEvent.props.globalErrorCategory, undefined);
});

it("records a top-level error category from the CLI's JSON result", async () => {
const loaders: ScopeLoaders = {};
const {manager, mockCli, stubTelemetry} = setup(loaders);
when(
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenResolve({
output: {
scope: "global",
agents: [],
error: 'skill "__nope__" not found',
error_category: "SKILL_NOT_FOUND",
},
error: new Error("Command failed"),
});
when(mockCli.aitoolsList(anything())).thenResolve(listResult([]));

await assert.rejects(
() => manager.install("global", "sidePane", ["claude-code"]),
(e: unknown) =>
e instanceof ProcessError &&
e.message.includes('skill "__nope__" not found')
);

const [installEvent] = stubTelemetry.eventsOfType(
Events.AITOOLS_INSTALL
);
assert.strictEqual(installEvent.props.result, "error");
assert.strictEqual(
installEvent.props.globalErrorCategory,
"SKILL_NOT_FOUND"
);
assert.strictEqual(installEvent.props.agentErrors, undefined);
});

it("handles unparseable JSON output (success)", async () => {
const loaders: ScopeLoaders = {};
const {manager, mockCli, stubTelemetry} = setup(loaders);
// An older CLI ignores --output json and prints text; aitoolsInstall
// resolves with undefined on its success exit.
when(
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenCall(async () => {
loaders.global = loadSuccess;
return {output: undefined, error: undefined};
});
when(mockCli.aitoolsList(anything())).thenResolve(listResult([]));

await manager.install("global", "sidePane", ["claude-code"]);

const [installEvent] = stubTelemetry.eventsOfType(
Events.AITOOLS_INSTALL
);
assert.strictEqual(installEvent.props.result, "success");
assert.strictEqual(installEvent.props.agentErrors, undefined);
});

it("handles unparseable JSON output (failure)", async () => {
const loaders: ScopeLoaders = {};
const {manager, mockCli, stubTelemetry} = setup(loaders);
// An older CLI ignores --output json and prints text; aitoolsInstall
// resolves with undefined on its success exit.
when(
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenCall(async () => {
loaders.global = loadSuccess;
return {output: undefined, error: new Error("Command failed")};
});
when(mockCli.aitoolsList(anything())).thenResolve(listResult([]));

await assert.rejects(
() => manager.install("global", "sidePane", ["claude-code"]),
(e: unknown) =>
e instanceof Error && e.message.includes("Command failed")
);

const [installEvent] = stubTelemetry.eventsOfType(
Events.AITOOLS_INSTALL
);
assert.strictEqual(installEvent.props.result, "error");
assert.strictEqual(installEvent.props.agentErrors, undefined);
});

it("reconciles per-agent state after a JSON-reported partial failure", async () => {
// A partial failure still lands some tools, so the panel must be
// reconciled afterwards. `loaders.global` resolving means the install
// wrote its state file (some agent succeeded); the manager's `finally`
// should then run detect + resolve and pick up the CLI's real state.
const loaders: ScopeLoaders = {};
const {manager, mockCli} = setup(loaders);
when(
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenCall(async () => {
loaders.global = loadSuccess;
return {
output: {
scope: "global",
agents: [
{
name: "claude-code",
delivery: "plugin",
status: "installed",
},
{
name: "copilot",
delivery: "plugin",
status: "failed",
error_category: "PLUGIN_INSTALL_FAILED",
},
],
},
error: new Error("Command failed"),
};
});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
{
name: "databricks-core",
latest_version: "0.1.0",
installed: {global: "0.1.0"},
},
])
);

await assert.rejects(
() =>
manager.install("global", "sidePane", [
"claude-code",
"copilot",
]),
ProcessError
);

// The reconcile ran despite the throw: `aitoolsList` was consumed and the
// model reflects the CLI's real post-install state rather than staying
// stale.
verify(mockCli.aitoolsList(anything())).atLeast(1);
assert.strictEqual(manager.model.state.installLocation, "global");
assert.strictEqual(manager.model.state.updateStatus, "upToDate");
});

describe("Cursor install flow", () => {
let originalExecuteCommand: typeof commands.executeCommand;
let executed: Array<{command: string; args: any[]}>;
Expand Down Expand Up @@ -478,7 +708,7 @@ describe(__filename, () => {
anything(),
anything()
)
).thenResolve();
).thenResolve({output: installSuccessOutput, error: undefined});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
{
Expand Down Expand Up @@ -561,7 +791,7 @@ describe(__filename, () => {
anything(),
anything()
)
).thenResolve();
).thenResolve({output: installSuccessOutput, error: undefined});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
{
Expand Down Expand Up @@ -634,7 +864,7 @@ describe(__filename, () => {
anything(),
anything()
)
).thenResolve();
).thenResolve({output: installSuccessOutput, error: undefined});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
{
Expand Down Expand Up @@ -691,7 +921,7 @@ describe(__filename, () => {
anything(),
anything()
)
).thenReject(new ProcessError("boom", 1));
).thenResolve({output: undefined, error: new Error("boom!")});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
{
Expand All @@ -716,9 +946,8 @@ describe(__filename, () => {
when(
mockCli.aitoolsInstall("global", anything(), anything(), anything())
).thenCall(async () => {
// Simulate a partial install: some tools landed before the failure.
loaders.global = loadSuccess;
throw new ProcessError("boom", 1);
return {output: undefined, error: new Error("boom")};
});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
Expand Down Expand Up @@ -1008,6 +1237,7 @@ describe(__filename, () => {
)
).thenCall(async () => {
loaders.global = loadSuccess;
return {output: installSuccessOutput, error: undefined};
});
when(mockCli.aitoolsList(anything())).thenResolve(
listResult([
Expand Down
Loading
Loading