Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions apps/dev-playground/server/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ import {
serving,
} from "@databricks/appkit";
import { WorkspaceClient } from "@databricks/sdk-experimental";
import { vectorSearch } from "../../../packages/appkit/src/plugins/vector-search";
// TODO: re-enable once vector-search is exported from @databricks/appkit
// import { vectorSearch } from "@databricks/appkit";
import { lakebaseExamples } from "./lakebase-examples-plugin";
import { reconnect } from "./reconnect-plugin";
import { telemetryExamples } from "./telemetry-example-plugin";
Expand All @@ -35,16 +36,17 @@ createApp({
lakebaseExamples(),
files(),
serving(),
vectorSearch({
indexes: {
demo: {
indexName:
process.env.DATABRICKS_VS_INDEX_NAME ?? "catalog.schema.index",
columns: ["id", "text", "title"],
queryType: "hybrid",
},
},
}),
// TODO: re-enable once vector-search is exported from @databricks/appkit
// vectorSearch({
// indexes: {
// demo: {
// indexName:
// process.env.DATABRICKS_VS_INDEX_NAME ?? "catalog.schema.index",
// columns: ["id", "text", "title"],
// queryType: "hybrid",
// },
// },
// }),
Comment thread
atilafassina marked this conversation as resolved.
],
...(process.env.APPKIT_E2E_TEST && { client: createMockClient() }),
}).then((appkit) => {
Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/cache/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { createHash } from "node:crypto";
import { ApiError, WorkspaceClient } from "@databricks/sdk-experimental";
import type { CacheConfig, CacheStorage } from "shared";
import { createLakebasePool } from "@/connectors/lakebase";
import { createLakebasePool } from "../connectors/lakebase";
Comment thread
atilafassina marked this conversation as resolved.
import { AppKitError, ExecutionError, InitializationError } from "../errors";
import { createLogger } from "../logging/logger";
import type { Counter, TelemetryProvider } from "../telemetry";
Expand Down
14 changes: 7 additions & 7 deletions packages/appkit/src/connectors/lakebase-v1/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,6 @@ import { randomUUID } from "node:crypto";
import type { WorkspaceClient } from "@databricks/sdk-experimental";
import { ApiClient, Config } from "@databricks/sdk-experimental";
import pg from "pg";
import {
type Counter,
type Histogram,
SpanStatusCode,
TelemetryManager,
type TelemetryProvider,
} from "@/telemetry";
import {
AppKitError,
AuthenticationError,
Expand All @@ -17,6 +10,13 @@ import {
ValidationError,
} from "../../errors";
import { createLogger } from "../../logging/logger";
import {
type Counter,
type Histogram,
SpanStatusCode,
TelemetryManager,
type TelemetryProvider,
} from "../../telemetry";
import { deepMerge } from "../../utils";
import { lakebaseV1Defaults } from "./defaults";
import type {
Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/connectors/lakebase/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ import {
type LakebasePoolConfig,
} from "@databricks/lakebase";
import type { Pool } from "pg";
import { createLogger } from "@/logging/logger";
import { createLogger } from "../../logging/logger";

/**
* Create a Lakebase pool with appkit's logger integration.
Expand Down
4 changes: 2 additions & 2 deletions packages/appkit/src/connectors/vector-search/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ export class VectorSearchConnector {
const response = (await workspaceClient.apiClient.request({
method: "POST",
path: `/api/2.0/vector-search/indexes/${params.indexName}/query`,
body,
payload: body,
headers: new Headers({ "Content-Type": "application/json" }),
raw: false,
query: {},
Expand Down Expand Up @@ -149,7 +149,7 @@ export class VectorSearchConnector {
const response = (await workspaceClient.apiClient.request({
method: "POST",
path: `/api/2.0/vector-search/indexes/${params.indexName}/query-next-page`,
body: {
payload: {
endpoint_name: params.endpointName,
page_token: params.pageToken,
},
Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/plugin/dev-reader.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { randomUUID } from "node:crypto";
import type { TunnelConnection } from "shared";
import { isRemoteTunnelAllowedByEnv } from "@/plugins/server/remote-tunnel/gate";
import { TunnelError } from "../errors";
import { createLogger } from "../logging/logger";
import { isRemoteTunnelAllowedByEnv } from "../plugins/server/remote-tunnel/gate";
Comment thread
atilafassina marked this conversation as resolved.

const logger = createLogger("plugin:dev-reader");

Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/plugins/server/vite-dev-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ import fs from "node:fs";
import path from "node:path";
import type express from "express";
import type { ViteDevServer as ViteDevServerType } from "vite";
import { mergeConfigDedup } from "@/utils";
import { ServerError } from "../../errors";
import { createLogger } from "../../logging/logger";
import { appKitServingTypesPlugin } from "../../type-generator/serving/vite-plugin";
import { appKitTypesPlugin } from "../../type-generator/vite-plugin";
import { mergeConfigDedup } from "../../utils";
Comment thread
atilafassina marked this conversation as resolved.
import { BaseServer } from "./base-server";
import type { PluginClientConfigs, PluginEndpoints } from "./utils";

Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/plugins/vector-search/index.ts
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
export * from "./vector-search";
export * from "./types";
export * from "./vector-search";
1 change: 1 addition & 0 deletions packages/appkit/src/plugins/vector-search/manifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
"$schema": "https://databricks.github.io/appkit/schemas/plugin-manifest.schema.json",
"name": "vector-search",
"displayName": "Vector Search Plugin",
"hidden": true,
"description": "Query Databricks Vector Search indexes with built-in hybrid search, reranking, and pagination",
"resources": {
"required": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ vi.mock("../../../logging/logger", () => ({
event: () => ({
setComponent: vi.fn().mockReturnThis(),
setContext: vi.fn().mockReturnThis(),
setExecution: vi.fn().mockReturnThis(),
}),
}),
}));
Expand Down Expand Up @@ -215,7 +216,7 @@ describe("VectorSearchPlugin", () => {
}),
);

const callBody = mockRequest.mock.calls[0][0].body;
const callBody = mockRequest.mock.calls[0][0].payload;
expect(callBody.query_text).toBe("test query");
expect(callBody.query_type).toBe("HYBRID");
expect(callBody.num_results).toBe(10);
Expand Down Expand Up @@ -250,7 +251,7 @@ describe("VectorSearchPlugin", () => {
filters: { category: ["books"] },
});

const callBody = mockRequest.mock.calls[0][0].body;
const callBody = mockRequest.mock.calls[0][0].payload;
expect(callBody.filters).toEqual({ category: ["books"] });
});

Expand All @@ -267,7 +268,7 @@ describe("VectorSearchPlugin", () => {
await plugin.setup();
await plugin.query("test", { queryText: "test" });

const callBody = mockRequest.mock.calls[0][0].body;
const callBody = mockRequest.mock.calls[0][0].payload;
expect(callBody.reranker.model).toBe("databricks_reranker");
expect(callBody.reranker.parameters.columns_to_rerank).toEqual([
"title",
Expand All @@ -290,7 +291,7 @@ describe("VectorSearchPlugin", () => {
await plugin.query("test", { queryText: "test" });

expect(mockEmbeddingFn).toHaveBeenCalledWith("test");
const callBody = mockRequest.mock.calls[0][0].body;
const callBody = mockRequest.mock.calls[0][0].payload;
expect(callBody.query_vector).toEqual([0.1, 0.2, 0.3]);
expect(callBody.query_text).toBeUndefined();
});
Expand Down
2 changes: 1 addition & 1 deletion packages/appkit/src/plugins/vector-search/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import type { BasePluginConfig } from "shared";

export interface IVectorSearchConfig extends BasePluginConfig {
timeout?: number;
indexes: Record<string, IndexConfig>;
indexes?: Record<string, IndexConfig>;
}

export interface IndexConfig {
Expand Down
35 changes: 22 additions & 13 deletions packages/appkit/src/plugins/vector-search/vector-search.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type express from "express";
import type { IAppRouter, PluginExecutionSettings } from "shared";
import { VectorSearchConnector } from "../../connectors";
import { VectorSearchConnector } from "../../connectors/vector-search/client";
import type { VsRawResponse } from "../../connectors/vector-search/types";
import { getWorkspaceClient } from "../../context";
import { createLogger } from "../../logging/logger";
Expand Down Expand Up @@ -40,6 +40,11 @@ export class VectorSearchPlugin extends Plugin<IVectorSearchConfig> {
}

async setup(): Promise<void> {
if (!this.config.indexes || Object.keys(this.config.indexes).length === 0) {
throw new Error(
'VectorSearchPlugin requires at least one index in "indexes" config',
);
}
Comment thread
atilafassina marked this conversation as resolved.
for (const [alias, idx] of Object.entries(this.config.indexes)) {
if (!idx.indexName) {
throw new Error(
Expand Down Expand Up @@ -109,11 +114,13 @@ export class VectorSearchPlugin extends Plugin<IVectorSearchConfig> {
querySettings,
);

if (result === undefined) {
res.status(500).json({ error: "Query failed", plugin: this.name });
if (!result.ok) {
res
.status(result.status)
.json({ error: result.message, plugin: this.name });
return;
}
res.json(this._parseResponse(result, prepared.queryType));
res.json(this._parseResponse(result.data, prepared.queryType));
} catch (error) {
this._handleError(res, error, "Query failed");
}
Expand Down Expand Up @@ -177,14 +184,14 @@ export class VectorSearchPlugin extends Plugin<IVectorSearchConfig> {
querySettings,
);

if (result === undefined) {
if (!result.ok) {
res
.status(500)
.json({ error: "Next-page query failed", plugin: this.name });
.status(result.status)
.json({ error: result.message, plugin: this.name });
return;
}
res.json(
this._parseResponse(result, indexConfig.queryType ?? "hybrid"),
this._parseResponse(result.data, indexConfig.queryType ?? "hybrid"),
);
} catch (error) {
this._handleError(res, error, "Next-page query failed");
Expand All @@ -196,7 +203,7 @@ export class VectorSearchPlugin extends Plugin<IVectorSearchConfig> {
name: "getConfig",
method: "get",
path: "/:alias/config",
handler: (req: express.Request, res: express.Response) => {
handler: async (req: express.Request, res: express.Response) => {
const { alias } = req.params;
const indexConfig = this._resolveIndex(alias);
if (!indexConfig) {
Expand Down Expand Up @@ -249,11 +256,13 @@ export class VectorSearchPlugin extends Plugin<IVectorSearchConfig> {
querySettings,
);

if (result === undefined) {
throw new Error(`Vector search query failed for index "${alias}"`);
if (!result.ok) {
throw new Error(
`Vector search query failed for index "${alias}": ${result.message}`,
);
}

return this._parseResponse(result, prepared.queryType);
return this._parseResponse(result.data, prepared.queryType);
}

async shutdown(): Promise<void> {
Expand All @@ -267,7 +276,7 @@ export class VectorSearchPlugin extends Plugin<IVectorSearchConfig> {
}

private _resolveIndex(alias: string): IndexConfig | undefined {
return this.config.indexes[alias];
return this.config.indexes?.[alias];
}

private async _prepareQuery(
Expand Down
Loading