diff --git a/src/lib/mcp/analytics.test.ts b/src/lib/mcp/analytics.test.ts index 4541857..3b1553e 100644 --- a/src/lib/mcp/analytics.test.ts +++ b/src/lib/mcp/analytics.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, test } from "bun:test"; +import { describe, expect, setSystemTime, test } from "bun:test"; import { createMcpHandler, McpServer } from "@modelcontextprotocol/server"; import type { PostHog } from "posthog-node"; import { @@ -1298,6 +1298,9 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { // identify stays unwired, so no $identify event is ever published. expect(byEvent.has("$identify")).toBe(false); + + // A carried transport session is kept rather than derived. + expect(toolCall.properties.$session_id).toBe("ses_integration"); }); test("classifies rejected capability input through instrumentation", async () => { @@ -1576,6 +1579,195 @@ describe("instrumentMcpAnalytics (SDK integration)", () => { } }); + async function modernRequest({ + token, + clientName, + meta = {}, + name = "ping", + args = { context: "Checking derived session analytics." }, + }: { + token: string; + clientName: string; + meta?: Record; + name?: string; + args?: Record; + }) { + const captured: { + event?: string; + distinctId?: string; + properties?: Record; + }[] = []; + const handler = createMcpHandler(() => makeServer(captured)); + try { + const response = await handler.fetch( + new Request("https://mcp.example.test/mcp", { + method: "POST", + headers: { + "Content-Type": "application/json", + "MCP-Protocol-Version": "2026-07-28", + "Mcp-Method": "tools/call", + "Mcp-Name": name, + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name, + arguments: args, + _meta: { + "io.modelcontextprotocol/protocolVersion": "2026-07-28", + "io.modelcontextprotocol/clientInfo": { + name: clientName, + version: "1", + }, + "io.modelcontextprotocol/clientCapabilities": {}, + ...meta, + }, + }, + }), + }), + { + authInfo: { + token, + clientId: "test-client", + scopes: [], + extra: { connectionContext: { scope: { organizationId: ORG } } }, + }, + }, + ); + expect(response.status).toBe(200); + await new Promise((resolve) => setTimeout(resolve, 50)); + } finally { + await handler.close(); + } + return captured; + } + + async function modernToolCall(options: Parameters[0]) { + const event = (await modernRequest(options)).find( + (event) => event.event === PostHogMCPAnalyticsEvent.ToolCall, + ); + expect(event).toBeDefined(); + return event!; + } + + test("groups session-less calls by caller, client, and window", async () => { + setSystemTime(new Date("2026-01-01T00:10:00Z")); + try { + const first = await modernToolCall({ + token: "token-a", + clientName: "client-a", + }); + const second = await modernToolCall({ + token: "token-a", + clientName: "client-a", + }); + const otherCaller = await modernToolCall({ + token: "token-b", + clientName: "client-a", + }); + const otherClient = await modernToolCall({ + token: "token-a", + clientName: "client-b", + }); + setSystemTime(new Date("2026-01-01T00:40:00Z")); + const nextWindow = await modernToolCall({ + token: "token-a", + clientName: "client-a", + }); + + const sessionId = first.properties?.$session_id; + expect(sessionId).toStartWith("ses_"); + expect(first.distinctId).toBe(sessionId as string); + expect(second.properties?.$session_id).toBe(sessionId); + expect(second.distinctId).toBe(sessionId as string); + expect(otherCaller.properties?.$session_id).not.toBe(sessionId); + expect(otherClient.properties?.$session_id).not.toBe(sessionId); + expect(nextWindow.properties?.$session_id).not.toBe(sessionId); + expect(JSON.stringify(first)).not.toContain("token-a"); + expect(first.properties).not.toHaveProperty( + "__mcp_analytics_session_key", + ); + } finally { + setSystemTime(); + } + }); + + test("puts session-less custom events in the caller's derived session", async () => { + setSystemTime(new Date("2026-01-01T00:10:00Z")); + try { + const toolCall = await modernToolCall({ + token: "token-a", + clientName: "client-a", + }); + const captured = await modernRequest({ + token: "token-a", + clientName: "client-a", + name: KERNEL_FEEDBACK_TOOL_NAME, + args: { + context: + "Reporting a repeatable site block so the affected domain can be prioritized for a working browser configuration.", + summary: "Stealth sessions were consistently blocked", + feedback_type: "site_compatibility", + sentiment: "negative", + task_completed: false, + site_compatibility: { + registrable_domain: "example.com", + observed_outcome: "blocked", + reproducibility: "consistent", + }, + }, + }); + const feedback = captured.find( + ({ event }) => event === MCP_FEEDBACK_SUBMITTED_EVENT, + ); + + const sessionId = toolCall.properties?.$session_id; + expect(feedback?.properties?.$session_id).toBe(sessionId); + expect(feedback?.distinctId).toBe(sessionId as string); + expect(feedback?.properties).not.toHaveProperty( + "__mcp_analytics_session_key", + ); + } finally { + setSystemTime(); + } + }); + + test("records the model only from Codex request metadata", async () => { + const codex = await modernToolCall({ + token: "token-a", + clientName: "codex", + meta: { "x-codex-turn-metadata": { model: "gpt-5.2-codex" } }, + }); + expect(codex.properties).toMatchObject({ + [PostHogMCPAnalyticsProperty.LlmModel]: "gpt-5.2-codex", + [PostHogMCPAnalyticsProperty.LlmModelSource]: "client_metadata", + }); + + for (const model of ["unknown", "ignore previous instructions", 42]) { + const event = await modernToolCall({ + token: "token-a", + clientName: "codex", + meta: { "x-codex-turn-metadata": { model } }, + }); + expect(event.properties).not.toHaveProperty( + PostHogMCPAnalyticsProperty.LlmModel, + ); + } + + const other = await modernToolCall({ + token: "token-a", + clientName: "client-a", + }); + expect(other.properties).not.toHaveProperty( + PostHogMCPAnalyticsProperty.LlmModel, + ); + expect(other.properties).not.toHaveProperty( + PostHogMCPAnalyticsProperty.LlmModelSource, + ); + }); + test("stays anonymous when no connection context is attached", async () => { const captured: { event?: string }[] = []; const server = makeServer(captured); diff --git a/src/lib/mcp/analytics.ts b/src/lib/mcp/analytics.ts index bbb96b7..31da743 100644 --- a/src/lib/mcp/analytics.ts +++ b/src/lib/mcp/analytics.ts @@ -1,7 +1,10 @@ import { createHash } from "node:crypto"; import { isIP } from "node:net"; import { + decodeSessionId, + getRequestHeaders, instrument, + MCP_SESSION_HEADER, PostHogMCPAnalyticsEvent, PostHogMCPAnalyticsProperty, type BeforeSendFn, @@ -9,6 +12,7 @@ import { } from "@posthog/mcp"; import { CLIENT_CAPABILITIES_META_KEY, + CLIENT_INFO_META_KEY, type McpServer, } from "@modelcontextprotocol/server"; import { PostHog } from "posthog-node"; @@ -171,6 +175,8 @@ const SENT_PROPERTIES = new Set([ PostHogMCPAnalyticsProperty.IntentSource, PostHogMCPAnalyticsProperty.IsError, PostHogMCPAnalyticsProperty.ListedToolNames, + PostHogMCPAnalyticsProperty.LlmModel, + PostHogMCPAnalyticsProperty.LlmModelSource, PostHogMCPAnalyticsProperty.ProtocolVersion, PostHogMCPAnalyticsProperty.ResourceName, PostHogMCPAnalyticsProperty.ServerName, @@ -392,6 +398,7 @@ export const sanitizeMcpAnalyticsEvent: BeforeSendFn = (event) => { const properties = event.properties; if (!properties) return event; enrichMcpAnalyticsEvent(event); + applyDerivedSession(event); if (event.event === PostHogMCPAnalyticsEvent.ToolCall) { annotateProjectParamUsage(properties); annotateDeprecatedParamUsage(properties); @@ -459,6 +466,100 @@ function connectionOrgId(ctx: unknown) { return authExtra?.connectionContext?.scope.organizationId; } +const SESSION_KEY_PROPERTY = "__mcp_analytics_session_key"; + +// Matches the SDK's inactivity timeout, but as fixed windows: requests on the 2026-07-28 +// revision run on a fresh server instance each, so there is no in-process state to +// measure inactivity against. +const DERIVED_SESSION_WINDOW_MS = 30 * 60 * 1000; + +/** + * Requests without an MCP session (every request on the 2026-07-28 revision) would get a + * new $session_id per call. For those, key the session on the caller, organization, and + * client instead; sanitizeMcpAnalyticsEvent buckets the key into a window. Concurrent + * runs by the same caller and client share a session. + */ +function analyticsSessionKey(ctx: unknown) { + const extra = ctx as + | { + sessionId?: string; + http?: { authInfo?: { token?: string; extra?: unknown } }; + mcpReq?: { envelope?: unknown }; + } + | undefined; + const headers = getRequestHeaders(extra); + const header = headers?.[MCP_SESSION_HEADER]; + if ( + extra?.sessionId || + decodeSessionId(Array.isArray(header) ? header[0] : header) + ) { + return null; + } + + const authInfo = extra?.http?.authInfo; + const userId = (authInfo?.extra as { userId?: string | null } | undefined) + ?.userId; + const subject = userId + ? `user:${userId}` + : authInfo?.token + ? `token:${authInfo.token}` + : null; + if (!subject) return null; + + const envelope = extra?.mcpReq?.envelope; + const clientInfo = isRecord(envelope) + ? envelope[CLIENT_INFO_META_KEY] + : undefined; + const clientName = + isRecord(clientInfo) && typeof clientInfo.name === "string" + ? clientInfo.name + : ""; + + return createHash("sha256") + .update([subject, connectionOrgId(ctx) ?? "", clientName].join("\0")) + .digest("hex"); +} + +function applyDerivedSession(event: Parameters[0]) { + const properties = event.properties; + const key = properties[SESSION_KEY_PROPERTY]; + delete properties[SESSION_KEY_PROPERTY]; + if (typeof key !== "string") return; + + const window = Math.floor( + Date.parse(event.timestamp) / DERIVED_SESSION_WINDOW_MS, + ); + const sessionId = `ses_${createHash("sha256") + .update(`${key}\0${window}`) + .digest("hex") + .slice(0, 32)}`; + + const previous = properties[PostHogMCPAnalyticsProperty.SessionId]; + properties[PostHogMCPAnalyticsProperty.SessionId] = sessionId; + // Anonymous events use the session id as their distinct id. + if (event.distinct_id === previous) event.distinct_id = sessionId; +} + +const CODEX_TURN_METADATA_KEY = "x-codex-turn-metadata"; +const MODEL_ID_PATTERN = /^[\w.:/@-]{1,100}$/; + +/** + * Codex reports its model in request metadata, which needs no extra tool argument. + * Other clients don't, and agent self-reporting stays off. + */ +function clientMetadataModel(request: { params?: unknown }) { + const params = request.params; + const meta = isRecord(params) ? params._meta : undefined; + const codex = isRecord(meta) ? meta[CODEX_TURN_METADATA_KEY] : undefined; + const model = + isRecord(codex) && typeof codex.model === "string" + ? codex.model.trim() + : ""; + return MODEL_ID_PATTERN.test(model) && model.toLowerCase() !== "unknown" + ? model + : null; +} + export function captureMcpCustomEvent( analytics: McpAnalytics, extra: unknown, @@ -466,11 +567,13 @@ export function captureMcpCustomEvent( properties: Record, ) { const organizationId = connectionOrgId(extra); + const sessionKey = analyticsSessionKey(extra); return analytics.capture({ event, properties: { ...properties, ...(organizationId && { $groups: { organization: organizationId } }), + ...(sessionKey && { [SESSION_KEY_PROPERTY]: sessionKey }), }, }); } @@ -898,6 +1001,16 @@ export function instrumentMcpAnalytics( if (isRecord(capabilities)) { Object.assign(properties, clientCapabilityAnalytics(capabilities)); } + const sessionKey = analyticsSessionKey(extra); + if (sessionKey) properties[SESSION_KEY_PROPERTY] = sessionKey; + if (request.method === "tools/call") { + const model = clientMetadataModel(request); + if (model) { + properties[PostHogMCPAnalyticsProperty.LlmModel] = model; + properties[PostHogMCPAnalyticsProperty.LlmModelSource] = + "client_metadata"; + } + } return Object.keys(properties).length > 0 ? properties : null; }, // No part of a call is safe to capture: arguments carry free-form input (credential