Group session-less MCP calls and record Codex model in analytics

src/lib/mcp/analytics.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "bun:test";
+import { describe, expect, setSystemTime, test } from "bun:test";
 import { createMcpHandler, McpServer } from "@modelcontextprotocol/server";
 import type { PostHog } from "posthog-node";
 import {
@@ -1298,6 +1298,9 @@ describe("instrumentMcpAnalytics (SDK integration)", () => {
 
     // identify stays unwired, so no $identify event is ever published.
     expect(byEvent.has("$identify")).toBe(false);
+
+    // A carried transport session is kept rather than derived.
+    expect(toolCall.properties.$session_id).toBe("ses_integration");
   });
 
   test("classifies rejected capability input through instrumentation", async () => {
@@ -1576,6 +1579,195 @@ describe("instrumentMcpAnalytics (SDK integration)", () => {
     }
   });
 
+  async function modernRequest({
+    token,
+    clientName,
+    meta = {},
+    name = "ping",
+    args = { context: "Checking derived session analytics." },
+  }: {
+    token: string;
+    clientName: string;
+    meta?: Record<string, unknown>;
+    name?: string;
+    args?: Record<string, unknown>;
+  }) {
+    const captured: {
+      event?: string;
+      distinctId?: string;
+      properties?: Record<string, unknown>;
+    }[] = [];
+    const handler = createMcpHandler(() => makeServer(captured));
+    try {
+      const response = await handler.fetch(
+        new Request("https://mcp.example.test/mcp", {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "MCP-Protocol-Version": "2026-07-28",
+            "Mcp-Method": "tools/call",
+            "Mcp-Name": name,
+          },
+          body: JSON.stringify({
+            jsonrpc: "2.0",
+            id: 1,
+            method: "tools/call",
+            params: {
+              name,
+              arguments: args,
+              _meta: {
+                "io.modelcontextprotocol/protocolVersion": "2026-07-28",
+                "io.modelcontextprotocol/clientInfo": {
+                  name: clientName,
+                  version: "1",
+                },
+                "io.modelcontextprotocol/clientCapabilities": {},
+                ...meta,
+              },
+            },
+          }),
+        }),
+        {
+          authInfo: {
+            token,
+            clientId: "test-client",
+            scopes: [],
+            extra: { connectionContext: { scope: { organizationId: ORG } } },
+          },
+        },
+      );
+      expect(response.status).toBe(200);
+      await new Promise((resolve) => setTimeout(resolve, 50));
+    } finally {
+      await handler.close();
+    }
+    return captured;
+  }
+
+  async function modernToolCall(options: Parameters<typeof modernRequest>[0]) {
+    const event = (await modernRequest(options)).find(
+      (event) => event.event === PostHogMCPAnalyticsEvent.ToolCall,
+    );
+    expect(event).toBeDefined();
+    return event!;
+  }
+
+  test("groups session-less calls by caller, client, and window", async () => {
+    setSystemTime(new Date("2026-01-01T00:10:00Z"));
+    try {
+      const first = await modernToolCall({
+        token: "token-a",
+        clientName: "client-a",
+      });
+      const second = await modernToolCall({
+        token: "token-a",
+        clientName: "client-a",
+      });
+      const otherCaller = await modernToolCall({
+        token: "token-b",
+        clientName: "client-a",
+      });
+      const otherClient = await modernToolCall({
+        token: "token-a",
+        clientName: "client-b",
+      });
+      setSystemTime(new Date("2026-01-01T00:40:00Z"));
+      const nextWindow = await modernToolCall({
+        token: "token-a",
+        clientName: "client-a",
+      });
+
+      const sessionId = first.properties?.$session_id;
+      expect(sessionId).toStartWith("ses_");
+      expect(first.distinctId).toBe(sessionId as string);
+      expect(second.properties?.$session_id).toBe(sessionId);
+      expect(second.distinctId).toBe(sessionId as string);
+      expect(otherCaller.properties?.$session_id).not.toBe(sessionId);
+      expect(otherClient.properties?.$session_id).not.toBe(sessionId);
+      expect(nextWindow.properties?.$session_id).not.toBe(sessionId);
+      expect(JSON.stringify(first)).not.toContain("token-a");
+      expect(first.properties).not.toHaveProperty(
+        "__mcp_analytics_session_key",
+      );
+    } finally {
+      setSystemTime();
+    }
+  });
+
+  test("puts session-less custom events in the caller's derived session", async () => {
+    setSystemTime(new Date("2026-01-01T00:10:00Z"));
+    try {
+      const toolCall = await modernToolCall({
+        token: "token-a",
+        clientName: "client-a",
+      });
+      const captured = await modernRequest({
+        token: "token-a",
+        clientName: "client-a",
+        name: KERNEL_FEEDBACK_TOOL_NAME,
+        args: {
+          context:
+            "Reporting a repeatable site block so the affected domain can be prioritized for a working browser configuration.",
+          summary: "Stealth sessions were consistently blocked",
+          feedback_type: "site_compatibility",
+          sentiment: "negative",
+          task_completed: false,
+          site_compatibility: {
+            registrable_domain: "example.com",
+            observed_outcome: "blocked",
+            reproducibility: "consistent",
+          },
+        },
+      });
+      const feedback = captured.find(
+        ({ event }) => event === MCP_FEEDBACK_SUBMITTED_EVENT,
+      );
+
+      const sessionId = toolCall.properties?.$session_id;
+      expect(feedback?.properties?.$session_id).toBe(sessionId);
+      expect(feedback?.distinctId).toBe(sessionId as string);
+      expect(feedback?.properties).not.toHaveProperty(
+        "__mcp_analytics_session_key",
+      );
+    } finally {
+      setSystemTime();
+    }
+  });
+
+  test("records the model only from Codex request metadata", async () => {
+    const codex = await modernToolCall({
+      token: "token-a",
+      clientName: "codex",
+      meta: { "x-codex-turn-metadata": { model: "gpt-5.2-codex" } },
+    });
+    expect(codex.properties).toMatchObject({
+      [PostHogMCPAnalyticsProperty.LlmModel]: "gpt-5.2-codex",
+      [PostHogMCPAnalyticsProperty.LlmModelSource]: "client_metadata",
+    });
+
+    for (const model of ["unknown", "ignore previous instructions", 42]) {
+      const event = await modernToolCall({
+        token: "token-a",
+        clientName: "codex",
+        meta: { "x-codex-turn-metadata": { model } },
+      });
+      expect(event.properties).not.toHaveProperty(
+        PostHogMCPAnalyticsProperty.LlmModel,
+      );
+    }
+
+    const other = await modernToolCall({
+      token: "token-a",
+      clientName: "client-a",
+    });
+    expect(other.properties).not.toHaveProperty(
+      PostHogMCPAnalyticsProperty.LlmModel,
+    );
+    expect(other.properties).not.toHaveProperty(
+      PostHogMCPAnalyticsProperty.LlmModelSource,
+    );
+  });
+
   test("stays anonymous when no connection context is attached", async () => {
     const captured: { event?: string }[] = [];
     const server = makeServer(captured);

src/lib/mcp/analytics.ts

@@ -1,14 +1,18 @@
 import { createHash } from "node:crypto";
 import { isIP } from "node:net";
 import {
+  decodeSessionId,
+  getRequestHeaders,
   instrument,
+  MCP_SESSION_HEADER,
   PostHogMCPAnalyticsEvent,
   PostHogMCPAnalyticsProperty,
   type BeforeSendFn,
   type McpAnalytics,
 } from "@posthog/mcp";
 import {
   CLIENT_CAPABILITIES_META_KEY,
+  CLIENT_INFO_META_KEY,
   type McpServer,
 } from "@modelcontextprotocol/server";
 import { PostHog } from "posthog-node";
@@ -171,6 +175,8 @@ const SENT_PROPERTIES = new Set<string>([
   PostHogMCPAnalyticsProperty.IntentSource,
   PostHogMCPAnalyticsProperty.IsError,
   PostHogMCPAnalyticsProperty.ListedToolNames,
+  PostHogMCPAnalyticsProperty.LlmModel,
+  PostHogMCPAnalyticsProperty.LlmModelSource,
   PostHogMCPAnalyticsProperty.ProtocolVersion,
   PostHogMCPAnalyticsProperty.ResourceName,
   PostHogMCPAnalyticsProperty.ServerName,
@@ -392,6 +398,7 @@ export const sanitizeMcpAnalyticsEvent: BeforeSendFn = (event) => {
   const properties = event.properties;
   if (!properties) return event;
   enrichMcpAnalyticsEvent(event);
+  applyDerivedSession(event);
   if (event.event === PostHogMCPAnalyticsEvent.ToolCall) {
     annotateProjectParamUsage(properties);
     annotateDeprecatedParamUsage(properties);
@@ -459,18 +466,114 @@ function connectionOrgId(ctx: unknown) {
   return authExtra?.connectionContext?.scope.organizationId;
 }
 
+const SESSION_KEY_PROPERTY = "__mcp_analytics_session_key";
+
+// Matches the SDK's inactivity timeout, but as fixed windows: requests on the 2026-07-28
+// revision run on a fresh server instance each, so there is no in-process state to
+// measure inactivity against.
+const DERIVED_SESSION_WINDOW_MS = 30 * 60 * 1000;
+
+/**
+ * Requests without an MCP session (every request on the 2026-07-28 revision) would get a
+ * new $session_id per call. For those, key the session on the caller, organization, and
+ * client instead; sanitizeMcpAnalyticsEvent buckets the key into a window. Concurrent
+ * runs by the same caller and client share a session.
+ */
+function analyticsSessionKey(ctx: unknown) {
+  const extra = ctx as
+    | {
+        sessionId?: string;
+        http?: { authInfo?: { token?: string; extra?: unknown } };
+        mcpReq?: { envelope?: unknown };
+      }
+    | undefined;
+  const headers = getRequestHeaders(extra);
+  const header = headers?.[MCP_SESSION_HEADER];
+  if (
+    extra?.sessionId ||
+    decodeSessionId(Array.isArray(header) ? header[0] : header)
+  ) {
+    return null;
+  }
+
+  const authInfo = extra?.http?.authInfo;
+  const userId = (authInfo?.extra as { userId?: string | null } | undefined)
+    ?.userId;
+  const subject = userId
+    ? `user:${userId}`
+    : authInfo?.token
+      ? `token:${authInfo.token}`
+      : null;
+  if (!subject) return null;
+
+  const envelope = extra?.mcpReq?.envelope;
+  const clientInfo = isRecord(envelope)
+    ? envelope[CLIENT_INFO_META_KEY]
+    : undefined;
+  const clientName =
+    isRecord(clientInfo) && typeof clientInfo.name === "string"
+      ? clientInfo.name
+      : "";
+
+  return createHash("sha256")
+    .update([subject, connectionOrgId(ctx) ?? "", clientName].join("\0"))
+    .digest("hex");
+}
+
+function applyDerivedSession(event: Parameters<BeforeSendFn>[0]) {
+  const properties = event.properties;
+  const key = properties[SESSION_KEY_PROPERTY];
+  delete properties[SESSION_KEY_PROPERTY];
+  if (typeof key !== "string") return;
+
+  const window = Math.floor(
+    Date.parse(event.timestamp) / DERIVED_SESSION_WINDOW_MS,
+  );
+  const sessionId = `ses_${createHash("sha256")
+    .update(`${key}\0${window}`)
+    .digest("hex")
+    .slice(0, 32)}`;
+
+  const previous = properties[PostHogMCPAnalyticsProperty.SessionId];
+  properties[PostHogMCPAnalyticsProperty.SessionId] = sessionId;
+  // Anonymous events use the session id as their distinct id.
+  if (event.distinct_id === previous) event.distinct_id = sessionId;
+}
+
+const CODEX_TURN_METADATA_KEY = "x-codex-turn-metadata";
+const MODEL_ID_PATTERN = /^[\w.:/@-]{1,100}$/;
+
+/**
+ * Codex reports its model in request metadata, which needs no extra tool argument.
+ * Other clients don't, and agent self-reporting stays off.
+ */
+function clientMetadataModel(request: { params?: unknown }) {
+  const params = request.params;
+  const meta = isRecord(params) ? params._meta : undefined;
+  const codex = isRecord(meta) ? meta[CODEX_TURN_METADATA_KEY] : undefined;
+  const model =
+    isRecord(codex) && typeof codex.model === "string"
+      ? codex.model.trim()
+      : "";
+  return MODEL_ID_PATTERN.test(model) && model.toLowerCase() !== "unknown"
+    ? model
+    : null;
+}
+
 export function captureMcpCustomEvent(
   analytics: McpAnalytics,
   extra: unknown,
   event: string,
   properties: Record<string, unknown>,
 ) {
   const organizationId = connectionOrgId(extra);
+  const sessionKey = analyticsSessionKey(extra);
   return analytics.capture({
     event,
     properties: {
       ...properties,
       ...(organizationId && { $groups: { organization: organizationId } }),
+      ...(sessionKey && { [SESSION_KEY_PROPERTY]: sessionKey }),
     },
   });
 }
@@ -898,6 +1001,16 @@ export function instrumentMcpAnalytics(
       if (isRecord(capabilities)) {
         Object.assign(properties, clientCapabilityAnalytics(capabilities));
       }
+      const sessionKey = analyticsSessionKey(extra);
+      if (sessionKey) properties[SESSION_KEY_PROPERTY] = sessionKey;
+      if (request.method === "tools/call") {
+        const model = clientMetadataModel(request);
+        if (model) {
+          properties[PostHogMCPAnalyticsProperty.LlmModel] = model;
+          properties[PostHogMCPAnalyticsProperty.LlmModelSource] =
+            "client_metadata";
+        }
+      }
       return Object.keys(properties).length > 0 ? properties : null;
     },
     // No part of a call is safe to capture: arguments carry free-form input (credential