Check durable MCP documentation schema parity

.github/workflows/ci.yml

@@ -28,3 +28,17 @@ jobs:
 
       - name: Test
         run: bun test
+
+      - name: Check out Kernel docs
+        uses: actions/checkout@v4
+        with:
+          repository: kernel/docs
+          path: kernel-docs
+          sparse-checkout: |
+            reference/mcp-server/tools/manage-browser-pools.mdx
+            reference/mcp-server/tools/manage-profiles.mdx
+            reference/mcp-server/tools/manage-proxies.mdx
+          sparse-checkout-cone-mode: false
+
+      - name: Check MCP documentation schema parity
+        run: bun scripts/check-doc-schema-parity.ts "$GITHUB_WORKSPACE/kernel-docs"

scripts/check-doc-schema-parity.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, test } from "bun:test";
+import { parityError, parseParameterNames } from "./check-doc-schema-parity";
+
+describe("parseParameterNames", () => {
+  test("reads individual and grouped parameter cells", () => {
+    const markdown = `
+## Parameters
+
+| Parameter | Description |
+| --- | --- |
+| \`action\` | Required. |
+| \`profile_id\` or \`profile_name\` | Choose one. |
+
+## Examples
+`;
+
+    expect([...parseParameterNames(markdown)].sort()).toEqual([
+      "action",
+      "profile_id",
+      "profile_name",
+    ]);
+  });
+
+  test("rejects a missing parameter table", () => {
+    expect(() => parseParameterNames("## Examples\n")).toThrow(
+      "missing Parameters section",
+    );
+  });
+
+  test("reads a parameter table at the end of a document", () => {
+    expect([
+      ...parseParameterNames(`## Parameters
+
+| Parameter | Description |
+| --- | --- |
+| \`action\` | Required. |
+`),
+    ]).toEqual(["action"]);
+  });
+});
+
+describe("parityError", () => {
+  test("accepts equal parameter sets", () => {
+    expect(
+      parityError("manage_profiles", new Set(["action"]), new Set(["action"])),
+    ).toBeUndefined();
+  });
+
+  test("reports undocumented and stale parameters", () => {
+    expect(
+      parityError(
+        "manage_profiles",
+        new Set(["action", "query"]),
+        new Set(["action", "old_query"]),
+      ),
+    ).toBe(
+      "manage_profiles: missing from docs: query; not in schema: old_query",
+    );
+  });
+});

scripts/check-doc-schema-parity.ts

@@ -0,0 +1,126 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { registerBrowserPoolCapabilities } from "@/lib/mcp/tools/browser-pools";
+import { registerProfileCapabilities } from "@/lib/mcp/tools/profiles";
+import { registerProxyTools } from "@/lib/mcp/tools/proxies";
+
+const documentedTools = {
+  manage_browser_pools: "reference/mcp-server/tools/manage-browser-pools.mdx",
+  manage_profiles: "reference/mcp-server/tools/manage-profiles.mdx",
+  manage_proxies: "reference/mcp-server/tools/manage-proxies.mdx",
+} as const;
+
+export function parseParameterNames(markdown: string): Set<string> {
+  const heading = markdown.match(/^## Parameters\s*$/m);
+  if (!heading || heading.index === undefined) {
+    throw new Error("missing Parameters section");
+  }
+  const parameters = markdown
+    .slice(heading.index + heading[0].length)
+    .split(/^##\s/m, 1)[0];
+
+  const names = new Set<string>();
+  for (const line of parameters.split("\n")) {
+    const firstCell = line.match(/^\|\s*(.*?)\s*\|/)?.[1];
+    if (!firstCell) continue;
+
+    for (const match of firstCell.matchAll(/`([a-z][a-z0-9_]*)`/g)) {
+      names.add(match[1]);
+    }
+  }
+  if (names.size === 0) {
+    throw new Error("Parameters table contains no parameter names");
+  }
+  return names;
+}
+
+export function parityError(
+  toolName: string,
+  schemaNames: Set<string>,
+  documentedNames: Set<string>,
+): string | undefined {
+  const missing = [...schemaNames].filter((name) => !documentedNames.has(name));
+  const stale = [...documentedNames].filter((name) => !schemaNames.has(name));
+  if (missing.length === 0 && stale.length === 0) return undefined;
+
+  const details = [];
+  if (missing.length > 0)
+    details.push(`missing from docs: ${missing.sort().join(", ")}`);
+  if (stale.length > 0)
+    details.push(`not in schema: ${stale.sort().join(", ")}`);
+  return `${toolName}: ${details.join("; ")}`;
+}
+
+async function listDurableToolParameters(): Promise<Map<string, Set<string>>> {
+  const server = new McpServer({ name: "schema-parity", version: "0.0.0" });
+  registerBrowserPoolCapabilities(server);
+  registerProfileCapabilities(server);
+  registerProxyTools(server);
+
+  const client = new Client({ name: "schema-parity", version: "0.0.0" });
+  const [clientTransport, serverTransport] =
+    InMemoryTransport.createLinkedPair();
+  await Promise.all([
+    server.connect(serverTransport),
+    client.connect(clientTransport),
+  ]);
+
+  try {
+    const { tools } = await client.listTools();
+    return new Map(
+      tools.map((tool) => [
+        tool.name,
+        new Set(Object.keys(tool.inputSchema.properties ?? {})),
+      ]),
+    );
+  } finally {
+    await Promise.all([client.close(), server.close()]);
+  }
+}
+
+export async function checkDocSchemaParity(docsRoot: string): Promise<void> {
+  const schemas = await listDurableToolParameters();
+  const errors: string[] = [];
+
+  for (const [toolName, relativePath] of Object.entries(documentedTools)) {
+    const schemaNames = schemas.get(toolName);
+    if (!schemaNames) {
+      errors.push(`${toolName}: tool was not registered`);
+      continue;
+    }
+
+    const markdown = await readFile(join(docsRoot, relativePath), "utf8");
+    try {
+      const error = parityError(
+        toolName,
+        schemaNames,
+        parseParameterNames(markdown),
+      );
+      if (error) errors.push(error);
+    } catch (error) {
+      errors.push(
+        `${toolName}: ${error instanceof Error ? error.message : error}`,
+      );
+    }
+  }
+
+  if (errors.length > 0) {
+    throw new Error(`MCP documentation schema drift:\n${errors.join("\n")}`);
+  }
+}
+
+if (import.meta.main) {
+  const docsRoot = process.argv[2];
+  if (!docsRoot) {
+    throw new Error(
+      "usage: bun scripts/check-doc-schema-parity.ts <docs-root>",
+    );
+  }
+  await checkDocSchemaParity(docsRoot);
+  console.log(
+    "MCP documentation parameter tables match the registered schemas.",
+  );
+}