Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions apps/web/src/app/api/public/leaderboard-provider-race/route.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
import { describe, test, expect, jest } from '@jest/globals';
import type { resolveSnowflakeConfig } from '@/lib/snowflake';
import type { GET as RouteGet } from './route';

jest.mock('@sentry/nextjs', () => ({ captureException: jest.fn() }));

jest.mock('@/lib/redis', () => ({
redisClient: {
get: jest.fn<() => Promise<string | null>>().mockResolvedValue(null),
set: jest.fn<() => Promise<string>>().mockResolvedValue('OK'),
},
}));

jest.mock('@/lib/snowflake', () => ({
resolveSnowflakeConfig: jest.fn(),
executeSnowflakeStatement: jest.fn(),
}));

type SnowflakeConfig = ReturnType<typeof resolveSnowflakeConfig>;

const FAKE_CONFIG = { accountHost: 'test.snowflakecomputing.com' } as SnowflakeConfig;

// The route builds a module-level in-process cache (1h TTL) at import time, so
// each test loads it in an isolated module registry to keep that cache from
// leaking between cases.
async function loadGet(opts: {
config?: SnowflakeConfig;
rows?: string[][];
statementError?: Error;
}): Promise<typeof RouteGet> {
let GET!: typeof RouteGet;
await jest.isolateModulesAsync(async () => {
const snowflake = await import('@/lib/snowflake');
const config = opts.config === undefined ? FAKE_CONFIG : opts.config;
jest.mocked(snowflake.resolveSnowflakeConfig).mockReturnValue(config);
if (opts.statementError) {
jest.mocked(snowflake.executeSnowflakeStatement).mockRejectedValue(opts.statementError);
} else {
jest.mocked(snowflake.executeSnowflakeStatement).mockResolvedValue(opts.rows ?? []);
}
({ GET } = await import('./route'));
});
return GET;
}

describe('GET /api/public/leaderboard-provider-race', () => {
test('returns 503 when Snowflake is not configured', async () => {
const GET = await loadGet({ config: null });

const response = await GET();

expect(response.status).toBe(503);
expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*');
});

test('returns 502 when the Snowflake query fails (e.g. is_open_weights backfill not landed)', async () => {
const GET = await loadGet({ statementError: new Error('invalid identifier IS_OPEN_WEIGHTS') });

const response = await GET();

expect(response.status).toBe(502);
});

test('maps explicit is_open_weights to booleans and NULL/unknown to null', async () => {
const GET = await loadGet({
rows: [
['2026-07-14', 'Anthropic', 'false', '12345'],
['2026-07-14', 'Alibaba', 'true', '6789'],
// The SQL API returns NULL cells as null at runtime despite the
// string[] row type; unmapped models land here.
['2026-07-14', 'other', null, '999'] as unknown as string[],
// A future model_dim mapping gap must not be silently bucketed as closed.
['2026-07-14', 'Mystery', 'unexpected', '111'],
],
});

const response = await GET();

expect(response.status).toBe(200);
expect(await response.json()).toEqual([
{ weekStart: '2026-07-14', provider: 'Anthropic', isOpenWeights: false, tokens: 12345 },
{ weekStart: '2026-07-14', provider: 'Alibaba', isOpenWeights: true, tokens: 6789 },
{ weekStart: '2026-07-14', provider: 'other', isOpenWeights: null, tokens: 999 },
{ weekStart: '2026-07-14', provider: 'Mystery', isOpenWeights: null, tokens: 111 },
]);
expect(response.headers.get('Cache-Control')).toContain('public');
expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*');
});

test('returns 502 when a row is malformed', async () => {
const GET = await loadGet({ rows: [['2026-07-14', 'Anthropic', 'true', 'not-a-number']] });

const response = await GET();

expect(response.status).toBe(502);
});
});

describe('OPTIONS /api/public/leaderboard-provider-race', () => {
test('returns 204 with CORS headers', async () => {
const { OPTIONS } = await import('./route');

const response = OPTIONS();

expect(response.status).toBe(204);
expect(response.headers.get('Access-Control-Allow-Origin')).toBe('*');
expect(response.headers.get('Access-Control-Allow-Methods')).toBe('GET');
});
});
80 changes: 80 additions & 0 deletions apps/web/src/app/api/public/leaderboard-provider-race/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
import { z } from 'zod';

import {
createPublicSnowflakeReport,
publicSnowflakeReportOptions,
} from '@/lib/public-snowflake-report';
import { LEADERBOARD_PROVIDER_RACE_REDIS_KEY } from '@/lib/redis-keys';

// Weekly token volume per model lab, from a fixed start date through the most
// recent complete week. Grouped at week x model_provider_company x
// is_open_weights so a single payload drives both the per-lab "race" view and
// an open-weight vs proprietary toggle. model_provider_company and
// is_open_weights are maintained in the dbt model_dim seed (kilocode-dbt), so
// the lab mapping lives in one place rather than being re-derived here.
// The partial current week is excluded so every returned week is complete.
const LEADERBOARD_PROVIDER_RACE_QUERY = `
select
to_char(date_trunc('week', ud.usage_date), 'YYYY-MM-DD') as week_start
, ud.model_provider_company as provider
, ud.is_open_weights
, sum(ud.total_tokens) as tokens
from kilo_dw.dbt_prod.usage_daily as ud
where
ud.usage_date >= '2025-07-01'
and date_trunc('week', ud.usage_date) < date_trunc('week', current_date())
and ud.total_tokens > 0
group by 1, 2, 3
order by 1, 4 desc;
`;

const providerRaceSchema = z.array(
z.object({
weekStart: z.string().regex(/^\d{4}-\d{2}-\d{2}$/),
provider: z.string().min(1),
isOpenWeights: z.boolean().nullable(),
tokens: z.number(),
})
);

type ProviderRace = z.infer<typeof providerRaceSchema>;

// is_open_weights is maintained in the dbt model_dim seed and is NULL for
// unmapped models (the ones bucketed as provider = 'other'). Only the explicit
// 'true'/'false' strings map to a boolean; anything else (NULL from the SQL API,
// or a future mapping gap) becomes null so the client can distinguish "unknown"
// from a confirmed closed-weight lab instead of silently bucketing gaps as closed.
function parseOpenWeights(raw: string): boolean | null {
if (raw === 'true') return true;
if (raw === 'false') return false;
return null;
}

function parseProviderRace(rows: string[][]): ProviderRace {
return rows.map(row => {
const [weekStart, provider, rawOpenWeights, rawTokens] = row;
const tokens = Number(rawTokens);

if (!weekStart || !provider || !Number.isFinite(tokens)) {
throw new Error('Snowflake returned an invalid provider race row');
}

return providerRaceSchema.element.parse({
weekStart,
provider,
isOpenWeights: parseOpenWeights(rawOpenWeights),
tokens,
});
});
}

export const GET = createPublicSnowflakeReport({
cacheKey: LEADERBOARD_PROVIDER_RACE_REDIS_KEY,
errorMessage: 'Failed to fetch leaderboard provider race',
parseRows: parseProviderRace,
query: LEADERBOARD_PROVIDER_RACE_QUERY,
schema: providerRaceSchema,
source: 'public-leaderboard-provider-race-api',
});

export const OPTIONS = publicSnowflakeReportOptions;
1 change: 1 addition & 0 deletions apps/web/src/lib/redis-keys.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ export const LEADERBOARD_MODEL_PROVIDER_USAGE_REDIS_KEY = redisKey(
'public-api:leaderboard-model-provider-usage'
);
export const LEADERBOARD_MODEL_USAGE_REDIS_KEY = redisKey('public-api:leaderboard-model-usage');
export const LEADERBOARD_PROVIDER_RACE_REDIS_KEY = redisKey('public-api:leaderboard-provider-race');

export const REQUEST_LOGGING_OPT_INS_REDIS_KEY = redisKey('ai-gateway:request-logging-opt-ins');

Expand Down