Skip to content

Commit 5757cb2

Browse files
lindesvardlindesvard
committed
wip
1 parent ccff908 commit 5757cb2

File tree

4 files changed

+120
-40
lines changed

4 files changed

+120
-40
lines changed

‎apps/api/src/routes/misc.router.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import * as controller from '@/controllers/misc.controller';
2-
import { insightsQueue } from '@openpanel/queue';
32
import type { FastifyPluginCallback } from 'fastify';
43

54
const miscRouter: FastifyPluginCallback = async (fastify) => {

‎apps/worker/src/index.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,10 @@ import {
1414
import express from 'express';
1515
import client from 'prom-client';
1616

17-
import { getRedisQueue } from '@openpanel/redis';
18-
import { Worker } from 'bullmq';
1917
import { BullBoardGroupMQAdapter } from 'groupmq';
2018
import sourceMapSupport from 'source-map-support';
2119
import { bootCron } from './boot-cron';
2220
import { bootWorkers } from './boot-workers';
23-
import { insightsProjectJob } from './jobs/insights';
2421
import { register } from './metrics';
2522
import { logger } from './utils/logger';
2623

‎packages/db/src/services/insights/modules/page-trends.module.ts‎

Lines changed: 54 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,11 @@ async function fetchPageTrendAggregates(ctx: ComputeContext): Promise<{
164164
export const pageTrendsModule: InsightModule = {
165165
key: 'page-trends',
166166
cadence: ['daily'],
167-
thresholds: { minTotal: 100, minAbsDelta: 30, minPct: 0.2, maxDims: 100 },
167+
// Share-based thresholds (values in basis points: 100 = 1%)
168+
// minTotal: require at least 0.5% combined share (current + baseline)
169+
// minAbsDelta: require at least 0.5 percentage point shift
170+
// minPct: require at least 25% relative change in share
171+
thresholds: { minTotal: 50, minAbsDelta: 50, minPct: 0.25, maxDims: 100 },
168172

169173
async enumerateDimensions(ctx) {
170174
const { currentMap, baselineMap } = await fetchPageTrendAggregates(ctx);
@@ -185,28 +189,40 @@ export const pageTrendsModule: InsightModule = {
185189
if (!dimKey.startsWith('page:')) continue;
186190
const originPath = dimKey.replace('page:', '');
187191

188-
const currentValue = currentMap.get(originPath) ?? 0;
189-
const compareValue = baselineMap.get(originPath) ?? 0;
192+
const pageviewsCurrent = currentMap.get(originPath) ?? 0;
193+
const pageviewsCompare = baselineMap.get(originPath) ?? 0;
190194

191-
const currentShare = totalCurrent > 0 ? currentValue / totalCurrent : 0;
192-
const compareShare = totalBaseline > 0 ? compareValue / totalBaseline : 0;
195+
const currentShare =
196+
totalCurrent > 0 ? pageviewsCurrent / totalCurrent : 0;
197+
const compareShare =
198+
totalBaseline > 0 ? pageviewsCompare / totalBaseline : 0;
199+
200+
// Use share values in basis points (100 = 1%) for thresholding
201+
// This makes thresholds intuitive: minAbsDelta=50 means 0.5pp shift
202+
const currentShareBp = currentShare * 10000;
203+
const compareShareBp = compareShare * 10000;
193204

194205
const shareShiftPp = (currentShare - compareShare) * 100;
195-
const changePct = computeChangePct(currentValue, compareValue);
196-
const direction = computeDirection(changePct);
206+
// changePct is relative change in share, not absolute pageviews
207+
const shareChangePct = computeChangePct(currentShare, compareShare);
208+
const direction = computeDirection(shareChangePct);
197209

198210
results.push({
199211
ok: true,
200212
dimensionKey: dimKey,
201-
currentValue,
202-
compareValue,
203-
changePct,
213+
// Use share in basis points for threshold checks
214+
currentValue: currentShareBp,
215+
compareValue: compareShareBp,
216+
changePct: shareChangePct,
204217
direction,
205218
extra: {
219+
// Keep absolute values for display
220+
pageviewsCurrent,
221+
pageviewsCompare,
206222
shareShiftPp,
207223
currentShare,
208224
compareShare,
209-
isNew: compareValue === 0 && currentValue > 0,
225+
isNew: pageviewsCompare === 0 && pageviewsCurrent > 0,
210226
},
211227
});
212228
}
@@ -218,58 +234,63 @@ export const pageTrendsModule: InsightModule = {
218234
const originPath = result.dimensionKey.replace('page:', '');
219235
const [origin, path] = originPath.split(DELIMITER);
220236
const displayValue = origin ? `${origin}${path}` : path || '/';
221-
const pct = ((result.changePct ?? 0) * 100).toFixed(1);
222-
const isIncrease = (result.changePct ?? 0) >= 0;
237+
238+
// Get absolute pageviews from extra (currentValue/compareValue are now share-based)
239+
const pageviewsCurrent = Number(result.extra?.pageviewsCurrent ?? 0);
240+
const pageviewsCompare = Number(result.extra?.pageviewsCompare ?? 0);
241+
const shareCurrent = Number(result.extra?.currentShare ?? 0);
242+
const shareCompare = Number(result.extra?.compareShare ?? 0);
243+
const shareShiftPp = Number(result.extra?.shareShiftPp ?? 0);
223244
const isNew = result.extra?.isNew as boolean | undefined;
224245

246+
// Display share shift in percentage points
247+
const isIncrease = shareShiftPp >= 0;
248+
const shareShiftDisplay = Math.abs(shareShiftPp).toFixed(1);
249+
225250
const title = isNew
226251
? `New page getting views: ${displayValue}`
227-
: `Page ${displayValue} ${isIncrease ? '↑' : '↓'} ${Math.abs(Number(pct))}%`;
228-
229-
const pageviewsCurrent = result.currentValue ?? 0;
230-
const pageviewsCompare = result.compareValue ?? 0;
231-
const shareCurrent = Number(result.extra?.currentShare ?? 0);
232-
const shareCompare = Number(result.extra?.compareShare ?? 0);
252+
: `Page ${displayValue} share ${isIncrease ? '↑' : '↓'} ${shareShiftDisplay}pp`;
233253

234254
return {
235255
title,
236-
summary: `${ctx.window.label}. Pageviews ${pageviewsCurrent} vs ${pageviewsCompare}.`,
256+
summary: `${ctx.window.label}. Share ${(shareCurrent * 100).toFixed(1)}% vs ${(shareCompare * 100).toFixed(1)}%.`,
237257
displayName: displayValue,
238258
payload: {
239259
kind: 'insight_v1',
240260
dimensions: [
241261
{ key: 'origin', value: origin ?? '', displayName: origin ?? '' },
242262
{ key: 'path', value: path ?? '', displayName: path ?? '' },
243263
],
244-
primaryMetric: 'pageviews',
264+
primaryMetric: 'share',
245265
metrics: {
246266
pageviews: {
247267
current: pageviewsCurrent,
248268
compare: pageviewsCompare,
249269
delta: pageviewsCurrent - pageviewsCompare,
250-
changePct: pageviewsCompare > 0 ? (result.changePct ?? 0) : null,
251-
direction: result.direction ?? 'flat',
252-
unit: 'count',
253-
},
254-
share: {
255-
current: shareCurrent,
256-
compare: shareCompare,
257-
delta: shareCurrent - shareCompare,
258270
changePct:
259-
shareCompare > 0
260-
? (shareCurrent - shareCompare) / shareCompare
271+
pageviewsCompare > 0
272+
? (pageviewsCurrent - pageviewsCompare) / pageviewsCompare
261273
: null,
262274
direction:
263-
shareCurrent - shareCompare > 0.0005
275+
pageviewsCurrent > pageviewsCompare
264276
? 'up'
265-
: shareCurrent - shareCompare < -0.0005
277+
: pageviewsCurrent < pageviewsCompare
266278
? 'down'
267279
: 'flat',
280+
unit: 'count',
281+
},
282+
share: {
283+
current: shareCurrent,
284+
compare: shareCompare,
285+
delta: shareCurrent - shareCompare,
286+
changePct: result.changePct ?? null, // This is now share-based
287+
direction: result.direction ?? 'flat',
268288
unit: 'ratio',
269289
},
270290
},
271291
extra: {
272292
isNew: result.extra?.isNew,
293+
shareShiftPp,
273294
},
274295
},
275296
};

‎packages/db/src/services/insights/store.ts‎

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -247,13 +247,76 @@ export const insightStore: InsightStore = {
247247
return { suppressed: 0, unsuppressed: 0 };
248248
}
249249

250+
let suppressed = 0;
251+
let unsuppressed = 0;
252+
253+
// For "yesterday" insights, suppress any that are stale (windowEnd is not actually yesterday)
254+
// This prevents showing confusing insights like "Yesterday traffic dropped" when it's from 2+ days ago
255+
if (windowKind === 'yesterday') {
256+
const yesterday = new Date(now);
257+
yesterday.setUTCHours(0, 0, 0, 0);
258+
yesterday.setUTCDate(yesterday.getUTCDate() - 1);
259+
const yesterdayTime = yesterday.getTime();
260+
261+
for (const insight of insights) {
262+
// If windowEnd is null, consider it stale
263+
const isStale = insight.windowEnd
264+
? new Date(insight.windowEnd).setUTCHours(0, 0, 0, 0) !==
265+
yesterdayTime
266+
: true;
267+
268+
if (isStale && insight.state === 'active') {
269+
await db.projectInsight.update({
270+
where: { id: insight.id },
271+
data: { state: 'suppressed', lastUpdatedAt: now },
272+
});
273+
suppressed++;
274+
}
275+
}
276+
277+
// Filter to only non-stale insights for top-N logic
278+
const freshInsights = insights.filter((insight) => {
279+
if (!insight.windowEnd) return false;
280+
const windowEndTime = new Date(insight.windowEnd).setUTCHours(
281+
0,
282+
0,
283+
0,
284+
0,
285+
);
286+
return windowEndTime === yesterdayTime;
287+
});
288+
289+
const topN = freshInsights.slice(0, keepTopN);
290+
const belowN = freshInsights.slice(keepTopN);
291+
292+
for (const insight of belowN) {
293+
if (insight.state === 'active') {
294+
await db.projectInsight.update({
295+
where: { id: insight.id },
296+
data: { state: 'suppressed', lastUpdatedAt: now },
297+
});
298+
suppressed++;
299+
}
300+
}
301+
302+
for (const insight of topN) {
303+
if (insight.state === 'suppressed') {
304+
await db.projectInsight.update({
305+
where: { id: insight.id },
306+
data: { state: 'active', lastUpdatedAt: now },
307+
});
308+
unsuppressed++;
309+
}
310+
}
311+
312+
return { suppressed, unsuppressed };
313+
}
314+
315+
// For non-yesterday windows, apply standard top-N suppression
250316
const topN = insights.slice(0, keepTopN);
251317
const belowN = insights.slice(keepTopN);
252318

253319
// Suppress those below top N
254-
let suppressed = 0;
255-
let unsuppressed = 0;
256-
257320
for (const insight of belowN) {
258321
if (insight.state === 'active') {
259322
await db.projectInsight.update({

0 commit comments

Comments
 (0)