Skip to content

Commit 79b6053

Browse files
ericallamericallam
authored
feat(server): add TaskIdentifier registry to replace expensive distinct query (#3368)
Replace the expensive DISTINCT query for task filter dropdowns with a dedicated TaskIdentifier registry table backed by Redis. Environments migrate automatically on their next deploy, with a transparent fallback to the legacy query for unmigrated environments. Also fixes duplicate dropdown entries when a task changes trigger source, and adds active/archived grouping for removed tasks. Moves BackgroundWorkerTask reads in the trigger hot path to the read replica.
1 parent 94abe97 commit 79b6053

File tree

26 files changed

+804
-78
lines changed

26 files changed

+804
-78
lines changed

‎.server-changes/task-identifier-registry.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
area: webapp
3+
type: improvement
4+
---
5+
6+
Replace the expensive DISTINCT query for task filter dropdowns with a dedicated TaskIdentifier registry table backed by Redis. Environments migrate automatically on their next deploy, with a transparent fallback to the legacy query for unmigrated environments. Also fixes duplicate dropdown entries when a task changes trigger source, and adds active/archived grouping for removed tasks. Moves BackgroundWorkerTask reads in the trigger hot path to the read replica.

‎apps/webapp/app/components/logs/LogsTaskFilter.tsx‎

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { useMemo } from "react";
44
import * as Ariakit from "@ariakit/react";
55
import {
66
ComboBox,
7+
SelectGroup,
8+
SelectGroupLabel,
79
SelectItem,
810
SelectList,
911
SelectPopover,
@@ -21,6 +23,7 @@ const shortcut = { key: "t" };
2123
type TaskOption = {
2224
slug: string;
2325
triggerSource: TaskTriggerSource;
26+
isInLatestDeployment: boolean;
2427
};
2528

2629
interface LogsTaskFilterProps {
@@ -126,17 +129,42 @@ function TasksDropdown({
126129
>
127130
<ComboBox placeholder={"Filter by task..."} value={searchValue} />
128131
<SelectList>
129-
{filtered.map((item, index) => (
130-
<SelectItem
131-
key={`${item.triggerSource}-${item.slug}`}
132-
value={item.slug}
133-
icon={
134-
<TaskTriggerSourceIcon source={item.triggerSource} className="size-4 flex-none" />
135-
}
136-
>
137-
{item.slug}
138-
</SelectItem>
139-
))}
132+
{filtered
133+
.filter((item) => item.isInLatestDeployment)
134+
.map((item) => (
135+
<SelectItem
136+
key={item.slug}
137+
value={item.slug}
138+
icon={
139+
<TaskTriggerSourceIcon source={item.triggerSource} className="size-4 flex-none" />
140+
}
141+
>
142+
{item.slug}
143+
</SelectItem>
144+
))}
145+
{filtered.some((item) => !item.isInLatestDeployment) && (
146+
<SelectGroup>
147+
<SelectGroupLabel>Archived</SelectGroupLabel>
148+
{filtered
149+
.filter((item) => !item.isInLatestDeployment)
150+
.map((item) => (
151+
<SelectItem
152+
key={item.slug}
153+
value={item.slug}
154+
icon={
155+
<span className="opacity-50">
156+
<TaskTriggerSourceIcon
157+
source={item.triggerSource}
158+
className="size-4 flex-none"
159+
/>
160+
</span>
161+
}
162+
>
163+
{item.slug}
164+
</SelectItem>
165+
))}
166+
</SelectGroup>
167+
)}
140168
</SelectList>
141169
</SelectPopover>
142170
</SelectProvider>

‎apps/webapp/app/components/runs/v3/RunFilters.tsx‎

Lines changed: 40 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ import { Paragraph } from "~/components/primitives/Paragraph";
3636
import {
3737
ComboBox,
3838
SelectButtonItem,
39+
SelectGroup,
40+
SelectGroupLabel,
3941
SelectItem,
4042
SelectList,
4143
SelectPopover,
@@ -322,7 +324,7 @@ export function getRunFiltersFromSearchParams(
322324
}
323325

324326
type RunFiltersProps = {
325-
possibleTasks: { slug: string; triggerSource: TaskTriggerSource }[];
327+
possibleTasks: { slug: string; triggerSource: TaskTriggerSource; isInLatestDeployment: boolean }[];
326328
bulkActions: {
327329
id: string;
328330
type: BulkActionType;
@@ -627,7 +629,7 @@ function TasksDropdown({
627629
clearSearchValue: () => void;
628630
searchValue: string;
629631
onClose?: () => void;
630-
possibleTasks: { slug: string; triggerSource: TaskTriggerSource }[];
632+
possibleTasks: { slug: string; triggerSource: TaskTriggerSource; isInLatestDeployment: boolean }[];
631633
}) {
632634
const { values, replace } = useSearchParams();
633635

@@ -658,17 +660,42 @@ function TasksDropdown({
658660
>
659661
<ComboBox placeholder={"Filter by task..."} value={searchValue} />
660662
<SelectList>
661-
{filtered.map((item, index) => (
662-
<SelectItem
663-
key={`${item.triggerSource}-${item.slug}`}
664-
value={item.slug}
665-
icon={
666-
<TaskTriggerSourceIcon source={item.triggerSource} className="size-4 flex-none" />
667-
}
668-
>
669-
<MiddleTruncate text={item.slug} />
670-
</SelectItem>
671-
))}
663+
{filtered
664+
.filter((item) => item.isInLatestDeployment)
665+
.map((item) => (
666+
<SelectItem
667+
key={item.slug}
668+
value={item.slug}
669+
icon={
670+
<TaskTriggerSourceIcon source={item.triggerSource} className="size-4 flex-none" />
671+
}
672+
>
673+
<MiddleTruncate text={item.slug} />
674+
</SelectItem>
675+
))}
676+
{filtered.some((item) => !item.isInLatestDeployment) && (
677+
<SelectGroup>
678+
<SelectGroupLabel>Archived</SelectGroupLabel>
679+
{filtered
680+
.filter((item) => !item.isInLatestDeployment)
681+
.map((item) => (
682+
<SelectItem
683+
key={item.slug}
684+
value={item.slug}
685+
icon={
686+
<span className="opacity-50">
687+
<TaskTriggerSourceIcon
688+
source={item.triggerSource}
689+
className="size-4 flex-none"
690+
/>
691+
</span>
692+
}
693+
>
694+
<MiddleTruncate text={item.slug} />
695+
</SelectItem>
696+
))}
697+
</SelectGroup>
698+
)}
672699
</SelectList>
673700
</SelectPopover>
674701
</SelectProvider>

‎apps/webapp/app/models/task.server.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import type { TaskTriggerSource } from "@trigger.dev/database";
22
import { PrismaClientOrTransaction, sqlDatabaseSchema } from "~/db.server";
33

4+
export { getTaskIdentifiers } from "~/services/taskIdentifierRegistry.server";
5+
export type { TaskIdentifierEntry } from "~/services/taskIdentifierCache.server";
6+
47
/**
58
*
69
* @param prisma An efficient query to get all task identifiers for a project.

‎apps/webapp/app/presenters/v3/ErrorsListPresenter.server.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import { type ErrorGroupStatus, type PrismaClientOrTransaction } from "@trigger.
1313
import { type Direction } from "~/components/ListPagination";
1414
import { timeFilterFromTo } from "~/components/runs/v3/SharedFilters";
1515
import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server";
16-
import { getAllTaskIdentifiers } from "~/models/task.server";
16+
import { getTaskIdentifiers } from "~/models/task.server";
1717
import { ServiceValidationError } from "~/v3/services/baseService.server";
1818
import { BasePresenter } from "~/presenters/v3/basePresenter.server";
1919

@@ -170,7 +170,7 @@ export class ErrorsListPresenter extends BasePresenter {
170170
(search !== undefined && search !== "") ||
171171
(statuses !== undefined && statuses.length > 0);
172172

173-
const possibleTasksAsync = getAllTaskIdentifiers(this.replica, environmentId);
173+
const possibleTasksAsync = getTaskIdentifiers(environmentId);
174174

175175
// Pre-filter by status: since status lives in Postgres (ErrorGroupState) and the error
176176
// list comes from ClickHouse, we resolve inclusion/exclusion sets upfront so that

‎apps/webapp/app/presenters/v3/LogsListPresenter.server.ts‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import parseDuration from "parse-duration";
77
import { type Direction } from "~/components/ListPagination";
88
import { timeFilterFromTo, timeFilters } from "~/components/runs/v3/SharedFilters";
99
import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server";
10-
import { getAllTaskIdentifiers } from "~/models/task.server";
10+
import { getTaskIdentifiers } from "~/models/task.server";
1111
import { ServiceValidationError } from "~/v3/services/baseService.server";
1212
import { kindToLevel, type LogLevel, LogLevelSchema } from "~/utils/logUtils";
1313
import { BasePresenter } from "~/presenters/v3/basePresenter.server";
@@ -176,7 +176,7 @@ export class LogsListPresenter extends BasePresenter {
176176
(search !== undefined && search !== "") ||
177177
!time.isDefault;
178178

179-
const possibleTasksAsync = getAllTaskIdentifiers(this.replica, environmentId);
179+
const possibleTasksAsync = getTaskIdentifiers(environmentId);
180180

181181
const bulkActionsAsync = this.replica.bulkActionGroup.findMany({
182182
select: {
@@ -386,12 +386,7 @@ export class LogsListPresenter extends BasePresenter {
386386
next: nextCursor,
387387
previous: undefined, // For now, only support forward pagination
388388
},
389-
possibleTasks: possibleTasks
390-
.map((task) => ({
391-
slug: task.slug,
392-
triggerSource: task.triggerSource,
393-
}))
394-
.sort((a, b) => a.slug.localeCompare(b.slug)),
389+
possibleTasks,
395390
bulkActions: bulkActions.map((bulkAction) => ({
396391
id: bulkAction.friendlyId,
397392
type: bulkAction.type,

‎apps/webapp/app/presenters/v3/NextRunListPresenter.server.ts‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import {
88
import { type Direction } from "~/components/ListPagination";
99
import { timeFilters } from "~/components/runs/v3/SharedFilters";
1010
import { findDisplayableEnvironment } from "~/models/runtimeEnvironment.server";
11-
import { getAllTaskIdentifiers } from "~/models/task.server";
11+
import { getTaskIdentifiers } from "~/models/task.server";
1212
import { RunsRepository } from "~/services/runsRepository/runsRepository.server";
1313
import { machinePresetFromRun } from "~/v3/machinePresets.server";
1414
import { ServiceValidationError } from "~/v3/services/baseService.server";
@@ -105,7 +105,7 @@ export class NextRunListPresenter {
105105
!time.isDefault;
106106

107107
//get all possible tasks
108-
const possibleTasksAsync = getAllTaskIdentifiers(this.replica, environmentId);
108+
const possibleTasksAsync = getTaskIdentifiers(environmentId);
109109

110110
//get possible bulk actions
111111
const bulkActionsAsync = this.replica.bulkActionGroup.findMany({
@@ -256,11 +256,7 @@ export class NextRunListPresenter {
256256
next: pagination.nextCursor ?? undefined,
257257
previous: pagination.previousCursor ?? undefined,
258258
},
259-
possibleTasks: possibleTasks
260-
.map((task) => ({ slug: task.slug, triggerSource: task.triggerSource }))
261-
.sort((a, b) => {
262-
return a.slug.localeCompare(b.slug);
263-
}),
259+
possibleTasks,
264260
bulkActions: bulkActions.map((bulkAction) => ({
265261
id: bulkAction.friendlyId,
266262
type: bulkAction.type,

‎apps/webapp/app/presenters/v3/ScheduleListPresenter.server.ts‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { type RuntimeEnvironmentType, type ScheduleType } from "@trigger.dev/database";
22
import { type ScheduleListFilters } from "~/components/runs/v3/ScheduleFilters";
33
import { displayableEnvironment } from "~/models/runtimeEnvironment.server";
4+
import { getTaskIdentifiers } from "~/models/task.server";
45
import { getLimit } from "~/services/platform.v3.server";
56
import { findCurrentWorkerFromEnvironment } from "~/v3/models/workerDeployment.server";
67
import { ServiceValidationError } from "~/v3/services/baseService.server";
@@ -123,14 +124,10 @@ export class ScheduleListPresenter extends BasePresenter {
123124
}
124125

125126
//get all possible scheduled tasks
126-
const possibleTasks = await this._replica.backgroundWorkerTask.findMany({
127-
where: {
128-
workerId: latestWorker.id,
129-
projectId: project.id,
130-
runtimeEnvironmentId: environmentId,
131-
triggerSource: "SCHEDULED",
132-
},
133-
});
127+
const allIdentifiers = await getTaskIdentifiers(environmentId);
128+
const possibleTasks = allIdentifiers
129+
.filter((t) => t.triggerSource === "SCHEDULED" && t.isInLatestDeployment)
130+
.map((t) => ({ slug: t.slug }));
134131

135132
//do this here to protect against SQL injection
136133
search = search && search !== "" ? `%${search}%` : undefined;
@@ -285,7 +282,7 @@ export class ScheduleListPresenter extends BasePresenter {
285282
totalPages: Math.ceil(totalCount / pageSize),
286283
totalCount: totalCount,
287284
schedules,
288-
possibleTasks: possibleTasks.map((task) => task.slug).sort((a, b) => a.localeCompare(b)),
285+
possibleTasks: possibleTasks.map((task) => task.slug),
289286
hasFilters,
290287
limits: {
291288
used: schedulesCount,

‎apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.$dashboardKey/route.tsx‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,13 @@ import { TitleWidget } from "~/components/metrics/TitleWidget";
1717
import { CreateDashboardPageButton } from "~/components/navigation/DashboardDialogs";
1818
import { NavBar, PageAccessories, PageTitle } from "~/components/primitives/PageHeader";
1919
import { TimeFilter } from "~/components/runs/v3/SharedFilters";
20-
import { $replica } from "~/db.server";
2120
import { useEnvironment } from "~/hooks/useEnvironment";
2221
import { useOrganization } from "~/hooks/useOrganizations";
2322
import { useProject } from "~/hooks/useProject";
2423
import { useSearchParams } from "~/hooks/useSearchParam";
2524
import { findProjectBySlug } from "~/models/project.server";
2625
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
27-
import { getAllTaskIdentifiers } from "~/models/task.server";
26+
import { getTaskIdentifiers } from "~/models/task.server";
2827
import {
2928
type BuiltInDashboardFilter,
3029
type LayoutItem,
@@ -70,7 +69,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
7069
organizationId: project.organizationId,
7170
key: dashboardKey,
7271
}),
73-
getAllTaskIdentifiers($replica, environment.id),
72+
getTaskIdentifiers(environment.id),
7473
]);
7574

7675
const filters = dashboard.filters ?? ["tasks", "queues"];
@@ -114,9 +113,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
114113
return typedjson({
115114
...dashboard,
116115
filters,
117-
possibleTasks: possibleTasks
118-
.map((task) => ({ slug: task.slug, triggerSource: task.triggerSource }))
119-
.sort((a, b) => a.slug.localeCompare(b.slug)),
116+
possibleTasks,
120117
possibleModels,
121118
possiblePrompts,
122119
possibleOperations,
@@ -201,7 +198,7 @@ export function MetricDashboard({
201198
/** Which filters to show. Defaults to ["tasks", "queues"]. */
202199
filters?: BuiltInDashboardFilter[];
203200
/** Possible tasks for filtering */
204-
possibleTasks?: { slug: string; triggerSource: TaskTriggerSource }[];
201+
possibleTasks?: { slug: string; triggerSource: TaskTriggerSource; isInLatestDeployment: boolean }[];
205202
/** Possible models for filtering */
206203
possibleModels?: ModelOption[];
207204
/** Possible prompt slugs for filtering */

‎apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.dashboards.custom.$dashboardId/route.tsx‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import { Sheet, SheetContent } from "~/components/primitives/SheetV3";
3535
import { useToast } from "~/components/primitives/Toast";
3636
import { SimpleTooltip } from "~/components/primitives/Tooltip";
3737
import { QueryEditor, type QueryEditorSaveData } from "~/components/query/QueryEditor";
38-
import { $replica, prisma } from "~/db.server";
38+
import { prisma } from "~/db.server";
3939
import { env } from "~/env.server";
4040
import { useDashboardEditor } from "~/hooks/useDashboardEditor";
4141
import { useEnvironment } from "~/hooks/useEnvironment";
@@ -44,7 +44,7 @@ import { useProject } from "~/hooks/useProject";
4444
import { redirectWithSuccessMessage } from "~/models/message.server";
4545
import { findProjectBySlug } from "~/models/project.server";
4646
import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server";
47-
import { getAllTaskIdentifiers } from "~/models/task.server";
47+
import { getTaskIdentifiers } from "~/models/task.server";
4848
import { MetricDashboardPresenter } from "~/presenters/v3/MetricDashboardPresenter.server";
4949
import { QueryPresenter } from "~/presenters/v3/QueryPresenter.server";
5050
import { requireUser, requireUserId } from "~/services/session.server";
@@ -93,7 +93,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
9393
queryPresenter.call({
9494
organizationId: project.organizationId,
9595
}),
96-
getAllTaskIdentifiers($replica, environment.id),
96+
getTaskIdentifiers(environment.id),
9797
]);
9898

9999
// Admins and impersonating users can use EXPLAIN
@@ -109,9 +109,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => {
109109
queryHistory: history,
110110
isAdmin,
111111
maxRows: env.QUERY_CLICKHOUSE_MAX_RETURNED_ROWS,
112-
possibleTasks: possibleTasks
113-
.map((task) => ({ slug: task.slug, triggerSource: task.triggerSource }))
114-
.sort((a, b) => a.slug.localeCompare(b.slug)),
112+
possibleTasks,
115113
widgetCount,
116114
});
117115
};

0 commit comments

Comments
 (0)