perf(files): resolve name references through an index instead of listing the workspace

apps/sim/lib/uploads/contexts/workspace/__integration__/file-names.integration.ts

@@ -7,9 +7,16 @@ import { rm } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import path from 'node:path'
 import { db, dbFor } from '@sim/db'
-import { copilotChats, organization, user, workspace, workspaceFiles } from '@sim/db/schema'
+import {
+  copilotChats,
+  displaySegmentKey,
+  organization,
+  user,
+  workspace,
+  workspaceFiles,
+} from '@sim/db/schema'
 import { generateId } from '@sim/utils/id'
-import { and, eq, inArray, isNull, sql } from 'drizzle-orm'
+import { and, eq, inArray, isNull, or, sql } from 'drizzle-orm'
 import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
 
 const fixtureStorage = vi.hoisted(() => ({ root: '' }))
@@ -21,6 +28,7 @@ vi.mock('@/lib/uploads/core/setup.server', () => ({
 
 import { fileParseBodySchema } from '@/lib/api/contracts/storage-transfer'
 import * as inputValidation from '@/lib/core/security/input-validation.server'
+import { textArrayLiteral } from '@/lib/db/arrays'
 import { executeFileParserOperation } from '@/lib/internal/file/parser'
 import {
   createKnowledgeAclFixtureIds,
@@ -39,6 +47,7 @@ import {
   uploadWorkspaceFile,
   workspaceFileVfsPath,
 } from '@/lib/uploads/contexts/workspace/workspace-file-manager'
+import { normalizeDisplaySegment } from '@/lib/vfs/path'
 import { createWorkspaceFileDelegatedPrincipal } from '@/lib/workspace-files/application/delegated-principal'
 
 describe('workspace file names in PostgreSQL', () => {
@@ -196,6 +205,83 @@ describe('workspace file names in PostgreSQL', () => {
     }
   })
 
+  // Names stored before upload normalization: U+202F before AM/PM, doubled and edge whitespace,
+  // a decomposed accent, a control character, and a BOM.
+  const unnormalizedNames = [
+    'Screenshot 2026-01-15 at 9.41.07\u202fAM.png',
+    'Quarterly  Report.pdf',
+    ' padded.txt\t',
+    'Cafe\u0301 menu.png',
+    'ring\u0007ing.png',
+    '\ufeffbom.md',
+  ]
+
+  it('normalizes a display name in SQL exactly as normalizeDisplaySegment does', async () => {
+    const rows = await db.execute<{ name: string; key: string }>(
+      sql`SELECT name, ${displaySegmentKey(sql`name`)} AS key FROM unnest(${textArrayLiteral(unnormalizedNames)}) AS name`
+    )
+    expect(rows).toHaveLength(unnormalizedNames.length)
+    for (const row of rows) expect(row.key).toBe(normalizeDisplaySegment(row.name))
+  })
+
+  it('resolves a stored name that differs from its VFS path through the display-name index', async () => {
+    const fixture = await seedWorkspace()
+    const folder = await createWorkspaceFileFolder({
+      workspaceId: fixture.workspaceId,
+      userId: fixture.aliceId,
+      name: 'Reports',
+    })
+
+    const files = unnormalizedNames.map((name, n) => ({
+      name,
+      id: `wf_legacy_${n}_${fixture.workspaceId}`,
+      folderId: n === 0 ? folder.id : null,
+    }))
+    for (const file of files) {
+      await db.execute(sql`
+        INSERT INTO ${workspaceFiles} (id, key, user_id, workspace_id, folder_id, context, original_name, content_type, size_bytes)
+        VALUES (${file.id}, ${`legacy/${file.id}`}, ${fixture.aliceId}, ${fixture.workspaceId},
+          ${file.folderId}, 'workspace', ${file.name}, 'text/plain', 1)`)
+    }
+    await db.execute(sql`
+      INSERT INTO ${workspaceFiles} (id, key, user_id, workspace_id, context, original_name, content_type)
+      SELECT 'wf_pad_' || n || '_' || ${fixture.workspaceId}, 'pad/' || n || '/' || ${fixture.workspaceId},
+        ${fixture.aliceId}, ${fixture.workspaceId}, 'workspace', 'pad-' || n || '.txt', 'text/plain'
+      FROM generate_series(1, 2000) AS n`)
+    await db.execute(sql`ANALYZE ${workspaceFiles}`)
+
+    for (const file of files) {
+      const vfsPath = workspaceFileVfsPath({
+        folderPath: file.folderId ? 'Reports' : null,
+        name: file.name,
+      })
+      const displayed = normalizeDisplaySegment(file.name)
+      for (const reference of [vfsPath, displayed]) {
+        expect((await resolveWorkspaceFileReference(fixture.workspaceId, reference))?.id).toBe(
+          file.id
+        )
+      }
+    }
+    expect(await resolveWorkspaceFileReference(fixture.workspaceId, 'brand-new.txt')).toBeNull()
+
+    // The resolver's candidate query for a miss: the id arm, the display-name arm, its order.
+    const plan = await db.execute(
+      sql`EXPLAIN (FORMAT JSON) SELECT id, original_name, folder_id, uploaded_at
+        FROM ${workspaceFiles} WHERE ${and(
+          eq(workspaceFiles.workspaceId, fixture.workspaceId),
+          eq(workspaceFiles.context, 'workspace'),
+          isNull(workspaceFiles.deletedAt),
+          or(
+            inArray(workspaceFiles.id, ['brand-new.txt']),
+            eq(displaySegmentKey(workspaceFiles.originalName), 'brand-new.txt')
+          )
+        )} ORDER BY ${workspaceFiles.uploadedAt}`
+    )
+    const scan = JSON.stringify(plan[0]['QUERY PLAN'])
+    expect(scan).toContain('workspace_files_workspace_display_name_idx')
+    expect(scan).not.toContain('workspace_files_workspace_active_keyset_idx')
+  })
+
   it('falls back to a short-id suffix after 20 numbered copies, including under concurrency', async () => {
     const fixture = await seedWorkspace()
     await upload(fixture.workspaceId, fixture.aliceId, 'page.html')

apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts

@@ -5,7 +5,13 @@
 
 import { randomBytes } from 'crypto'
 import { db } from '@sim/db'
-import { uploadSession, type WorkspaceFileRow, workspace, workspaceFiles } from '@sim/db/schema'
+import {
+  displaySegmentKey,
+  uploadSession,
+  type WorkspaceFileRow,
+  workspace,
+  workspaceFiles,
+} from '@sim/db/schema'
 import { createLogger } from '@sim/logger'
 import { sha256Hex } from '@sim/security/hash'
 import {
@@ -100,7 +106,11 @@ import {
 import { getWorkspaceFileSize, MAX_WORKSPACE_FILE_SIZE } from '@/lib/uploads/shared/types'
 import { isMarkdownFile } from '@/lib/uploads/utils/file-utils'
 import type { ServableFile } from '@/lib/uploads/utils/file-utils.server'
-import { decodeVfsPathSegments, displaySegmentPattern } from '@/lib/vfs/path'
+import {
+  decodeVfsPathSegments,
+  displaySegmentPattern,
+  normalizeDisplaySegment,
+} from '@/lib/vfs/path'
 import {
   OWNED_FILE_CONTEXTS,
   type OwnedFileContext,
@@ -1801,8 +1811,41 @@ export async function resolveWorkspaceFileReference(
   const exactReferenceFile = await getWorkspaceFileByExactReference(workspaceId, referenceSegments)
   if (exactReferenceFile) return exactReferenceFile
 
-  const files = await listWorkspaceFiles(workspaceId)
-  return findWorkspaceFileRecord(files, fileReference)
+  const candidates = await listWorkspaceFileReferenceCandidates(
+    workspaceId,
+    [fileReference, normalizedReference],
+    referenceSegments.at(-1)
+  )
+  return findWorkspaceFileRecord(candidates, fileReference)
+}
+
+/**
+ * The live workspace files {@link findWorkspaceFileRecord} can match for a reference: those
+ * whose id is one of `ids`, or whose name displays as `leafName`. Each of its rules requires one
+ * of the two, so it picks the same file from these rows as from the whole workspace — read
+ * through `workspace_files_workspace_display_name_idx` instead of every file the workspace has.
+ */
+async function listWorkspaceFileReferenceCandidates(
+  workspaceId: string,
+  ids: string[],
+  leafName: string | undefined
+): Promise<WorkspaceFileRecord[]> {
+  const rows = await db
+    .select(workspaceFileListColumns)
+    .from(workspaceFiles)
+    .where(
+      and(
+        workspaceFileScopeCondition(workspaceId, 'active'),
+        or(
+          inArray(workspaceFiles.id, ids),
+          leafName === undefined
+            ? undefined
+            : eq(displaySegmentKey(workspaceFiles.originalName), normalizeDisplaySegment(leafName))
+        )
+      )
+    )
+    .orderBy(workspaceFiles.uploadedAt)
+  return hydrateWorkspaceFilePaths(rows, workspaceId)
 }
 
 /**

apps/sim/lib/vfs/path.ts

@@ -1,10 +1,7 @@
-import { escapeRegExp } from '@sim/utils/string'
+import { escapeRegExp, WHITESPACE_CHARACTER_CLASS } from '@sim/utils/string'
 
 const CONTROL_CHARS = /[\x00-\x1f\x7f]/g
-/** The characters `\s` matches, spelled out so a PostgreSQL pattern can share the class. */
-const WHITESPACE_CLASS =
-  '[ \\t\\n\\v\\f\\r\\u00a0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff]'
-const WHITESPACE = new RegExp(`${WHITESPACE_CLASS}+`, 'g')
+const WHITESPACE = new RegExp(`${WHITESPACE_CHARACTER_CLASS}+`, 'g')
 
 export class VfsPathError extends Error {
   constructor(message: string) {
@@ -13,7 +10,12 @@ export class VfsPathError extends Error {
   }
 }
 
-function normalizeDisplaySegment(segment: string): string {
+/**
+ * The name a VFS path segment displays: NFC-composed, trimmed, control characters removed,
+ * whitespace runs collapsed. Its SQL twin is `displaySegmentKey` in `@sim/db/schema`, which an
+ * index serves; the two must apply the same steps in the same order.
+ */
+export function normalizeDisplaySegment(segment: string): string {
   return segment.normalize('NFC').trim().replace(CONTROL_CHARS, '').replace(WHITESPACE, ' ')
 }
 
@@ -24,7 +26,7 @@ function normalizeDisplaySegment(segment: string): string {
  */
 export function displaySegmentPattern(name: string): string {
   const words = name.split(' ').map(escapeRegExp)
-  return `^${WHITESPACE_CLASS}*${words.join(`${WHITESPACE_CLASS}+`)}${WHITESPACE_CLASS}*$`
+  return `^${WHITESPACE_CHARACTER_CLASS}*${words.join(`${WHITESPACE_CHARACTER_CLASS}+`)}${WHITESPACE_CHARACTER_CLASS}*$`
 }
 
 export function encodeVfsSegment(segment: string): string {

packages/db/migrations/0407_workspace_files_display_name_index.sql

@@ -0,0 +1,8 @@
+-- Serves name references that miss the exact-name lookup (see displaySegmentKey in schema.ts).
+-- Partial on live workspace files, so the concurrent build is small.
+COMMIT;--> statement-breakpoint
+SET lock_timeout = 0;--> statement-breakpoint
+-- migration-safe: replay replaces only this new index to recover an interrupted concurrent build; existing indexes remain available.
+DROP INDEX CONCURRENTLY IF EXISTS "workspace_files_workspace_display_name_idx";--> statement-breakpoint
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "workspace_files_workspace_display_name_idx" ON "workspace_files" USING btree ("workspace_id",regexp_replace(regexp_replace(regexp_replace(normalize("original_name", NFC), '^[ \t\n\v\f\r\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+|[ \t\n\v\f\r\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+$', '', 'g'), '[\x01-\x1f\x7f]', '', 'g'), '[ \t\n\v\f\r\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff]+', ' ', 'g')) WHERE "workspace_files"."deleted_at" IS NULL AND "workspace_files"."context" = 'workspace' AND "workspace_files"."workspace_id" IS NOT NULL;--> statement-breakpoint
+SET lock_timeout = '5s';