Surface incomplete scatter-gather results and server query timeouts

src/Frontend/src/components/ResultsCount.spec.ts

@@ -10,6 +10,12 @@ describe("FEATURE: Results count", () => {
     expect(screen.getByText(expected)).toBeInTheDocument();
   });
 
+  test("EXAMPLE: A partial result presents its total as a floor", () => {
+    render(ResultsCount, { props: { displayed: 3, total: 87421337, incomplete: true } });
+
+    expect(screen.getByText(`Showing 3 of at least ${(87421337).toLocaleString()} result(s)`)).toBeInTheDocument();
+  });
+
   test("EXAMPLE: The query duration is shown when known", () => {
     render(ResultsCount, { props: { displayed: 100, total: 500, durationMs: 2700 } });
 

src/Frontend/src/components/ResultsCount.vue

@@ -5,6 +5,8 @@ import { useDateFormatter } from "@/composables/dateFormatter";
 const props = defineProps<{
   displayed: number;
   total: number;
+  // The total only covers the instances that answered (partial scatter-gather result)
+  incomplete?: boolean;
   durationMs?: number | null;
   completedAt?: string | null;
 }>();
@@ -34,7 +36,7 @@ const formattedDuration = computed(() => {
 <template>
   <div class="col format-showing-results">
     <div>
-      Showing {{ formattedDisplayed }} of {{ formattedTotal }} result(s)<template v-if="formattedDuration"> · took {{ formattedDuration }}</template
+      Showing {{ formattedDisplayed }} of {{ incomplete ? "at least " : "" }}{{ formattedTotal }} result(s)<template v-if="formattedDuration"> · took {{ formattedDuration }}</template
       ><template v-if="ranAgo">
         · ran <span :title="ranTooltip" data-testid="ran-ago">{{ ranAgo }}</span></template
       >

src/Frontend/src/components/audit/AuditList.spec.ts

@@ -414,6 +414,65 @@ describe("FEATURE: Audit Messages Query State", () => {
     });
   });
 
+  describe("RULE: Partial results name the instances whose data is missing", () => {
+    test("EXAMPLE: The warning lists each missing instance with its reason", async () => {
+      const { store } = await renderAuditList([createMessage()]);
+
+      await waitForFirstLoadToComplete();
+
+      store.incompleteInstances = [
+        { instanceId: "audit-2", reason: "timeout" },
+        { instanceId: "audit-3", reason: "unavailable" },
+      ];
+      await nextTick();
+
+      const warning = screen.getByTestId("query-incomplete");
+      expect(warning.textContent).toContain("audit-2 (timed out)");
+      expect(warning.textContent).toContain("audit-3 (unreachable)");
+      // the partial data itself stays on screen
+      expect(screen.queryAllByTestId("message-item").length).toBeGreaterThan(0);
+    });
+
+    test("EXAMPLE: A ServiceControl instance id is shown as host and port, with the API URL on hover", async () => {
+      const { store } = await renderAuditList([createMessage()]);
+
+      await waitForFirstLoadToComplete();
+
+      store.incompleteInstances = [{ instanceId: "aHR0cDovL2xvY2FsaG9zdDo0NDQ0NC9hcGkv", reason: "timeout" }];
+      await nextTick();
+
+      const warning = screen.getByTestId("query-incomplete");
+      expect(warning.textContent).toContain("localhost:44444 (timed out)");
+      expect(warning.textContent).not.toContain("aHR0");
+      expect(warning.querySelector('[title="http://localhost:44444/api/"]')).not.toBeNull();
+    });
+
+    test("EXAMPLE: No warning while a retry is in flight or when results are complete", async () => {
+      const { store, isRefreshing } = await renderAuditList([createMessage()]);
+
+      await waitForFirstLoadToComplete();
+      expect(screen.queryByTestId("query-incomplete")).not.toBeInTheDocument();
+
+      store.incompleteInstances = [{ instanceId: "audit-2", reason: "timeout" }];
+      isRefreshing.value = true;
+      await nextTick();
+
+      expect(screen.queryByTestId("query-incomplete")).not.toBeInTheDocument();
+    });
+
+    test("EXAMPLE: A query stopped by the server's time limit says so", async () => {
+      const { store } = await renderAuditList([]);
+
+      await waitForFirstLoadToComplete();
+
+      store.queryFailed = true;
+      store.queryTimedOut = true;
+      await nextTick();
+
+      expect(screen.getByTestId("query-error").textContent).toContain("exceeded the ServiceControl query time limit");
+    });
+  });
+
   describe("RULE: A query-control change results in exactly one query", () => {
     test("EXAMPLE: Changing the filter text fires a single query", async () => {
       const { refreshNow, store } = await renderAuditList([createMessage()]);

src/Frontend/src/components/audit/AuditList.vue

@@ -17,9 +17,11 @@ import { CapabilityStatus } from "@/components/platformcapabilities/constants";
 import PageBanner, { type BannerMessage } from "@/components/PageBanner.vue";
 import { useConfigurationStore } from "@/stores/ConfigurationStore";
 import { loadDefaultRange, narrowingPresets, resolveTimeRange, type RangePreset } from "@/components/audit/timeRange";
+import { describeIncompleteReason, describeInstance } from "@/components/incompleteResults";
 
 const store = useAuditStore();
-const { messages, newMessageIds, totalCount, sortBy, messageFilterString, selectedEndpointName, itemsPerPage, timeRangeFrom, timeRangeTo, queryFailed, queryStartedAt, queryDurationMs, queryCompletedAt } = storeToRefs(store);
+const { messages, newMessageIds, totalCount, sortBy, messageFilterString, selectedEndpointName, itemsPerPage, timeRangeFrom, timeRangeTo, queryFailed, queryTimedOut, incompleteInstances, queryStartedAt, queryDurationMs, queryCompletedAt } =
+  storeToRefs(store);
 const newRowIds = computed(() => new Set(newMessageIds.value));
 const route = useRoute();
 const router = useRouter();
@@ -72,6 +74,15 @@ function applyNarrowing(preset: RangePreset) {
   timeRangeTo.value = preset.to;
 }
 
+// "audit-2:44444 (timed out), audit-3:44444 (unreachable)" — why the current page is partial.
+// The id is the instance's base64 API URL; readers get host and port, the URL on hover.
+const incompleteSummary = computed(() =>
+  incompleteInstances.value.map((instance) => {
+    const { label, apiUrl } = describeInstance(instance.instanceId);
+    return { key: instance.instanceId, label, apiUrl, reason: describeIncompleteReason(instance.reason) };
+  })
+);
+
 onBeforeMount(() => {
   setQuery();
 
@@ -186,20 +197,27 @@ watch(autoRefreshValue, (newValue) => {
         </FiltersPanel>
       </div>
       <div class="row results-row">
-        <ResultsCount :displayed="messages.length" :total="totalCount" :duration-ms="queryDurationMs" :completed-at="queryCompletedAt" />
+        <ResultsCount :displayed="messages.length" :total="totalCount" :incomplete="incompleteInstances.length > 0" :duration-ms="queryDurationMs" :completed-at="queryCompletedAt" />
         <ResultsOptions />
       </div>
       <PageBanner v-if="bannerMessage && isMassTransitConnected === false" :message="bannerMessage" :show-action="showBannerAction" @action="showWizard = true" />
     </div>
     <WizardDialog v-if="showWizard" title="Getting Started with Auditing" :pages="wizardPages" @close="showWizard = false" />
     <div v-if="queryFailed && !queryInProgress" class="query-error" role="alert" data-testid="query-error">
-      <strong>The query failed or took too long and was stopped.</strong>
+      <strong v-if="queryTimedOut">The query exceeded the ServiceControl query time limit and was stopped.</strong>
+      <strong v-else>The query failed or took too long and was stopped.</strong>
       <p v-if="hasNoTimeFilter">This query has no time filter, so it scans the whole audit store. Bounding it is the quickest fix — or try again in an off-peak period.</p>
       <p v-else>Query cost grows with the size of the time window. Try a narrower range, add a search term or endpoint filter, or reduce the number of results ("Show").</p>
       <div v-if="narrowOptions.length > 0" class="error-actions">
         <button v-for="preset in narrowOptions" :key="preset.label" type="button" class="narrow-action" data-testid="narrow-range" @click="applyNarrowing(preset)">{{ preset.label }}</button>
       </div>
     </div>
+    <div v-if="incompleteInstances.length > 0 && !queryInProgress" class="query-incomplete" role="status" data-testid="query-incomplete">
+      <strong>Partial results.</strong> No data from
+      <template v-for="(instance, index) in incompleteSummary" :key="instance.key"
+        ><template v-if="index > 0">, </template><span :title="instance.apiUrl ?? undefined">{{ instance.label }} ({{ instance.reason }})</span></template
+      >.
+    </div>
     <div class="row results-table">
       <!-- Only when there is nothing to show yet. A re-fetch over existing rows leaves them
            visible and usable: the refresh button already signals the running query -->
@@ -237,6 +255,15 @@ watch(autoRefreshValue, (newValue) => {
   margin: 0.25rem 0 0;
 }
 
+.query-incomplete {
+  margin-top: 1rem;
+  padding: 0.6rem 1rem;
+  border: 1px solid #f0e0b6;
+  border-left: 4px solid #f0ad4e;
+  border-radius: 4px;
+  background-color: #fdf9ef;
+}
+
 .error-actions {
   display: flex;
   gap: 0.5rem;

src/Frontend/src/components/incompleteResults.spec.ts

@@ -0,0 +1,62 @@
+import { describe, expect, test } from "vitest";
+import { decodeInstanceId, describeIncompleteReason, describeInstance, parseIncompleteResults } from "@/components/incompleteResults";
+
+describe("FEATURE: Incomplete-results header parsing", () => {
+  test("EXAMPLE: A missing header means the response is complete", () => {
+    expect(parseIncompleteResults(null)).toEqual([]);
+    expect(parseIncompleteResults("")).toEqual([]);
+  });
+
+  test("EXAMPLE: Entries carry the instance and why it contributed nothing", () => {
+    expect(parseIncompleteResults("audit-2:timeout, audit-3:unavailable, audit-4:error")).toEqual([
+      { instanceId: "audit-2", reason: "timeout" },
+      { instanceId: "audit-3", reason: "unavailable" },
+      { instanceId: "audit-4", reason: "error" },
+    ]);
+  });
+
+  test("EXAMPLE: The reason follows the last colon, so instance ids can contain colons", () => {
+    expect(parseIncompleteResults("http://audit-host:44444/api:timeout")).toEqual([{ instanceId: "http://audit-host:44444/api", reason: "timeout" }]);
+  });
+
+  test("EXAMPLE: An unknown reason is treated as an error", () => {
+    expect(parseIncompleteResults("audit-2:exploded")).toEqual([{ instanceId: "audit-2", reason: "error" }]);
+    expect(parseIncompleteResults("audit-2")).toEqual([{ instanceId: "audit-2", reason: "error" }]);
+  });
+
+  test("EXAMPLE: Reasons read as prose", () => {
+    expect(describeIncompleteReason("timeout")).toBe("timed out");
+    expect(describeIncompleteReason("unavailable")).toBe("unreachable");
+    expect(describeIncompleteReason("error")).toBe("returned an error");
+  });
+
+  // ServiceControl identifies an instance by its API URL, lower-cased and base64 encoded with
+  // the URL-safe alphabet: '-' for '+', '_' for '/', '.' for '=' (InstanceIdGenerator)
+  describe("RULE: Instance ids are ServiceControl's base64 API URLs and are shown as host and port", () => {
+    test("EXAMPLE: An id decodes to the instance's API URL", () => {
+      expect(decodeInstanceId("aHR0cDovL2xvY2FsaG9zdDo0NDQ0NC9hcGkv")).toBe("http://localhost:44444/api/");
+    });
+
+    test("EXAMPLE: URL-safe substitutions and dot padding are reversed before decoding", () => {
+      expect(decodeInstanceId("aHR0cDovL3NjLWF1ZGl0OjQ0NDQ0L2FwaQ..")).toBe("http://sc-audit:44444/api");
+    });
+
+    test("EXAMPLE: Something that is not a base64 URL is left alone", () => {
+      expect(decodeInstanceId("audit-2")).toBeNull();
+      expect(decodeInstanceId("")).toBeNull();
+    });
+
+    test("EXAMPLE: The label is host and port; scheme and path are noise for a reader", () => {
+      expect(describeInstance("aHR0cDovL2xvY2FsaG9zdDo0NDQ0NC9hcGkv")).toEqual({ label: "localhost:44444", apiUrl: "http://localhost:44444/api/" });
+      expect(describeInstance("aHR0cDovL2F1ZGl0LTIuaW50ZXJuYWw6MzMzMzMvYXBp")).toEqual({ label: "audit-2.internal:33333", apiUrl: "http://audit-2.internal:33333/api" });
+    });
+
+    test("EXAMPLE: A default port is omitted from the label", () => {
+      expect(describeInstance("aHR0cHM6Ly9hdWRpdC5leGFtcGxlLmNvbS9hcGkv")).toEqual({ label: "audit.example.com", apiUrl: "https://audit.example.com/api/" });
+    });
+
+    test("EXAMPLE: An id that does not decode is shown as it is, with nothing to hover", () => {
+      expect(describeInstance("audit-2")).toEqual({ label: "audit-2", apiUrl: null });
+    });
+  });
+});

src/Frontend/src/components/incompleteResults.ts

@@ -0,0 +1,74 @@
+// Parsing for the X-Particular-Incomplete-Results response header.
+//
+// ServiceControl's composite (scatter-gather) endpoints return a bare array,
+// so when an instance contributes nothing the response stays 200 with the
+// partial data and this header names what is missing, as comma-separated
+// "instanceId:reason" entries (reasons: timeout, unavailable, error).
+// A response without the header is complete.
+
+export type IncompleteReason = "timeout" | "unavailable" | "error";
+
+export interface IncompleteInstance {
+  instanceId: string;
+  reason: IncompleteReason;
+}
+
+export const incompleteResultsHeader = "X-Particular-Incomplete-Results";
+
+const knownReasons: ReadonlySet<string> = new Set(["timeout", "unavailable", "error"]);
+
+export function parseIncompleteResults(header: string | null): IncompleteInstance[] {
+  if (!header) return [];
+  return header
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry !== "")
+    .map((entry) => {
+      // The reason follows the last colon; instance ids can contain colons (e.g. URLs)
+      const separator = entry.lastIndexOf(":");
+      const instanceId = separator > 0 ? entry.slice(0, separator) : entry;
+      const rawReason = separator > 0 ? entry.slice(separator + 1).trim() : "";
+      const reason: IncompleteReason = knownReasons.has(rawReason) ? (rawReason as IncompleteReason) : "error";
+      return { instanceId, reason };
+    });
+}
+
+export function describeIncompleteReason(reason: IncompleteReason): string {
+  switch (reason) {
+    case "timeout":
+      return "timed out";
+    case "unavailable":
+      return "unreachable";
+    default:
+      return "returned an error";
+  }
+}
+
+// ServiceControl identifies an instance by its API URL, lower-cased and base64 encoded with the
+// URL-safe alphabet ('-' for '+', '_' for '/', '.' for '=': InstanceIdGenerator.FromApiUrl).
+// Returns the URL, or null when the id is not one of those.
+export function decodeInstanceId(instanceId: string): string | null {
+  if (instanceId === "") return null;
+  try {
+    const binary = atob(instanceId.replace(/-/g, "+").replace(/_/g, "/").replace(/\./g, "="));
+    const decoded = new TextDecoder("utf-8", { fatal: true }).decode(Uint8Array.from(binary, (c) => c.charCodeAt(0)));
+    const url = new URL(decoded);
+    return url.protocol === "http:" || url.protocol === "https:" ? decoded : null;
+  } catch {
+    return null;
+  }
+}
+
+export interface InstanceDescription {
+  // What a reader needs to tell instances apart: host and port (default port omitted)
+  label: string;
+  // The full API URL for a tooltip, when the id decoded to one
+  apiUrl: string | null;
+}
+
+export function describeInstance(instanceId: string): InstanceDescription {
+  const apiUrl = decodeInstanceId(instanceId);
+  if (apiUrl === null) return { label: instanceId, apiUrl: null };
+  // URL.host already omits the port when it is the scheme's default
+  return { label: new URL(apiUrl).host, apiUrl };
+}

src/Frontend/src/stores/AuditStore.spec.ts

@@ -12,9 +12,10 @@ vi.mock("@/components/serviceControlClient", () => ({
 }));
 
 import { useAuditStore } from "@/stores/AuditStore";
+import { HttpError } from "@/utils/HttpError";
 
-function responseWithTotalCount(count: number): Response {
-  return { headers: new Headers({ "total-count": count.toString() }) } as Response;
+function responseWithTotalCount(count: number, extraHeaders: Record<string, string> = {}): Response {
+  return { headers: new Headers({ "total-count": count.toString(), ...extraHeaders }) } as Response;
 }
 
 function abortablePendingFetch(onSignal?: (signal: AbortSignal | undefined) => void) {
@@ -59,6 +60,35 @@ describe("AuditStore refresh", () => {
     expect(store.queryFailed).toBe(false);
   });
 
+  test("a partial response surfaces the instances whose data is missing", async () => {
+    fetchTypedFromServiceControl.mockResolvedValueOnce([responseWithTotalCount(1, { "X-Particular-Incomplete-Results": "audit-2:timeout, audit-3:unavailable" }), [message]]);
+    const store = useAuditStore();
+
+    await store.refresh();
+
+    expect(store.queryFailed).toBe(false);
+    expect(store.incompleteInstances).toEqual([
+      { instanceId: "audit-2", reason: "timeout" },
+      { instanceId: "audit-3", reason: "unavailable" },
+    ]);
+
+    // The next complete response clears the warning
+    fetchTypedFromServiceControl.mockResolvedValueOnce([responseWithTotalCount(1), [message]]);
+    await store.refresh();
+    expect(store.incompleteInstances).toEqual([]);
+  });
+
+  test("a 504 is flagged as the server's query time limit", async () => {
+    fetchTypedFromServiceControl.mockRejectedValue(new HttpError(504, "Gateway Timeout"));
+    const store = useAuditStore();
+
+    await store.refresh();
+
+    expect(store.queryFailed).toBe(true);
+    expect(store.queryTimedOut).toBe(true);
+    expect(store.incompleteInstances).toEqual([]);
+  });
+
   describe("new rows since the previous result of the same query", () => {
     const msg = (id: string) => ({ id });
 
@@ -250,6 +280,21 @@ describe("AuditStore refresh", () => {
     expect(store.newMessageIds).toEqual([]);
   });
 
+  test("clearResults forgets the incomplete-results state", async () => {
+    const store = useAuditStore();
+    fetchTypedFromServiceControl.mockResolvedValueOnce([responseWithTotalCount(1, { "X-Particular-Incomplete-Results": "audit-2:timeout" }), [message]]);
+    await store.refresh();
+    expect(store.incompleteInstances).toHaveLength(1);
+    fetchTypedFromServiceControl.mockRejectedValueOnce(new HttpError(504, "Gateway Timeout"));
+    await store.refresh();
+    expect(store.queryTimedOut).toBe(true);
+
+    store.clearResults();
+
+    expect(store.incompleteInstances).toEqual([]);
+    expect(store.queryTimedOut).toBe(false);
+  });
+
   test("a superseded query is not reported as a failure", async () => {
     const store = useAuditStore();