diff --git a/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json b/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json new file mode 100644 index 000000000..e68477ffe --- /dev/null +++ b/docs/outstanding-issues-inbox/2054f3ed-afce-4be9-a7e2-88cdeb8b147c.json @@ -0,0 +1,14 @@ +{ + "version": 2, + "id": "2054f3ed-afce-4be9-a7e2-88cdeb8b147c", + "createdOn": "2026-09-07", + "action": "add", + "payload": { + "pri": "P2", + "type": "issue", + "summary": "Weekly eval canary has failed on the same three answer-quality cases since at least 2026-08-23, and nothing tracks it", + "detail": "Golden retrieval passes throughout: run 34099357211 (2026-09-07, repository_dispatch) recorded cases=36, document_recall@5=1, content_recall@5=1, mrr@10=0.8921, zero failing retrieval cases. The run is red only because the separate live answer-quality subset fails its blocking threshold: 'RAG source_backed_review_fallback unaccounted' on clozapine-monitoring (high_confidence_extractive_retrieval; final_quality_gate:guidance_wrapper_fragment; extractive_quality_gate:provider_source_gap), clozapine-monitoring-paraphrase (identical signature), and nocc-requirements (strong_routine_retrieval; generation_fallback:generation_quality_failed; extractive_quality_gate:guidance_wrapper_fragment). The five most recent canary runs are all red: 34099357211, 34050934409 (scheduled), 33582708071, 33327352850 (scheduled), 32656925301 (scheduled, 2026-08-23). CONSEQUENCE: the canary is the pair-comparison instrument for every protected RAG change, and a permanently red run trains readers to ignore it. A future retrieval regression would land inside an already-red signal. NEXT: this is the same failure family as #J8SJQ9 (source-backed review stub where an authoritative source exists) but names different cases, so check that row for overlap before starting. First diagnostic step is whether generation was attempted at all for these three, exactly as #J8SJQ9 sets out. STOP: do not add acceptSourceOnly to these eval cases and do not relax the threshold to get the canary green. The retrieval half of the run remains a valid baseline in the meantime.", + "source": "Canary run logs 34099357211 and 34050934409, read during PR #2711 (source authority register), 2026-09-07", + "issueUlid": "01M1XSYJ9DSXPQ0A9B1TZ8J6NY" + } +} diff --git a/docs/rag-behaviour/safeguards.md b/docs/rag-behaviour/safeguards.md index 5f0f64154..d01725d45 100644 --- a/docs/rag-behaviour/safeguards.md +++ b/docs/rag-behaviour/safeguards.md @@ -13,6 +13,13 @@ Code and ground truth whose edits change (or re-measure) retrieval/ranking behav `src/lib/released-search-order.ts`, `src/lib/ranking-config.ts`, `src/lib/evidence.ts`, `src/lib/result-sort.ts`, `src/lib/answer-ranking.ts`, `src/lib/evidence-relevance.ts`, `src/lib/semantic-rerank.ts`, `src/lib/eval-document-matching.ts` +- `src/lib/source-authority-registry.ts`, `src/lib/australian-source-priority.ts` — one step + further back, but still ordering: `classifySourceAuthority` assigns a result's tier, + `australianSourceTier` reads it, and `selectAustralianClinicalContext` orders and trims the + model's context by it. Registering or re-scoping a publisher moves that ordering without + touching a score. Added 2026-09-07 (PR #2711 declared its RAG impact voluntarily because the + gate did not yet ask). Note this is authority _classification_, not a governance ranking + penalty or boost — the latter stays refuted, see `refuted-approaches.md` § Refutation 3. - `scripts/eval-retrieval.ts`, `scripts/lib/clinical-aliases.ts`, `scripts/lib/ranking-tuning.ts`, `scripts/lib/ranking-snapshot-builder.ts`, `scripts/build-ranking-snapshot.ts`, `scripts/tune-search-weights.ts` diff --git a/docs/source-acquisition-protocol.md b/docs/source-acquisition-protocol.md index fa1fa4eb1..6aed9c475 100644 --- a/docs/source-acquisition-protocol.md +++ b/docs/source-acquisition-protocol.md @@ -135,9 +135,11 @@ prints the outstanding queue, most local first. - **An unregistered publisher cannot leave D band.** The catalogue derives jurisdiction solely from `src/lib/source-authority-registry.ts`, so a source from a publisher that is not registered has no jurisdiction and is forced to D. Therapeutic Guidelines, the Australian - Medicines Handbook, Australian Prescriber, RACP, Cochrane, AIHW, HealthyWA and the Mental - Health Commission WA are all currently unregistered. The gate detects this and names the - publisher rather than failing obscurely. + Medicines Handbook, Australian Prescriber, RACP, Cochrane, AIHW, Healthdirect Australia, + HealthyWA and the Mental Health Commission WA were registered on 2026-09-07. Beyond Blue, + the NSW Agency for Clinical Innovation and Western Sydney Local Health District are examples + that remain unregistered. The gate detects this and names the publisher rather than failing + obscurely. Registering a publisher changes `classifySourceAuthority`, which retrieval selection uses to tier results. That makes it a retrieval behaviour change under the RAG ranking protection diff --git a/scripts/pr-policy.mjs b/scripts/pr-policy.mjs index 5213eac91..4fa917966 100644 --- a/scripts/pr-policy.mjs +++ b/scripts/pr-policy.mjs @@ -159,6 +159,13 @@ const migrationSubjectPattern = const ragRankingPatterns = [ /^src\/lib\/rag\//, /^src\/lib\/(?:clinical-search|retrieval-selection|released-search-order|ranking-config|evidence|result-sort|answer-ranking|evidence-relevance|semantic-rerank|eval-document-matching)\.ts$/, + // Source authority feeds ordering too, one step further back: classifySourceAuthority tiers a + // result, australianSourceTier reads that tier, and selectAustralianClinicalContext orders and + // trims the model's context by it. Registering a publisher is therefore a retrieval behaviour + // change even though neither file computes a score. Added 2026-09-07 after PR #2711 registered + // nine Australian publishers and had to declare its RAG impact voluntarily, because this gate + // did not ask. + /^src\/lib\/(?:source-authority-registry|australian-source-priority)\.ts$/, /^scripts\/(?:eval-retrieval|build-ranking-snapshot|tune-search-weights)\.ts$/, /^scripts\/lib\/(?:clinical-aliases|ranking-tuning|ranking-snapshot-builder)\.ts$/, /^scripts\/fixtures\/(?:rag-retrieval-golden|rag-ranking-candidate-snapshot\.v1)\.json$/, @@ -690,6 +697,8 @@ function selfTest() { // The golden fixture and contract tests are protected surfaces too. assert.equal(classifyPullRequestFiles(["scripts/fixtures/rag-retrieval-golden.json"]).ragRanking, true); assert.equal(classifyPullRequestFiles(["tests/ranking-tuning.test.ts"]).ragRanking, true); + assert.equal(classifyPullRequestFiles(["src/lib/source-authority-registry.ts"]).ragRanking, true); + assert.equal(classifyPullRequestFiles(["src/lib/australian-source-priority.ts"]).ragRanking, true); // Answer synthesis is clinical-risk but NOT rag-ranking (retrieval ordering is the // protected axis here; generation keeps the governance gate only). assert.equal(classifyPullRequestFiles(["src/lib/answer-synthesis.ts"]).ragRanking, false); diff --git a/src/lib/source-authority-registry.ts b/src/lib/source-authority-registry.ts index 5633cb1a5..ad953efa6 100644 --- a/src/lib/source-authority-registry.ts +++ b/src/lib/source-authority-registry.ts @@ -195,6 +195,15 @@ export const sourceAuthorityRegistry = [ designation: "official", officialBasis: "wa_hospital", }), + authority({ + key: "healthywa", + codes: ["HEALTHYWA"], + publisher: "HealthyWA", + publisherAliases: ["Healthy WA", "HealthyWA - Western Australian Government health information"], + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), authority({ key: "king-edward-memorial-hospital", codes: ["KEMH", "KEMHS"], @@ -205,6 +214,19 @@ export const sourceAuthorityRegistry = [ designation: "official", officialBasis: "wa_hospital", }), + authority({ + key: "mental-health-commission-wa", + codes: ["MHCWA", "MHC WA"], + publisher: "Mental Health Commission WA", + publisherAliases: [ + "Mental Health Commission Western Australia", + "Western Australian Mental Health Commission", + "Government of Western Australia Mental Health Commission", + ], + jurisdictions: waJurisdictions, + scope: "wa", + tier: "wa_validated", + }), authority({ key: "north-metropolitan-health-service", codes: ["NMHS"], @@ -275,9 +297,17 @@ export const sourceAuthorityRegistry = [ scope: "australian_national", tier: "australian_national", }), + authority({ + key: "aihw", + codes: ["AIHW"], + publisher: "Australian Institute of Health and Welfare", + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), authority({ key: "australian-prescriber", - codes: ["AUSPRES"], + codes: ["AUSPRES", "AUSTPRESC"], publisher: "Australian Prescriber", jurisdictions: nationalJurisdictions, scope: "australian_national", @@ -293,6 +323,24 @@ export const sourceAuthorityRegistry = [ scope: "australian_national", tier: "australian_national", }), + authority({ + key: "australian-medicines-handbook", + codes: ["AMH"], + publisher: "Australian Medicines Handbook", + publisherAliases: ["Australian Medicines Handbook Pty Ltd"], + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), + authority({ + key: "healthdirect-australia", + codes: ["HEALTHDIRECT"], + publisher: "Healthdirect Australia", + publisherAliases: ["healthdirect"], + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), authority({ key: "nhmrc", codes: ["NHMRC"], @@ -327,6 +375,14 @@ export const sourceAuthorityRegistry = [ scope: "australian_national", tier: "australian_national", }), + authority({ + key: "racp", + codes: ["RACP"], + publisher: "Royal Australasian College of Physicians", + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), authority({ key: "ranzcp", codes: ["RANZCP"], @@ -343,6 +399,15 @@ export const sourceAuthorityRegistry = [ scope: "australian_national", tier: "australian_national", }), + authority({ + key: "therapeutic-guidelines", + codes: ["TG", "ETG"], + publisher: "Therapeutic Guidelines", + publisherAliases: ["Therapeutic Guidelines Limited", "Therapeutic Guidelines Ltd", "eTG complete"], + jurisdictions: nationalJurisdictions, + scope: "australian_national", + tier: "australian_national", + }), ...[ ["act-health", "ACTHEALTH", "ACT Health", "Australia/ACT"], ["nsw-health", "NSWHEALTH", "NSW Health", "Australia/NSW"], @@ -370,6 +435,15 @@ export const sourceAuthorityRegistry = [ scope: "international", tier: "supplementary", }), + authority({ + key: "cochrane", + codes: ["COCHRANE"], + publisher: "Cochrane", + publisherAliases: ["The Cochrane Collaboration", "Cochrane Library", "Cochrane Database of Systematic Reviews"], + jurisdictions: ["International", "Global"], + scope: "international", + tier: "supplementary", + }), authority({ key: "nice", codes: ["NICE"], diff --git a/tests/rag-context-budget.test.ts b/tests/rag-context-budget.test.ts index 0c5fcc9ad..68ab62381 100644 --- a/tests/rag-context-budget.test.ts +++ b/tests/rag-context-budget.test.ts @@ -427,6 +427,50 @@ describe("RAG model context budgeting", () => { expect(selected.map((result) => result.id)).toEqual(["chunk-1", "chunk-2", "chunk-3", "chunk-4"]); }); + it("orders newly registered WA and national publishers ahead of equally relevant supplementary context", () => { + const selected = selectModelContextResults({ + routeMode: "strong", + queryClass: "medication_dose_risk", + crossDocument: false, + results: [ + withRelevance(bmjSupplementarySource(1), "direct"), + withRelevance( + governedSource(2, { + documentId: "tg-doc", + publisherCode: "TG", + publisher: "Therapeutic Guidelines", + jurisdiction: "Australia/National", + validation: "unverified", + }), + "direct", + ), + withRelevance( + governedSource(3, { + documentId: "healthywa-doc", + publisherCode: "HEALTHYWA", + publisher: "HealthyWA", + jurisdiction: "Australia/WA", + }), + "direct", + ), + withRelevance( + governedSource(4, { + documentId: "cochrane-doc", + publisherCode: "COCHRANE", + publisher: "Cochrane", + jurisdiction: "International", + validation: "unverified", + }), + "direct", + ), + ], + }); + + // Registration changes only same-relevance context ordering: HealthyWA is WA-validated, + // Therapeutic Guidelines is Australian national, while Cochrane remains supplementary. + expect(selected.map((result) => result.id)).toEqual(["chunk-3", "chunk-2", "chunk-1", "chunk-4"]); + }); + it("keeps supplementary evidence when authoritative Australian coverage is not sufficient", () => { const selected = selectModelContextResults({ routeMode: "strong", diff --git a/tests/source-acquisition-ledger.test.ts b/tests/source-acquisition-ledger.test.ts index 81ed1a0ef..e002adae5 100644 --- a/tests/source-acquisition-ledger.test.ts +++ b/tests/source-acquisition-ledger.test.ts @@ -13,6 +13,7 @@ import { sourceAcquisitionRecords, type SourceAcquisitionRecord, } from "@/lib/sources/acquisition-ledger"; +import { sourceAuthorityForPublisher } from "@/lib/source-authority-registry"; import { canonicalizeSourceReferences } from "@/lib/sources/catalogue-core"; import { classifySourceAuthority, @@ -47,6 +48,10 @@ const baseRecord: SourceAcquisitionRecord = { notes: null, }; +// A real Australian clinical publisher that is deliberately absent from the source authority +// register, so the "unrecognised publisher" path has a genuine example to exercise. +const unregisteredPublisher = "Beyond Blue"; + function record(overrides: Partial = {}): SourceAcquisitionRecord { return { ...baseRecord, ...overrides }; } @@ -174,7 +179,11 @@ describe("source acquisition rungs", () => { }); it("names the register as the blocker when a publisher is not recognised, and warns that fixing it moves retrieval", () => { - const [issue] = issuesFor({ publisher: "Therapeutic Guidelines Limited", publisherCode: null, canonicalUrl: null }); + // The example publisher has to be one the register genuinely does not carry, so assert that + // here rather than trusting the constant: Therapeutic Guidelines used to sit in this slot and + // the premise silently went false the day it was registered. + expect(sourceAuthorityForPublisher(unregisteredPublisher)).toBeNull(); + const [issue] = issuesFor({ publisher: unregisteredPublisher, publisherCode: null, canonicalUrl: null }); expect(issue).toContain("is not in the source authority register"); expect(issue).toContain("can never leave D band"); expect(issue).toContain("changes retrieval selection"); diff --git a/tests/source-metadata.test.ts b/tests/source-metadata.test.ts index 6de5252a3..ac3611d01 100644 --- a/tests/source-metadata.test.ts +++ b/tests/source-metadata.test.ts @@ -392,6 +392,129 @@ describe("source authority classification", () => { expect(classifySourceAuthority(metadata)).toMatchObject({ tier, matchedBy, conflict: false }); }); + it.each([ + ["HealthyWA", "HEALTHYWA", "HealthyWA", "Australia/WA", "wa_validated", "healthywa"], + [ + "Mental Health Commission WA", + "MHCWA", + "Mental Health Commission WA", + "Australia/WA", + "wa_validated", + "mental-health-commission-wa", + ], + [ + "Australian Institute of Health and Welfare", + "AIHW", + "Australian Institute of Health and Welfare", + "Australia/National", + "australian_national", + "aihw", + ], + [ + "Australian Medicines Handbook", + "AMH", + "Australian Medicines Handbook", + "Australia/National", + "australian_national", + "australian-medicines-handbook", + ], + [ + "Healthdirect Australia", + "HEALTHDIRECT", + "Healthdirect Australia", + "Australia/National", + "australian_national", + "healthdirect-australia", + ], + [ + "Royal Australasian College of Physicians", + "RACP", + "Royal Australasian College of Physicians", + "Australia/National", + "australian_national", + "racp", + ], + [ + "Therapeutic Guidelines", + "TG", + "Therapeutic Guidelines", + "Australia/National", + "australian_national", + "therapeutic-guidelines", + ], + ["Cochrane", "COCHRANE", "Cochrane", "International", "supplementary", "cochrane"], + ] as const)( + "classifies registered %s from code-backed metadata", + (label, publisherCode, publisher, jurisdiction, tier, authorityKey) => { + // This would fail if a publisher were removed, assigned the wrong tier, or associated with a + // different authority. The expected values are deliberately literal rather than derived from + // the registry under test. + expect( + classifySourceAuthority({ + ...usable, + publisher_code: publisherCode, + publisher, + jurisdiction, + }), + ).toMatchObject({ authorityKey, tier, matchedBy: "publisher_code", conflict: false }); + }, + ); + + it.each([ + ["Healthy WA", "Australia/WA", "healthywa", "wa_validated"], + ["Mental Health Commission Western Australia", "Australia/WA", "mental-health-commission-wa", "wa_validated"], + [ + "Australian Medicines Handbook Pty Ltd", + "Australia/National", + "australian-medicines-handbook", + "australian_national", + ], + ["healthdirect", "Australia/National", "healthdirect-australia", "australian_national"], + ["Therapeutic Guidelines Ltd", "Australia/National", "therapeutic-guidelines", "australian_national"], + ["Cochrane Library", "International", "cochrane", "supplementary"], + ] as const)( + "classifies registered %s from a jurisdiction-bound alias", + (publisher, jurisdiction, authorityKey, tier) => { + expect(classifySourceAuthority({ ...usable, publisher, jurisdiction })).toMatchObject({ + authorityKey, + tier, + matchedBy: "publisher_alias", + conflict: false, + }); + }, + ); + + it("keeps catalogue-only Australian Prescriber out of runtime authority priority", () => { + for (const publisherCode of ["AUSPRES", "AUSTPRESC"] as const) { + expect( + classifySourceAuthority({ + ...usable, + publisher_code: publisherCode, + publisher: "Australian Prescriber", + jurisdiction: "Australia/National", + }), + ).toMatchObject({ + tier: "supplementary", + designation: "unclassified", + authorityKey: null, + matchedBy: "none", + codeKnown: false, + }); + } + }); + + it("keeps an unvalidated HealthyWA source supplementary", () => { + expect( + classifySourceAuthority({ + ...usable, + publisher_code: "HEALTHYWA", + publisher: "HealthyWA", + jurisdiction: "Australia/WA", + clinical_validation_status: "unverified", + }), + ).toMatchObject({ tier: "supplementary", authorityKey: "healthywa" }); + }); + it.each([ { label: "international code with WA jurisdiction",