Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions hub/.devcontainer/decision-walkthrough.compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
services:
app:
ports:
- "127.0.0.1:5173:5173"
environment:
NODE_ENV: development
HUB_LOCAL_MODE: "true"
HUB_LOCAL_SIGN_IN: "true"
HUB_APPLICATION_ORIGIN: http://127.0.0.1:5173
HUB_PREVIEW_ORIGIN: http://preview.localhost:3001
ARTIFACT_STORAGE_PROVIDER: filesystem
ARTIFACT_STORAGE_DIRECTORY: /workspaces/replica/hub/storage/decision-walkthrough-artifacts
REPLICA_LEARNING_ROOT: /workspaces/replica/hub/storage/decision-walkthrough-learning
REPLICA_LEARNING_CLI: /workspaces/replica/packages/learning/bin/replica-learning.mjs
REPLICA_DECISION_RUNTIME_URL: http://127.0.0.1:8010
REPLICA_DECISION_RUNTIME_AUTHENTICATION: environment-token
REPLICA_DECISION_RUNTIME_TOKEN: ${REPLICA_RUNTIME_TOKEN}
REPLICA_DECISION_RUNTIME_ENVIRONMENT: fixture
REPLICA_DECISION_RUNTIME_ALLOW_INSECURE_LOOPBACK: "true"
volumes:
- decision-walkthrough-hub-node-modules:/workspaces/replica/hub/node_modules
- decision-walkthrough-learning-node-modules:/workspaces/replica/packages/learning/node_modules
depends_on:
decision-runtime:
condition: service_healthy

decision-runtime:
build:
context: ../../packages/learning/environments/brand-check
dockerfile: server/runtime.Dockerfile
environment:
REPLICA_RUNTIME_TOKEN: ${REPLICA_RUNTIME_TOKEN}
command:
- python
- -m
- replica_brand_check.runtime_server
- --host
- 0.0.0.0
- --port
- "8010"
- --environment
- development
- --tenant
- local
- --repository
- local/decision-walkthrough
- --state
- /data/runtime
user: ${REPLICA_RUNTIME_USER:-10001:10001}
volumes:
- ${REPLICA_RUNTIME_STATE_DIR:?set REPLICA_RUNTIME_STATE_DIR}:/data/runtime
healthcheck:
test:
- CMD
- python
- -c
- >-
import json, urllib.request;
raise SystemExit(0 if json.load(urllib.request.urlopen("http://127.0.0.1:8010/health"))["status"] == "healthy" else 1)
interval: 5s
timeout: 5s
retries: 10

volumes:
decision-walkthrough-hub-node-modules:
decision-walkthrough-learning-node-modules:
2 changes: 2 additions & 0 deletions hub/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ SCHEDULED_JOB_CONCURRENCY=1
# REPLICA_DECISION_RUNTIME_AUTHENTICATION=environment-token
# REPLICA_DECISION_RUNTIME_TOKEN=
# REPLICA_DECISION_RUNTIME_ENVIRONMENT=fixture
# Development only: permit an explicit http:// loopback runtime.
# REPLICA_DECISION_RUNTIME_ALLOW_INSECURE_LOOPBACK=true

# Local development only. The devcontainer supplies its own database hostname.
# Run `npm run setup` before starting a database-backed API.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ function overview() {
{
id: "22222222-3333-4444-8555-666666666666",
runtimeCandidateId: "candidate-1",
backend: "fixture",
tasksetDigest: digest,
productionEligible: true,
baselineMeanReward: "0.61",
Expand Down Expand Up @@ -102,6 +103,17 @@ function overview() {
rollbackAvailable: true,
},
promotionHistory: [],
evaluationBuilds: [
{
id: "33333333-4444-4555-8666-777777777777",
number: 42,
prototype: {
id: "1",
name: "Settings",
branchName: "settings",
},
},
],
evaluations: [],
deletions: [],
}
Expand All @@ -123,6 +135,28 @@ describe("Decision-model contracts", () => {
).toBe(false)
})

it("bounds the successful Builds offered for evaluation", () => {
const response = overview()

expect(decisionModelOverviewResponseSchema.parse(response).evaluationBuilds).toEqual(
response.evaluationBuilds,
)
expect(
decisionModelOverviewResponseSchema.safeParse({
...response,
evaluationBuilds: Array.from({ length: 51 }, (_, index) => ({
id: `${String(index).padStart(8, "0")}-4444-4555-8666-777777777777`,
number: index + 1,
prototype: {
id: "1",
name: "Settings",
branchName: "settings",
},
})),
}).success,
).toBe(false)
})

it("keeps repository training settings within the supported bounds", () => {
const configuration = overview().configuration

Expand Down
35 changes: 23 additions & 12 deletions hub/client/application/hub/decision-model-contracts/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export const decisionModelConfigurationResponseSchema = z
})
.strict() satisfies z.ZodType<DecisionModelConfigurationResponse>

const decisionTrainingJobResponseSchema = z
export const decisionTrainingJobResponseSchema = z
.object({
id: identifier,
state: trainingState,
Expand All @@ -58,6 +58,7 @@ const decisionCandidateResponseSchema = z
.object({
id: identifier,
runtimeCandidateId: z.string().min(1),
backend: z.string().regex(/^[a-z][a-z0-9-]{0,99}$/),
tasksetDigest: digest,
productionEligible: z.boolean(),
baselineMeanReward: signedDecimal.nullable(),
Expand Down Expand Up @@ -174,6 +175,23 @@ export const decisionModelOverviewResponseSchema = z
})
.strict(),
),
evaluationBuilds: z
.array(
z
.object({
id: identifier,
number: z.number().int().positive(),
prototype: z
.object({
id: recordIdentifierSchema,
name: z.string().nullable(),
branchName: z.string().min(1),
})
.strict(),
})
.strict(),
)
.max(50),
evaluations: z.array(
z
.object({
Expand Down Expand Up @@ -241,16 +259,7 @@ export const decisionModelOverviewResponseSchema = z
export const decisionTrainingRequestResponseSchema = z
.object({
created: z.boolean(),
job: z
.object({
id: identifier,
state: trainingState,
runtimeJobId: z.string().nullable(),
candidateRuntimeId: z.string().nullable(),
failureCode: z.string().nullable(),
createdAt: timestamp,
})
.strict(),
job: decisionTrainingJobResponseSchema,
})
.strict() satisfies z.ZodType<DecisionTrainingRequestResponse>

Expand All @@ -266,13 +275,15 @@ export const decisionTrainingMutationResponseSchema = z.preprocess(
z.object({ id: identifier, state: trainingState }).loose(),
) satisfies z.ZodType<DecisionTrainingMutationResponse>

export const decisionBuildEvaluationMutationResponseSchema = z.object({ id: identifier }).loose()

export const decisionPromotionMutationResponseSchema = z
.object({
candidateId: identifier,
runtimeCandidateId: z.string().min(1),
previousCandidateId: identifier.nullable(),
previousRuntimeCandidateId: z.string().min(1).nullable(),
runtimeGeneration: count,
generation: count,
})
.strict() satisfies z.ZodType<DecisionPromotionMutationResponse>

Expand Down
23 changes: 13 additions & 10 deletions hub/client/application/hub/decision-model-contracts/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export interface DecisionTrainingJobResponse {
export interface DecisionCandidateResponse {
id: string
runtimeCandidateId: string
backend: string
tasksetDigest: string
productionEligible: boolean
baselineMeanReward: string | null
Expand Down Expand Up @@ -112,6 +113,11 @@ export interface DecisionModelOverviewResponse {
membershipId: string | null
createdAt: string
}>
evaluationBuilds: Array<{
id: string
number: number
prototype: { id: string; name: string | null; branchName: string }
}>
evaluations: Array<{
id: string
build: { id: string; number: number; prototypeId: string }
Expand Down Expand Up @@ -148,20 +154,17 @@ export interface DecisionModelOverviewResponse {
}

export type DecisionModelAvailability =
| { state: "available"; overview: DecisionModelOverviewResponse }
| {
state: "available"
overview: DecisionModelOverviewResponse
preparationStatus?: string
}
| { state: "not-configured" }
| { state: "unavailable"; error: string }

export interface DecisionTrainingRequestResponse {
created: boolean
job: {
id: string
state: DecisionTrainingJobResponse["state"]
runtimeJobId: string | null
candidateRuntimeId: string | null
failureCode: string | null
createdAt: string
}
job: DecisionTrainingJobResponse
}

export interface DecisionTrainingMutationResponse {
Expand All @@ -174,7 +177,7 @@ export interface DecisionPromotionMutationResponse {
runtimeCandidateId: string
previousCandidateId: string | null
previousRuntimeCandidateId: string | null
runtimeGeneration: number
generation: number
}

export interface DecisionCandidateRejectionMutationResponse {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@ interface DecisionModelMockState {
candidates: DecisionCandidateResponse[]
promotion: DecisionPromotionResponse | null
previousCandidateId: string | null
overview: Pick<DecisionModelOverviewResponse, "preflight" | "evaluations" | "deletions">
overview: Pick<
DecisionModelOverviewResponse,
"preflight" | "evaluationBuilds" | "evaluations" | "deletions"
>
}

const tasksetDigest = `sha256:${"b".repeat(64)}`
Expand Down Expand Up @@ -213,7 +216,7 @@ export function createDecisionModelHandlers(context: MockFlowContext) {
previousRuntimeCandidateId:
current.candidates.find((entry) => entry.id === current.previousCandidateId)
?.runtimeCandidateId ?? null,
runtimeGeneration: current.promotion.generation,
generation: current.promotion.generation,
})
},
context.session,
Expand Down Expand Up @@ -292,19 +295,80 @@ export function createDecisionModelHandlers(context: MockFlowContext) {
previousRuntimeCandidateId:
current.candidates.find((entry) => entry.id === current.previousCandidateId)
?.runtimeCandidateId ?? null,
runtimeGeneration: current.promotion.generation,
generation: current.promotion.generation,
})
},
context.session,
),
),
http.post(
api("/repositories/:repositoryId/decision-model/builds/:buildId/evaluation"),
({ request, params }) =>
respondMock(
request,
(_scenario, assertCurrentSession) => {
requireAdministrator()
const current = state(recordIdentifier(params.repositoryId))
const build = current.overview.evaluationBuilds.find(
(entry) => entry.id === params.buildId,
)
if (!build) throw new MockStoreError(404, "Published Build not found.")
if (!current.promotion) {
throw new MockStoreError(
409,
"No decision candidate is promoted for this repository.",
)
}
assertCurrentSession()
const evaluation = {
id: crypto.randomUUID(),
build: {
id: build.id,
number: build.number,
prototypeId: build.prototype.id,
},
model: {
id: current.promotion.candidateId,
runtimeCandidateId: current.promotion.runtimeCandidateId,
generation: current.promotion.generation,
},
questionDigest: `sha256:${"4".repeat(64)}`,
screenshotSetDigest: `sha256:${"5".repeat(64)}`,
responseDigest: `sha256:${"6".repeat(64)}`,
findingsAvailable: true,
findings: [
{
learning: {
id: "mem_1d9f0c2b7a4e5f6a8b9c0d1e",
text: "Use one primary button per form.",
},
route: "/settings/profile",
views: [
{ viewport: "desktop", theme: "dark" },
{ viewport: "narrow", theme: "light" },
],
followsProbability: 0.82,
breaksProbability: 0.18,
explanation:
"The promoted model found the Build more likely to follow this learning.",
},
],
createdAt: new Date().toISOString(),
}
current.overview.evaluations.unshift(evaluation)
return HttpResponse.json({ id: evaluation.id })
},
context.session,
),
),
]
}

function initialState(populated: boolean): DecisionModelMockState {
const baseline: DecisionCandidateResponse = {
id: "11111111-2222-4333-8444-555555555555",
runtimeCandidateId: "fixture-candidate-1",
backend: "fixture",
tasksetDigest,
productionEligible: true,
baselineMeanReward: "0.5",
Expand All @@ -322,6 +386,7 @@ function initialState(populated: boolean): DecisionModelMockState {
const candidate: DecisionCandidateResponse = {
id: "22222222-3333-4444-8555-666666666666",
runtimeCandidateId: "fixture-candidate-2",
backend: "fixture",
tasksetDigest,
productionEligible: true,
baselineMeanReward: "0.55",
Expand Down Expand Up @@ -418,6 +483,19 @@ function initialState(populated: boolean): DecisionModelMockState {
taskset: populated ? tasksetDigest : null,
},
},
evaluationBuilds: populated
? [
{
id: "88888888-8888-4888-8888-888888888888",
number: 3,
prototype: {
id: "1",
name: "Settings",
branchName: "settings",
},
},
]
: [],
evaluations: populated
? [
{
Expand Down
Loading
Loading