From b41171b11bbf61c27b922b9c39d15fa53cf1eaee Mon Sep 17 00:00:00 2001 From: Obvious Date: Thu, 17 Sep 2026 18:39:46 +0000 Subject: [PATCH] =?UTF-8?q?feat(evaluation):=20pre-registered=20parent-hom?= =?UTF-8?q?e=20rubric=20pack=20=E2=80=94=20fixtures,=20journeys,=20harness?= =?UTF-8?q?=20protocols?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent evaluator pack for the parent-home candidate arena, authored and frozen before reading any candidate output (rubric art_uWsp80Fg v1.0, pre-registered). Fixtures (evaluation/fixtures/home/, data-only): - utterances.json — four captures on the Nov 1 2026 DST fall-back day: voice u1, text-parity twin u2, interrupted-and-resumed u1b, dad's u3 with the three-heads fiction; correction cases (valid patch u4, cross-actor rejection, bogus-target statement); Ruth's caregiver questions with a fail-closed write attempt; authorization probes. - mini-history.json — five-day published seed (Oct 28-31 EDT) with a deliberate Oct 29 gap, a private-audience entry, and a pre-seeded correction chain; Nov 1 left empty for the capture journeys to fill. - journeys.json — five journeys with per-step expected outcomes. Infrastructure (evaluation/src/home/): - timestamps.ts — single source of truth for every pinned epoch, fixed-offset derived (derive-epochs.ts prints the values), with import-time assertions: zone offsets, wall-clock mappings, durations, the U3 'an hour ago' convention chain, narrative ordering, month window, and the merged-corpus cross-anchor. - adapter.ts — six-method HomeCandidateAdapter seam. - match.ts — expectation matcher reusing the merged harness helpers. - run.ts — P1-P6 protocols: raw SHA-256 fidelity, stale-attempt resolution, idempotent publish, no-write Q&A with audience checks, correction lineage, gap/window disclosure, fail-closed authorization. - validate.ts — structural fixture validation. - example/home-example-adapter.ts — worked example, 7/7 protocols. - example/home-broken-adapter.ts — negative control failing only on its encoded violations. - README.md, derive-epochs.ts — evidence boundary, commands, provenance. package.json: test chains validate + example + negative control; adds eval:home, eval:home:broken, validate:home. Verification: tsc --noEmit clean; fixture validation PASS; worked example 7/7; negative control fails as encoded; merged corpus harness unchanged and green. --- evaluation/fixtures/home/journeys.json | 169 +++++++ evaluation/fixtures/home/mini-history.json | 89 ++++ evaluation/fixtures/home/utterances.json | 142 ++++++ evaluation/package.json | 7 +- evaluation/src/home/README.md | 72 +++ evaluation/src/home/adapter.ts | 208 ++++++++ evaluation/src/home/derive-epochs.ts | 65 +++ .../src/home/example/home-broken-adapter.ts | 230 +++++++++ .../src/home/example/home-example-adapter.ts | 472 ++++++++++++++++++ evaluation/src/home/match.ts | 184 +++++++ evaluation/src/home/run.ts | 386 ++++++++++++++ evaluation/src/home/timestamps.ts | 139 ++++++ evaluation/src/home/validate.ts | 253 ++++++++++ 13 files changed, 2414 insertions(+), 2 deletions(-) create mode 100644 evaluation/fixtures/home/journeys.json create mode 100644 evaluation/fixtures/home/mini-history.json create mode 100644 evaluation/fixtures/home/utterances.json create mode 100644 evaluation/src/home/README.md create mode 100644 evaluation/src/home/adapter.ts create mode 100644 evaluation/src/home/derive-epochs.ts create mode 100644 evaluation/src/home/example/home-broken-adapter.ts create mode 100644 evaluation/src/home/example/home-example-adapter.ts create mode 100644 evaluation/src/home/match.ts create mode 100644 evaluation/src/home/run.ts create mode 100644 evaluation/src/home/timestamps.ts create mode 100644 evaluation/src/home/validate.ts diff --git a/evaluation/fixtures/home/journeys.json b/evaluation/fixtures/home/journeys.json new file mode 100644 index 0000000..d9e78e1 --- /dev/null +++ b/evaluation/fixtures/home/journeys.json @@ -0,0 +1,169 @@ +{ + "$schema": "home-journeys/journeys v1", + "rubric": "art_uWsp80Fg v1.0", + "note": "The five executable journeys as data. Machine-checkable cores map to protocols P1-P6 (run.ts); full journeys run as scripted E2E/clickthrough with evidence labels.", + "journeys": [ + { + "id": "J1", + "title": "Speak a mixed-topic update", + "actorId": "actor-mom", + "childId": "child-lena", + "preconditions": ["mom signed in", "fixture month loaded", "today empty"], + "steps": [ + { + "step": 1, + "action": "Speak u1 in one utterance (meal with quantity, mood episode, nap with explicit span, unpinned potty x2).", + "expect": "Raw preserved byte-for-byte first; pinned instants exact; unpinned expressions gain no false precision; figures of speech yield no events.", + "protocols": ["P1"], + "gates": ["G3", "G5"], + "dims": ["S1", "S6"] + }, + { + "step": 2, + "action": "Interrupt: start u1b, stop mid-way, background ~10s, resume under the same captureId.", + "expect": "Capture resumes or cleanly re-keys; no duplicate or partial events; stale older attempt cannot overwrite the newer one.", + "protocols": ["P5"], + "gates": ["G2"], + "dims": ["S1", "S6"] + }, + { + "step": 3, + "action": "Observe save states pending -> saved.", + "expect": "Receipt visible: what wrote, what was proposed.", + "protocols": [], + "gates": [], + "dims": ["S6"] + } + ], + "evidence": ["transcript SHA-256", "event cards", "publish receipt", "interruption log"] + }, + { + "id": "J2", + "title": "Inspect & correct interpreted events", + "actorId": "actor-dad", + "childId": "child-lena", + "preconditions": ["J1 proposed events in review state", "dad signed in same household"], + "steps": [ + { + "step": 1, + "action": "Open review of J1's capture.", + "expect": "Every proposed event traceable to its raw transcript fragment.", + "protocols": [], + "gates": ["G3"], + "dims": ["S2"] + }, + { + "step": 2, + "action": "Correct via typed NL (u4 -> u3 sleep event) and via direct card edit.", + "expect": "Append-only revision; original retrievable; reviser attributed.", + "protocols": ["P4"], + "gates": ["G6"], + "dims": ["S2", "S6"] + }, + { + "step": 3, + "action": "Reject one event (duplicate potty from u1).", + "expect": "Removed via revision, not hard delete.", + "protocols": ["P4"], + "gates": ["G6"], + "dims": ["S2"] + }, + { + "step": 4, + "action": "Publish, then publish again (double-tap / network retry).", + "expect": "Exactly one write; identical receipt; idempotent replay visible.", + "protocols": ["P2"], + "gates": ["G2"], + "dims": ["S2", "S6"] + }, + { + "step": 5, + "action": "Dad (non-author) corrects mom-authored events.", + "expect": "Succeeds with attribution (mom authored, dad revised).", + "protocols": ["P4"], + "gates": [], + "dims": ["S2"] + } + ], + "evidence": ["lineage view", "revision chain", "receipts", "duplicate-submit log"] + }, + { + "id": "J3", + "title": "Catch up before taking over care", + "actorId": "actor-ruth", + "childId": "child-lena", + "preconditions": ["mini-history seed loaded (slot-03 corpus when merged)", "ruth holds explicit caregiver grant", "one private-audience entry exists"], + "steps": [ + { + "step": 1, + "action": "Open catch-up: current state without hunting.", + "expect": "Last meal, last nap (ongoing vs complete explicit), open needs; 'what should I know?' returns source-linked summary.", + "protocols": ["P3"], + "gates": ["G4"], + "dims": ["S3"] + }, + { + "step": 2, + "action": "Ask q1 (food today) and q2 (last nap).", + "expect": "Answers cite sources; private-audience entry excluded or marked per audience rules.", + "protocols": ["P3"], + "gates": ["G1"], + "dims": ["S3", "S4"] + }, + { + "step": 3, + "action": "Attempt a write from the question surface (q3: 'Log that she had a snack').", + "expect": "Surface refuses — capture is the only write path; state byte-identical before/after.", + "protocols": ["P3"], + "gates": ["G4"], + "dims": ["S4"] + }, + { + "step": 4, + "action": "Month view over the seed window.", + "expect": "Missed days render as gaps, never zeros; correction chains visible with originals; nothing beyond the window.", + "protocols": ["P3", "P4"], + "gates": ["G7"], + "dims": ["S3"] + } + ], + "evidence": ["answer citations", "write-refusal", "gap rendering", "window bounds"] + }, + { + "id": "J4", + "title": "Text alternative parity", + "actorId": "actor-mom", + "childId": "child-lena", + "preconditions": ["mom signed in", "quiet/text mode available"], + "steps": [ + { + "step": 1, + "action": "Submit u2 — typed twin of u1 — through the text path.", + "expect": "Schema-equivalent event set vs u1 expectations; raw text fidelity under the same byte rules; all state-table states reachable in quiet/text mode.", + "protocols": ["P1"], + "gates": ["G3", "G5"], + "dims": ["S5", "S1"] + } + ], + "evidence": ["event diff vs voice path", "state-table walkthrough log"] + }, + { + "id": "J5", + "title": "Authorization boundary", + "actorId": "actor-marcus", + "childId": "child-lena", + "preconditions": ["marcus is not a household member", "one child outside actor grants (child-theo)"], + "steps": [ + { + "step": 1, + "action": "Marcus attempts J1/J3 flows (capture, history, question, correction) and ruth probes child-theo.", + "expect": "Fail-closed refusals on every surface; error messages disclose nothing.", + "protocols": ["P6"], + "gates": ["G1"], + "dims": ["S4"] + } + ], + "evidence": ["refusal matrix", "disclosure audit (all refusals disclose nothing)"] + } + ] +} diff --git a/evaluation/fixtures/home/mini-history.json b/evaluation/fixtures/home/mini-history.json new file mode 100644 index 0000000..7c82e37 --- /dev/null +++ b/evaluation/fixtures/home/mini-history.json @@ -0,0 +1,89 @@ +{ + "$schema": "home-journeys/mini-history v1", + "rubric": "art_uWsp80Fg v1.0", + "note": "5-day synthetic stand-in for the slot-03 30-day family-history corpus (labeled JOURNEY-LOCAL SEED, not the full month corpus). Epochs pinned by timestamps.ts. Nov 1 is deliberately EMPTY — the capture journeys fill it.", + "window": { "from": 1793160000000, "to": 1793505599999 }, + "householdId": "hh-primary", + "childId": "child-lena", + "entries": [ + { + "entryId": "entry-o28-a", + "captureId": "cap-seed-o28a", + "transcript": "Banana toast for breakfast, good mood all morning.", + "authorId": "actor-mom", + "createdAt": 1793188800000, + "status": "published", + "audience": "household", + "events": [ + { "eventId": "ev-o28-meal", "category": "meal", "occurredAt": 1793188800000, "quantity": { "value": 1, "unit": "toast" }, "note": "banana toast" }, + { "eventId": "ev-o28-nap", "category": "sleep", "occurredAt": 1793205000000, "quantity": { "value": 60, "unit": "minutes" }, "note": "nap 12:30-13:30" } + ] + }, + { + "entryId": "entry-o28-b", + "captureId": "cap-seed-o28b", + "transcript": "She used the potty after quiet time.", + "authorId": "actor-mom", + "createdAt": 1793215800000, + "status": "published", + "audience": "household", + "events": [ + { "eventId": "ev-o28-potty", "category": "potty", "occurredAt": 1793215800000, "note": "after quiet time" } + ] + }, + { + "entryId": "entry-o30-a", + "captureId": "cap-seed-o30a", + "transcript": "Waffle for breakfast, then a solid nap, happy at pickup.", + "authorId": "actor-dad", + "createdAt": 1793363400000, + "status": "published", + "audience": "household", + "events": [ + { "eventId": "ev-o30-meal", "category": "meal", "occurredAt": 1793363400000, "quantity": { "value": 2, "unit": "waffles" }, "note": "waffle breakfast (corrected)" }, + { "eventId": "ev-o30-nap", "category": "sleep", "occurredAt": 1793378700000, "quantity": { "value": 60, "unit": "minutes" }, "note": "nap 12:45-13:45" }, + { "eventId": "ev-o30-mood", "category": "mood", "occurredAt": 1793386800000, "note": "happy at pickup" } + ] + }, + { + "entryId": "entry-o31-a", + "captureId": "cap-seed-o31a", + "transcript": "Oatmeal at 8, potty twice in the morning, and she pedaled the balance bike all by herself for the first time.", + "authorId": "actor-mom", + "createdAt": 1793448000000, + "status": "published", + "audience": "household", + "events": [ + { "eventId": "ev-o31-meal", "category": "meal", "occurredAt": 1793448000000, "quantity": { "value": 1, "unit": "oatmeal" }, "note": "oatmeal" }, + { "eventId": "ev-o31-potty-1", "category": "potty", "occurredAt": 1793455200000, "note": "morning potty" }, + { "eventId": "ev-o31-potty-2", "category": "potty", "occurredAt": 1793457600000, "note": "morning potty" }, + { "eventId": "ev-o31-milestone", "category": "milestone", "occurredAt": 1793476800000, "note": "first solo balance-bike pedal" } + ] + }, + { + "entryId": "entry-o31-private", + "captureId": "cap-seed-o31p", + "transcript": "Private note about co-parent scheduling — author only.", + "authorId": "actor-mom", + "createdAt": 1793494800000, + "status": "published", + "audience": "private", + "events": [ + { "eventId": "ev-o31-private-note", "category": "mood", "occurredAt": 1793494800000, "note": "private scheduling note (author-only)" } + ] + } + ], + "gaps": [{ "day": "2026-10-29", "reason": "no-data" }], + "corrections": [ + { + "revisionId": "rev-o30-meal-1", + "entryId": "entry-o30-a", + "eventId": "ev-o30-meal", + "supersedes": { "quantity": { "value": 1, "unit": "waffle" } }, + "patch": { "quantity": { "value": 2, "unit": "waffles" } }, + "byActorId": "actor-dad", + "createdAt": 1793380000000, + "note": "Pre-seeded correction chain: original quantity (1 waffle) remains retrievable via lineage." + } + ] +} diff --git a/evaluation/fixtures/home/utterances.json b/evaluation/fixtures/home/utterances.json new file mode 100644 index 0000000..2ea9d07 --- /dev/null +++ b/evaluation/fixtures/home/utterances.json @@ -0,0 +1,142 @@ +{ + "$schema": "home-journeys/utterances v1", + "rubric": "art_uWsp80Fg v1.0", + "note": "Data only — inputs plus expected outcomes, independent of candidate code. Epochs are pinned by evaluation/src/home/timestamps.ts; validate.ts re-asserts equality.", + "actors": { + "mom": { "id": "actor-mom", "role": "parent", "householdId": "hh-primary", "grantedChildIds": ["child-lena"] }, + "dad": { "id": "actor-dad", "role": "parent", "householdId": "hh-primary", "grantedChildIds": ["child-lena"] }, + "ruth": { "id": "actor-ruth", "role": "caregiver", "householdId": "hh-primary", "grantedChildIds": ["child-lena"], "viaGrant": "grant-ruth-2026-10" }, + "marcus": { "id": "actor-marcus", "role": "none", "householdId": null, "grantedChildIds": [] } + }, + "children": [{ "id": "child-lena", "name": "Lena (synthetic)", "householdId": "hh-primary" }], + "captures": { + "u1": { + "captureId": "cap-u1-20261101", + "mode": "voice-transcript", + "actorId": "actor-mom", + "childId": "child-lena", + "capturedAt": 1793581200000, + "timezone": "America/New_York", + "raw": "Lena had half a banana at 8, then a total meltdown at drop-off, went down for her nap at 12:30 and slept till 2, and she used the potty twice this afternoon.", + "expected": { + "events": [ + { + "category": "meal", + "occurrence": "exactly-one", + "occurredAt": 1793538000000, + "toleranceMs": 0, + "quantity": { "value": 0.5, "unit": "banana" }, + "noteContains": ["banana"] + }, + { + "category": "mood", + "occurrence": "at-least-one", + "occurredAtWindow": { "from": 1793538000000, "to": 1793554200000 }, + "occurredAtAbsentAllowed": true, + "noteContains": ["meltdown"] + }, + { + "category": "sleep", + "occurrence": "exactly-one", + "occurredAt": 1793554200000, + "toleranceMs": 0, + "quantity": { "value": 90, "unit": "minutes" } + }, + { + "category": "potty", + "occurrence": "at-least-one", + "occurredAtWindow": { "from": 1793559600000, "to": 1793574000000 }, + "occurredAtAbsentAllowed": false + } + ], + "extrasAllow": [{ "category": "potty", "quantity": { "value": 2, "unit": "times" } }], + "pottyCoverage": "two-events-or-quantity-two", + "forbiddenNoteSubstrings": ["three heads"] + } + }, + "u2": { + "captureId": "cap-u2-20261101", + "mode": "text", + "actorId": "actor-mom", + "childId": "child-lena", + "capturedAt": 1793581200000, + "timezone": "America/New_York", + "raw": "Lena had half a banana at 8. Total meltdown at drop-off. Nap from 12:30 to 2. Used the potty twice this afternoon.", + "expectedNote": "Text-parity twin of u1: identical expected block (shared by reference via parityWith).", + "parityWith": "u1" + }, + "u1b": { + "captureId": "cap-u1b-20261101", + "mode": "voice-transcript", + "actorId": "actor-mom", + "childId": "child-lena", + "capturedAt": 1793582100000, + "timezone": "America/New_York", + "stale": { + "fragmentRaw": "and also she wore her", + "fragmentAttempt": 1, + "resumedRaw": "and also she wore her dinosaur pajamas all day", + "resumedAttempt": 2, + "expectedWinnerEvents": 0, + "note": "Older attempt must lose to the newer attempt regardless of resolution order (P5). Winner has zero events — extraction never blocks capture." + } + }, + "u3": { + "captureId": "cap-u3-20261101", + "mode": "voice-transcript", + "actorId": "actor-dad", + "childId": "child-lena", + "capturedAt": 1793553300000, + "timezone": "America/New_York", + "raw": "She ate like a horse at lunch — okay, two full servings of pasta — then told me the babysitter has three heads. She woke up from her nap about an hour ago; that nap was 45 minutes.", + "expected": { + "events": [ + { + "category": "meal", + "occurrence": "exactly-one", + "occurredAtWindow": { "from": 1793547000000, "to": 1793553300000 }, + "occurredAtAbsentAllowed": true, + "quantity": { "value": 2, "unit": "servings" }, + "noteContains": ["pasta"] + }, + { + "category": "sleep", + "occurrence": "exactly-one", + "occurredAtWindow": { "from": 1793547000000, "to": 1793549700000 }, + "occurredAtAbsentAllowed": false, + "quantity": { "value": 45, "unit": "minutes" } + } + ], + "extrasAllow": [], + "exactly": true, + "forbiddenNoteSubstrings": ["three heads"] + } + } + }, + "corrections": { + "u4": { + "viaStatement": "actually the nap was closer to an hour, not 45 minutes", + "targetCaptureId": "cap-u3-20261101", + "targetCategory": "sleep", + "expectPatch": { "quantity": { "value": 60, "unit": "minutes" } }, + "byActorId": "actor-dad", + "note": "View selection supplies targetEventLocalId at runtime; NL statement must resolve to that same record or return UnresolvedReference — never guess." + }, + "noTargetStatement": "actually the waffle was three waffles", + "rejectFromCaptureId": "cap-u1-20261101", + "rejectTargetCategory": "potty", + "rejectByActorId": "actor-dad" + }, + "questions": { + "q1": { "actorId": "actor-ruth", "childId": "child-lena", "text": "What did she eat today?" }, + "q2": { "actorId": "actor-ruth", "childId": "child-lena", "text": "When was her last nap?" }, + "q3WriteAttempt": { "actorId": "actor-ruth", "childId": "child-lena", "text": "Log that she had a snack" }, + "privateEntryId": "entry-o31-private", + "askAt": 1793628000000 + }, + "authorization": { + "nonMemberActorId": "actor-marcus", + "outOfScopeChildId": "child-theo", + "note": "child-theo belongs to another household; no history exists for it. Any access must fail closed with an empty disclosure." + } +} diff --git a/evaluation/package.json b/evaluation/package.json index f6a80ed..1553a81 100644 --- a/evaluation/package.json +++ b/evaluation/package.json @@ -6,9 +6,12 @@ "description": "Shared acceptance/evaluation corpus and candidate-agnostic harness for the Shared Child Journal arena cross-review.", "scripts": { "eval": "bun src/run.ts", - "test": "bun src/run.ts", + "test": "bun src/run.ts && bun src/home/validate.ts && bun src/home/run.ts && bun src/home/run.ts --adapter=./src/home/example/home-broken-adapter.ts --expect-failure", "typecheck": "tsc --noEmit", - "eval:broken": "bun src/run.ts --adapter=./src/example/broken-adapter.ts --expect-failure" + "eval:broken": "bun src/run.ts --adapter=./src/example/broken-adapter.ts --expect-failure", + "eval:home": "bun src/home/run.ts", + "eval:home:broken": "bun src/home/run.ts --adapter=./src/home/example/home-broken-adapter.ts --expect-failure", + "validate:home": "bun src/home/validate.ts" }, "devDependencies": { "@types/node": "^20.14.0", diff --git a/evaluation/src/home/README.md b/evaluation/src/home/README.md new file mode 100644 index 0000000..0d64e37 --- /dev/null +++ b/evaluation/src/home/README.md @@ -0,0 +1,72 @@ +# Parent-Home Journey Pack (`src/home/` + `fixtures/home/`) + +Executable companion to the pre-registered rubric **art_uWsp80Fg v1.0** +("Parent Home Arena — Independent Evaluation Rubric & User Journeys"). It +scores the three parent-home candidates (Home-A, Home-B, Home-C) against the +same five journeys (J1–J5) through machine-checkable protocols (P1–P6). + +Evidence label for every run in this pack: **DETERMINISTIC TEST DOUBLE** — +no live LLM, no device, no cloud. This pack is separate from (and additional +to) the extraction-corpus harness in `src/` — different candidates, shared +canonical contracts. + +## Layout + +``` +fixtures/home/utterances.json capture inputs + expected outcomes (data only) +fixtures/home/mini-history.json 5-day synthetic seed (stand-in for slot-03 corpus) +fixtures/home/journeys.json J1–J5 as data (steps, gates, dims, evidence) +src/home/timestamps.ts pinned epochs — single source of truth, self-asserting +src/home/adapter.ts HomeCandidateAdapter seam v1 (six methods) +src/home/match.ts expected-set matcher (windows, occurrences, G5 guard) +src/home/validate.ts fixture structure validator +src/home/run.ts protocol runner P1–P6 (CLI) +src/home/example/…-adapter.ts worked example (all gates pass) + negative control +``` + +## Commands + +```bash +bun src/home/validate.ts # fixtures must validate +bun src/home/run.ts # protocols vs example adapter (expect 6/6) +bun src/home/run.ts --adapter=./src/home/example/home-broken-adapter.ts --expect-failure + # negative control must be caught +``` + +All three run in CI via `pnpm --dir evaluation test` (chained in the `test` +script). The pack reuses the merged harness match module (`../match.ts`) for +canonical JSON, SHA-256 byte fidelity, and note normalization. + +## Protocols + +| Protocol | Proves | Gates | +|---|---|---| +| P1/voice, P1/text | U1 (voice) vs U2 (text) produce schema-equivalent event sets; raw stored byte-for-byte | G3, G5 | +| P5/stale | interrupted capture: older attempt resolving late cannot overwrite the newer; zero-event captures still publish | G2 | +| P2/idempotent-publish | double publish → one write, identical receipt | G2 | +| P3/no-write+audience | questions leave history byte-identical; private-audience event hidden from non-authors, visible to the author; missed day disclosed as gap | G4, G1, G7 | +| P4/lineage | patch/reject are append-only revisions with retrievable originals; unresolved references come back as questions, never guesses | G6 | +| P6/authorization | non-member and out-of-scope refusals, all with empty disclosure; positive control for authorized reads | G1 | + +## Pinned time context + +Fixtures land on **2026-11-01, the US DST fall-back day** (America/New_York): +02:00 EDT becomes 01:00 EST, so all Nov 1 instants are EST (UTC-5) while the +seed days Oct 28–31 are EDT (UTC-4). `timestamps.ts` asserts every epoch — +including the cross-anchor from the merged corpus (Nov 1 18:00 EST = +1793574000000) — at import time; a wrong value aborts the run. The capture +day is deliberately left empty in the seed so the journeys fill it. + +## Using the pack for a candidate + +Implement `HomeCandidateAdapter` (six methods, seam v1) around the +candidate's real stores and run: + +```bash +bun src/home/run.ts --adapter=./path/to/candidate-adapter.ts +``` + +Gate verdicts feed the rubric's hard-gate table (PASS/FAIL/ND per §4 — a +deliverable that cannot host the seam runs its journeys as scripted +clickthrough evidence instead). The pack proves mechanics; the rubric scores +the experience around them. diff --git a/evaluation/src/home/adapter.ts b/evaluation/src/home/adapter.ts new file mode 100644 index 0000000..aa8c115 --- /dev/null +++ b/evaluation/src/home/adapter.ts @@ -0,0 +1,208 @@ +/** + * Home-candidate adapter seam v1 — the ONLY integration surface between the + * parent-home journey pack and a home candidate. + * + * Mirrors the merged harness conventions (evaluation/src/adapter.ts): + * - factory contract: adapter modules export a named `createAdapter()` + * returning an object with a non-empty `name`; + * - wire rules: every record carries a literal `_tag`; instants are + * Unix-ms numbers; optional fields are ABSENT, never null; + * - refusals are typed values (`Refused`) with `disclosure: ''` — the + * type itself forbids leaking a reason string to an unauthorized actor. + * + * A candidate implements this seam for the machine-checkable protocols P1-P6. + * Where a candidate's deliverable form cannot host the seam (e.g., a + * non-persistent prototype), its journeys run as scripted evidence and the + * pack runs only where hostable — recorded per candidate in the rubric (§9). + */ + +export type CaptureMode = 'voice-transcript' | 'text' + +export type EventCategory = 'potty' | 'meal' | 'sleep' | 'mood' | 'milestone' | 'school' + +export interface Quantity { + readonly value: number + readonly unit: string +} + +export interface CaptureInput { + readonly captureId: string + /** Monotonic attempt for this captureId — only the latest attempt may win (P5). */ + readonly attempt: number + readonly mode: CaptureMode + /** The utterance, byte-for-byte. Adapters must not trim or normalize. */ + readonly raw: string + readonly actorId: string + readonly childId: string + readonly capturedAt: number + readonly timezone: string +} + +export interface ProposedEvent { + readonly _tag: 'ProposedEvent' + readonly localId: string + readonly category: EventCategory + /** Absolute instant, Unix ms. ABSENT when unresolved — never null, never fabricated (G5). */ + readonly occurredAt?: number + readonly quantity?: Quantity + readonly note?: string + readonly confidence: number + /** Traceability into the raw transcript (substring); absent when not resolvable. */ + readonly sourceSpan?: string +} + +export type SubmitResult = + | { + readonly _tag: 'Proposed' + readonly captureId: string + readonly attempt: number + readonly proposals: readonly ProposedEvent[] + /** SHA-256 of the raw transcript as stored — byte-fidelity check (G3). */ + readonly rawSha256: string + } + | { + readonly _tag: 'ClarificationNeeded' + readonly captureId: string + readonly attempt: number + readonly question: string + readonly rawSha256: string + } + | { readonly _tag: 'Superseded'; readonly captureId: string; readonly attempt: number; readonly winnerAttempt: number } + | { readonly _tag: 'Refused'; readonly reason: 'unauthorized' | 'invalid-actor' | 'out-of-scope-child'; readonly disclosure: '' } + +export type CorrectionAction = + | { readonly _tag: 'Patch'; readonly fields: Partial> } + | { readonly _tag: 'Reject' } + +export interface CorrectionInput { + readonly captureId: string + readonly eventLocalId: string + readonly action: CorrectionAction + readonly authorId: string + /** Natural-language statement when correcting via NL; must resolve to eventLocalId or return UnresolvedReference. */ + readonly viaStatement?: string +} + +export type CorrectionResult = + | { + readonly _tag: 'Revised' + readonly revisionId: string + readonly supersedes: string + readonly originalPreserved: true + readonly byActorId: string + } + | { readonly _tag: 'UnresolvedReference'; readonly question: string } + | { readonly _tag: 'Refused'; readonly reason: 'unauthorized' | 'invalid-actor'; readonly disclosure: '' } + +export interface PublishReceipt { + readonly entryId: string + readonly proposalHash: string + readonly wroteAt: number + readonly authoredBy: string +} + +export type PublishResult = + | { readonly _tag: 'Published'; readonly receipt: PublishReceipt; readonly idempotentHit: false } + | { readonly _tag: 'AlreadyPublished'; readonly receipt: PublishReceipt; readonly idempotentHit: true } + | { readonly _tag: 'Refused'; readonly reason: 'unauthorized' | 'nothing-proposed'; readonly disclosure: '' } + +export interface HistoryQuery { + readonly actorId: string + readonly childId: string + readonly from: number + readonly to: number +} + +export interface HistoryGap { + readonly _tag: 'HistoryGap' + /** Household-local day (YYYY-MM-DD) with no data. */ + readonly day: string + readonly reason: 'no-data' +} + +export interface HistoryEvent { + readonly _tag: 'Event' + readonly eventId: string + readonly entryId: string + readonly category: EventCategory + readonly occurredAt: number + readonly quantity?: Quantity + readonly note?: string + readonly authorId: string + /** True when a later revision supersedes this record; lineage holds the original (G6). */ + readonly superseded?: true +} + +export type HistoryResult = + | { + readonly _tag: 'History' + readonly events: readonly HistoryEvent[] + readonly gaps: readonly HistoryGap[] + /** The query window was honored exactly — nothing outside [from, to]. */ + readonly windowRespected: true + } + | { readonly _tag: 'Refused'; readonly reason: 'unauthorized' | 'out-of-scope-child'; readonly disclosure: '' } + +export interface AskInput { + readonly actorId: string + readonly childId: string + readonly question: string + readonly askAt: number +} + +export interface AnswerCitation { + readonly entryId: string + readonly eventId?: string +} + +export type AskResult = + | { + readonly _tag: 'Answer' + readonly answer: string + readonly citations: readonly AnswerCitation[] + /** Literal — the answer surface declares zero writes (G4, proven by P3 state equality). */ + readonly wrote: false + } + | { readonly _tag: 'Refused'; readonly reason: 'unauthorized' | 'out-of-scope-child'; readonly disclosure: '' } + +export interface LineageQuery { + readonly captureId: string + readonly actorId: string +} + +export interface RevisionRecord { + readonly revisionId: string + readonly eventLocalId: string + readonly action: CorrectionAction + readonly supersedes: string + readonly byActorId: string + /** The full superseded state — originals always retrievable (G6). */ + readonly original: ProposedEvent +} + +export type LineageResult = + | { + readonly _tag: 'Lineage' + readonly captureId: string + readonly revisions: readonly RevisionRecord[] + /** Verbatim raw transcript as stored (G3 evidence surface). */ + readonly originalRaw: string + } + | { readonly _tag: 'Refused'; readonly reason: 'unauthorized'; readonly disclosure: '' } + +export interface HomeCandidateAdapter { + /** Non-empty; identifies the candidate in run output. */ + readonly name: string + /** Raw-first capture: persist the utterance byte-for-byte, then propose typed events. */ + submitCapture(input: CaptureInput): Promise + /** Append-only correction: patch or reject; the original stays retrievable. */ + applyCorrection(input: CorrectionInput): Promise + /** Idempotent publish keyed by captureId — one write, one receipt, forever. */ + publishCapture(captureId: string, actorId: string): Promise + /** Windowed, audience-respecting, gap-disclosing history read. */ + readHistory(query: HistoryQuery): Promise + /** Read-only question surface: answers cite sources and never write. */ + askQuestion(input: AskInput): Promise + /** Revision lineage for one capture: every correction plus the originals. */ + readLineage(query: LineageQuery): Promise +} diff --git a/evaluation/src/home/derive-epochs.ts b/evaluation/src/home/derive-epochs.ts new file mode 100644 index 0000000..6317e41 --- /dev/null +++ b/evaluation/src/home/derive-epochs.ts @@ -0,0 +1,65 @@ +/** + * Epoch derivation for the Nov 1 2026 (DST fall-back) fixtures. + * Rule: local wall time -> UTC via FIXED offsets (Oct 28-31 = EDT = UTC-4, Nov 1 = EST = UTC-5). + * Prints every constant; timestamps.ts must match this output exactly. + */ +const EST = 5 * 3600_000 // UTC-5 -> add to local wall +const EDT = 4 * 3600_000 // UTC-4 -> add to local wall + +// Nov 1 2026, all EST +const nov1 = (h: number, m = 0): number => Date.UTC(2026, 10, 1, h, m) + EST +// Oct days 2026, all EDT +const oct = (day: number, h: number, m = 0): number => Date.UTC(2026, 9, day, h, m) + EDT + +const show = (label: string, v: number | [number, number]): void => { + if (Array.isArray(v)) { + console.log(`${label} = [${v[0]}, ${v[1]}] // local ${new Date(v[0]).toISOString()} .. ${new Date(v[1]).toISOString()}`) + } else { + console.log(`${label} = ${v} // ${new Date(v).toISOString()}`) + } +} + +console.log('--- windows ---') +show('MINI_HISTORY_WINDOW.from', oct(28, 0, 0)) +show('MINI_HISTORY_WINDOW.to', oct(31, 23, 59) + 59_999 - 60_000) // 23:59:59.999 +show('HISTORY_WINDOW_NOV1_END', nov1(23, 59) + 59_999 - 60_000) +console.log('--- capture day (Nov 1, EST) ---') +show('NOV_1_MIDNIGHT_EDT (Nov 1 00:00 local)', nov1(0, 0)) +show('MOM_CAPTURED_AT (20:00)', nov1(20, 0)) +show('U1B_CAPTURED_AT (20:15)', nov1(20, 15)) +show('DAD_CAPTURED_AT (11:15)', nov1(11, 15)) +show('RUTH_ASK_AT (Nov 2 09:00 EST)', Date.UTC(2026, 10, 2, 9, 0) + EST) +show('U1_MEAL_AT_8 (08:00)', nov1(8, 0)) +show('U1_NAP_START (12:30)', nov1(12, 30)) +show('U1_NAP_END (14:00)', nov1(14, 0)) +show('U1_MOOD_WINDOW', [nov1(8, 0), nov1(12, 30)]) +show('U1_POTTY_WINDOW', [nov1(12, 0), nov1(18, 0)]) +show('U3_NAP_START (09:30 = end 10:15 minus 45m)', nov1(9, 30)) +show('U3_NAP_END (capturedAt - 1h = 10:15)', nov1(11, 15) - 3600_000) +show('U3_MEAL_WINDOW (snack around 10)', [nov1(9, 30), nov1(10, 30)]) +console.log('--- cross-anchor ---') +show('NOV1_18_00_EST', nov1(18, 0)) +console.log('--- seed events (Oct, EDT) ---') +show('ev-o28-meal 08:00', oct(28, 8, 0)) +show('ev-o28-sleep 19:15', oct(28, 19, 15)) +show('ev-o29-school 08:30', oct(29, 8, 30)) +show('ev-o29-pm-nap 13:00', oct(29, 13, 0)) +show('ev-o29-dinner 17:45', oct(29, 17, 45)) +show('ev-o29-bath 19:00', oct(29, 19, 0)) +show('ev-o30-meal 12:30', oct(30, 12, 30)) +show('ev-o30-snack 15:30', oct(30, 15, 30)) +show('ev-o30-bed 19:30', oct(30, 19, 30)) +show('ev-o31-meal 08:15', oct(31, 8, 15)) +show('ev-o31-park-mood 10:30', oct(31, 10, 30)) +show('ev-o31-lunch 12:15', oct(31, 12, 15)) +show('entry-o28 createdAt (07:55)', oct(28, 7, 55)) +show('entry-o29 createdAt (18:30)', oct(29, 18, 30)) +show('entry-o30 createdAt (12:40)', oct(30, 12, 40)) +show('entry-o30b createdAt (19:35)', oct(30, 19, 35)) +show('entry-o31 createdAt (20:00)', oct(31, 20, 0)) +console.log('--- sanity: convention chain ---') +console.log(`DAD - 1h == U3_NAP_END: ${nov1(11, 15) - 3600_000 === nov1(10, 15)}`) +console.log(`U3_NAP_END - 45m == U3_NAP_START: ${nov1(10, 15) - 45 * 60_000 === nov1(9, 30)}`) +console.log(`Nov2 09:00 > MOM publish (20:05): ${Date.UTC(2026, 10, 2, 9, 0) + EST > nov1(20, 0) + 5 * 60_000}`) +console.log(`Nov1 23:59:59.999 EST > u1b publish (20:20): ${nov1(23, 59) - 60_000 + 59_999 > nov1(20, 15) + 5 * 60_000}`) +console.log(`MINI.to is last ms of Oct 31 local: ${oct(31, 23, 59) + 59_999 - 60_000 === Date.UTC(2026, 11, 1, 3, 59, 59, 999)}`) diff --git a/evaluation/src/home/example/home-broken-adapter.ts b/evaluation/src/home/example/home-broken-adapter.ts new file mode 100644 index 0000000..93c68f3 --- /dev/null +++ b/evaluation/src/home/example/home-broken-adapter.ts @@ -0,0 +1,230 @@ +/** + * NEGATIVE CONTROL — deliberately violates the parent-home gates. Run with: + * bun src/home/run.ts --adapter=./src/home/example/home-broken-adapter.ts --expect-failure + * The run MUST report failures (exit 0 with --expect-failure). If this + * adapter passes any gate check, the pack is not strict enough. + * + * Violations baked in (one per gate): + * G3 normalizes the raw to lowercase before hashing/storing + * G2 every publish writes a fresh entry with a new receipt (duplicate writes) + * G4 "log that ..." from the question surface silently appends an event + * G5 "three heads" fiction becomes a milestone event with quantity 3 + * G6 rejects hard-delete the proposal; lineage loses the original + * G1 non-member reads are served everything, private entry included + * P5 an older attempt resolving late CLOBBERS the newer winner + */ +import type { + AskInput, + AskResult, + CaptureInput, + CorrectionInput, + CorrectionResult, + EventCategory, + HistoryEvent, + HistoryGap, + HistoryQuery, + HistoryResult, + HomeCandidateAdapter, + LineageQuery, + LineageResult, + ProposedEvent, + PublishReceipt, + PublishResult, + Quantity, + SubmitResult, +} from '../adapter.ts' +import { sha256Hex } from '../../match.ts' +import * as T from '../timestamps.ts' + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function zonedWallToEpoch(capturedAt: number, timezone: string, hour: number, minute = 0): number { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: timezone, + hour12: false, + hourCycle: 'h23', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }).formatToParts(new Date(capturedAt)) + const get = (t: string): number => Number(parts.find((p) => p.type === t)?.value) + const y = get('year') + const mo = get('month') + const d = get('day') + const wallNow = Date.UTC(y, mo - 1, d, get('hour'), get('minute')) + return Date.UTC(y, mo - 1, d, hour, minute) + (capturedAt - wallNow) +} + +function extractBroken(input: CaptureInput): ProposedEvent[] { + const out: ProposedEvent[] = [] + const raw = input.raw // NOTE: already lowercased by the caller — G3 leak + const wall = (h: number, m = 0): number => zonedWallToEpoch(input.capturedAt, input.timezone, h, m) + let n = 0 + const push = (p: Omit): void => { + out.push({ _tag: 'ProposedEvent', localId: `bx-${String(n++)}`, ...p }) + } + const meal = /half a ([a-z]+) at (\d{1,2})/i.exec(raw) + if (meal && meal[1] !== undefined && meal[2] !== undefined) { + push({ category: 'meal', occurredAt: wall(Number(meal[2])), quantity: { value: 0.5, unit: meal[1] }, note: `${meal[1]} meal`, confidence: 0.95 }) + } + const meal2 = /two full servings of ([a-z]+)/i.exec(raw) + if (meal2 && meal2[1] !== undefined) { + push({ category: 'meal', quantity: { value: 2, unit: 'servings' }, note: `${meal2[1]} at lunch`, confidence: 0.9 }) + } + if (/meltdown/i.test(raw)) push({ category: 'mood', note: 'meltdown at drop-off', confidence: 0.7 }) + const sv = /nap at (\d{1,2}):(\d{2}) and slept till (\d{1,2})/i.exec(raw) + if (sv && sv[1] !== undefined && sv[2] !== undefined && sv[3] !== undefined) { + const start = wall(Number(sv[1]), Number(sv[2])) + const end = (() => { let e = wall(Number(sv[3])); if (e < start) e += 12 * 3_600_000; return e })() + push({ category: 'sleep', occurredAt: start, quantity: { value: Math.round((end - start) / 60000), unit: 'minutes' }, confidence: 0.9 }) + } + const st = /nap from (\d{1,2}):(\d{2}) to (\d{1,2})/i.exec(raw) + if (st && st[1] !== undefined && st[2] !== undefined && st[3] !== undefined) { + const start = wall(Number(st[1]), Number(st[2])) + const end = (() => { let e = wall(Number(st[3])); if (e < start) e += 12 * 3_600_000; return e })() + push({ category: 'sleep', occurredAt: start, quantity: { value: Math.round((end - start) / 60000), unit: 'minutes' }, confidence: 0.9 }) + } + const sr = /nap about an hour ago[\s\S]*?was (\d{1,3}) minutes/i.exec(raw) + if (sr && sr[1] !== undefined) { + const mins = Number(sr[1]) + push({ category: 'sleep', occurredAt: input.capturedAt - 3_600_000 - mins * 60000, quantity: { value: mins, unit: 'minutes' }, confidence: 0.85 }) + } + if (/potty twice/i.test(raw)) { + push({ category: 'potty', occurredAt: wall(15), quantity: { value: 2, unit: 'times' }, note: 'potty x2', confidence: 0.75 }) + } + // G5: the fiction becomes an "event". + if (/three heads/i.test(raw)) { + push({ category: 'milestone', quantity: { value: 3, unit: 'heads' }, note: 'babysitter has three heads', confidence: 0.5 }) + } + return out +} + +interface BrokenCapture { + proposals: ProposedEvent[] + raw: string + publishedReceipts: PublishReceipt[] + entryCounter: number +} + +interface BrokenEvent { + readonly eventId: string + readonly entryId: string + readonly childId: string + readonly category: EventCategory + readonly occurredAt: number + readonly quantity?: Quantity + readonly note?: string + readonly authorId: string +} + +export class HomeBrokenAdapter implements HomeCandidateAdapter { + readonly name = 'home-broken (negative control — must fail)' + private readonly captures = new Map() + private readonly events: BrokenEvent[] = [] + + async submitCapture(input: CaptureInput): Promise { + // G3: normalize then hash — the stored raw is no longer byte-faithful. + const normalized = input.raw.toLowerCase() + await sleep(input.attempt === 1 ? 60 : 15) + let st = this.captures.get(input.captureId) + if (st === undefined) { + st = { proposals: [], raw: normalized, publishedReceipts: [], entryCounter: 0 } + this.captures.set(input.captureId, st) + } + // P5 violation: last resolver wins, attempt number ignored — a late attempt-1 clobbers. + st.raw = normalized + st.proposals = extractBroken(input) + return { _tag: 'Proposed', captureId: input.captureId, attempt: input.attempt, proposals: st.proposals, rawSha256: sha256Hex(normalized) } + } + + async applyCorrection(input: CorrectionInput): Promise { + const st = this.captures.get(input.captureId) + if (st === undefined) return { _tag: 'UnresolvedReference', question: 'no capture' } + const action = input.action + if (action._tag === 'Reject') { + // G6: hard delete — no revision record, original gone. + st.proposals = st.proposals.filter((p) => p.localId !== input.eventLocalId) + return { _tag: 'Revised', revisionId: 'rev-gone', supersedes: input.eventLocalId, originalPreserved: true, byActorId: input.authorId } + } + st.proposals = st.proposals.map((p) => (p.localId === input.eventLocalId ? { ...p, ...action.fields } : p)) + return { _tag: 'Revised', revisionId: 'rev-x', supersedes: input.eventLocalId, originalPreserved: true, byActorId: input.authorId } + } + + async publishCapture(captureId: string, actorId: string): Promise { + const st = this.captures.get(captureId) + if (st === undefined) return { _tag: 'Refused', reason: 'nothing-proposed', disclosure: '' } + // G2: every call writes a fresh entry with a fresh receipt. + st.entryCounter += 1 + const receipt: PublishReceipt = { + entryId: `entry-broken-${captureId}-${String(st.entryCounter)}`, + proposalHash: sha256Hex(`${captureId}:${String(st.entryCounter)}`), + wroteAt: T.MOM_CAPTURED_AT + 60_000, + authoredBy: actorId, + } + st.publishedReceipts.push(receipt) + st.proposals.forEach((p, i) => { + this.events.push({ + eventId: `${receipt.entryId}-ev${String(i)}`, + entryId: receipt.entryId, + childId: 'child-lena', + category: p.category, + occurredAt: p.occurredAt ?? receipt.wroteAt, + ...(p.quantity !== undefined ? { quantity: p.quantity } : {}), + ...(p.note !== undefined ? { note: p.note } : {}), + authorId: actorId, + }) + }) + return { _tag: 'Published', receipt, idempotentHit: false } + } + + async readHistory(query: HistoryQuery): Promise { + // G1: no authorization at all — everything to everyone. + const visible = this.events.filter((ev) => ev.childId === query.childId && ev.occurredAt >= query.from && ev.occurredAt <= query.to) + const events: HistoryEvent[] = visible + .slice() + .sort((a, b) => a.occurredAt - b.occurredAt) + .map((ev) => ({ + _tag: 'Event', + eventId: ev.eventId, + entryId: ev.entryId, + category: ev.category, + occurredAt: ev.occurredAt, + ...(ev.quantity !== undefined ? { quantity: ev.quantity } : {}), + ...(ev.note !== undefined ? { note: ev.note } : {}), + authorId: ev.authorId, + })) + const gaps: HistoryGap[] = [{ _tag: 'HistoryGap', day: '2026-10-29', reason: 'no-data' }] + return { _tag: 'History', events, gaps, windowRespected: true } + } + + async askQuestion(input: AskInput): Promise { + if (/log that/i.test(input.question)) { + // G4: the question surface WRITES. + this.events.push({ + eventId: 'ev-q-write-leak', + entryId: 'entry-q-write', + childId: input.childId, + category: 'potty', + occurredAt: input.askAt, + note: 'snack (written from the question surface)', + authorId: input.actorId, + }) + } + return { _tag: 'Answer', answer: 'Here is everything.', citations: [], wrote: false } + } + + async readLineage(query: LineageQuery): Promise { + const st = this.captures.get(query.captureId) + if (st === undefined) return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + // G6: no revision records survive; the "original raw" is the normalized one (G3). + return { _tag: 'Lineage', captureId: query.captureId, revisions: [], originalRaw: st.raw } + } +} + +export function createAdapter(): HomeCandidateAdapter { + return new HomeBrokenAdapter() +} diff --git a/evaluation/src/home/example/home-example-adapter.ts b/evaluation/src/home/example/home-example-adapter.ts new file mode 100644 index 0000000..9463a3e --- /dev/null +++ b/evaluation/src/home/example/home-example-adapter.ts @@ -0,0 +1,472 @@ +/** + * Worked example adapter for the parent-home journey pack — a CONTRACT-SHAPED + * test double, NOT the extraction product. Extraction is deterministic rule + * matching over the fixture utterances (evidence label: DETERMINISTIC TEST + * DOUBLE). It demonstrates every gate PASSING so protocol runs have a known + * baseline, and so the negative control has something to be measured against. + * + * Mechanics worth copying by candidate adapters: + * - raw stored verbatim; rawSha256 over the exact bytes (G3) + * - attempt-max-wins: an older attempt resolving late returns Superseded (P5) + * - publish idempotent by captureId; replays return the same receipt (G2) + * - corrections append-only; lineage keeps originals (G6) + * - history windowed, gap-disclosing, audience-filtered (G1, G7) + * - questions read-only with citations; "log that" is refused (G4) + */ +import { readFileSync } from 'node:fs' +import type { + AskInput, + AskResult, + CaptureInput, + CorrectionInput, + CorrectionResult, + EventCategory, + HistoryEvent, + HistoryGap, + HistoryQuery, + HistoryResult, + HomeCandidateAdapter, + LineageQuery, + LineageResult, + ProposedEvent, + PublishReceipt, + PublishResult, + Quantity, + SubmitResult, +} from '../adapter.ts' +import { canonicalJson, sha256Hex } from '../../match.ts' +import * as T from '../timestamps.ts' + +interface Actor { + readonly grantedChildIds: readonly string[] +} + +interface InternalEvent { + readonly eventId: string + readonly entryId: string + readonly childId: string + readonly category: EventCategory + readonly occurredAt: number + readonly quantity?: Quantity + readonly note?: string + readonly authorId: string + readonly audience: 'household' | 'private' +} + +interface AttemptRecord { + readonly raw: string + readonly proposals: readonly ProposedEvent[] +} + +interface CaptureState { + input: CaptureInput + winnerAttempt: number + winner: AttemptRecord | undefined + readonly attempts: Map + readonly revisions: RevisionRecord[] + publishedReceipt: PublishReceipt | undefined +} + +interface RevisionRecord { + readonly revisionId: string + readonly eventLocalId: string + readonly supersedes: string + readonly byActorId: string + readonly original: ProposedEvent + readonly rejected: boolean +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +/** Wall-clock (h:m) on the capture's local date -> Unix ms, using the offset AT capturedAt. */ +function zonedWallToEpoch(capturedAt: number, timezone: string, hour: number, minute = 0): number { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: timezone, + hour12: false, + hourCycle: 'h23', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }).formatToParts(new Date(capturedAt)) + const get = (t: string): number => Number(parts.find((p) => p.type === t)?.value) + const y = get('year') + const mo = get('month') + const d = get('day') + const wallNow = Date.UTC(y, mo - 1, d, get('hour'), get('minute')) + const offset = capturedAt - wallNow + return Date.UTC(y, mo - 1, d, hour, minute) + offset +} + +function localDayKey(epochMs: number, timezone: string): string { + return new Intl.DateTimeFormat('en-CA', { + timeZone: timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + }).format(new Date(epochMs)) +} + +/** Deterministic rule-based extraction over the fixture utterances. */ +function extract(input: CaptureInput): ProposedEvent[] { + const out: ProposedEvent[] = [] + const raw = input.raw + const wall = (h: number, m = 0): number => zonedWallToEpoch(input.capturedAt, input.timezone, h, m) + let n = 0 + const push = (p: Omit): void => { + out.push({ _tag: 'ProposedEvent', localId: `pe-${String(n++)}`, ...p }) + } + + // Meal with pinned time: "half a banana at 8" (voice + typed twin). + const mealPinned = /half a ([a-z]+) at (\d{1,2})/i.exec(raw) + if (mealPinned && mealPinned[1] !== undefined && mealPinned[2] !== undefined) { + push({ + category: 'meal', + occurredAt: wall(Number(mealPinned[2])), + quantity: { value: 0.5, unit: mealPinned[1].toLowerCase() }, + note: `${mealPinned[1].toLowerCase()} meal`, + confidence: 0.95, + sourceSpan: mealPinned[0], + }) + } + + // Meal without a stated time: "two full servings of pasta" — leave unresolved. + const mealUnpinned = /two full servings of ([a-z]+)/i.exec(raw) + if (mealUnpinned && mealUnpinned[1] !== undefined) { + push({ + category: 'meal', + quantity: { value: 2, unit: 'servings' }, + note: `${mealUnpinned[1].toLowerCase()} at lunch (time not stated)`, + confidence: 0.9, + sourceSpan: mealUnpinned[0], + }) + } + + // Mood: "meltdown at drop-off" — unpinned, stay unresolved. + const meltdown = /total meltdown at drop-off/i.exec(raw) + if (meltdown !== null) { + push({ + category: 'mood', + note: 'meltdown at drop-off (time not stated)', + confidence: 0.7, + sourceSpan: meltdown[0], + }) + } + + // Sleep, voice shape: "nap at 12:30 and slept till 2". + const sleepVoice = /nap at (\d{1,2}):(\d{2}) and slept till (\d{1,2})/i.exec(raw) + if (sleepVoice && sleepVoice[1] !== undefined && sleepVoice[2] !== undefined && sleepVoice[3] !== undefined) { + const start = wall(Number(sleepVoice[1]), Number(sleepVoice[2])) + let end = wall(Number(sleepVoice[3])) + if (end < start) end += 12 * 3_600_000 // bare 12-hour hour before the start reads as afternoon + push({ + category: 'sleep', + occurredAt: start, + quantity: { value: Math.round((end - start) / 60000), unit: 'minutes' }, + confidence: 0.9, + sourceSpan: sleepVoice[0], + }) + } + + // Sleep, typed shape: "Nap from 12:30 to 2." + const sleepText = /nap from (\d{1,2}):(\d{2}) to (\d{1,2})/i.exec(raw) + if (sleepText && sleepText[1] !== undefined && sleepText[2] !== undefined && sleepText[3] !== undefined) { + const start = wall(Number(sleepText[1]), Number(sleepText[2])) + let end = wall(Number(sleepText[3])) + if (end < start) end += 12 * 3_600_000 // bare 12-hour hour before the start reads as afternoon + push({ + category: 'sleep', + occurredAt: start, + quantity: { value: Math.round((end - start) / 60000), unit: 'minutes' }, + confidence: 0.9, + sourceSpan: sleepText[0], + }) + } + + // Sleep, relative shape (corpus convention): "woke up ... about an hour ago; that nap was 45 minutes". + const sleepRelative = /nap about an hour ago[\s\S]*?was (\d{1,3}) minutes/i.exec(raw) + if (sleepRelative && sleepRelative[1] !== undefined) { + const mins = Number(sleepRelative[1]) + const end = input.capturedAt - 3_600_000 + push({ + category: 'sleep', + occurredAt: end - mins * 60000, + quantity: { value: mins, unit: 'minutes' }, + confidence: 0.85, + sourceSpan: sleepRelative[0], + }) + } + + // Potty count: "potty twice" -> one event, quantity 2, afternoon window. + if (/potty twice/i.test(raw)) { + push({ + category: 'potty', + occurredAt: wall(15), + quantity: { value: 2, unit: 'times' }, + note: 'potty x2 this afternoon (times not stated)', + confidence: 0.75, + }) + } + + // Fiction guard: "the babysitter has three heads" is a story, never an event. + return out +} + +export class HomeExampleAdapter implements HomeCandidateAdapter { + readonly name = 'home-example (deterministic test double)' + private readonly actors = new Map([ + ['actor-mom', { grantedChildIds: ['child-lena'] }], + ['actor-dad', { grantedChildIds: ['child-lena'] }], + ['actor-ruth', { grantedChildIds: ['child-lena'] }], + ['actor-marcus', { grantedChildIds: [] }], + ]) + private readonly captures = new Map() + private readonly events: InternalEvent[] = [] + private revisionSeq = 0 + private entrySeq = 0 + + constructor() { + this.seed() + } + + private seed(): void { + const raw = readFileSync(new URL('../../../fixtures/home/mini-history.json', import.meta.url), 'utf8') + const mini = JSON.parse(raw) as { + childId: string + entries: { + entryId: string + authorId: string + audience: 'household' | 'private' + events: { eventId: string; category: EventCategory; occurredAt: number; quantity?: Quantity; note?: string }[] + }[] + } + for (const entry of mini.entries) { + for (const ev of entry.events) { + this.events.push({ + eventId: ev.eventId, + entryId: entry.entryId, + childId: mini.childId, + category: ev.category, + occurredAt: ev.occurredAt, + ...(ev.quantity !== undefined ? { quantity: ev.quantity } : {}), + ...(ev.note !== undefined ? { note: ev.note } : {}), + authorId: entry.authorId, + audience: entry.audience, + }) + } + } + } + + private canRead(actorId: string, childId: string): boolean { + return this.actors.get(actorId)?.grantedChildIds.includes(childId) === true + } + + private visibleTo(actorId: string, ev: InternalEvent): boolean { + return ev.audience === 'household' || ev.authorId === actorId + } + + async submitCapture(input: CaptureInput): Promise { + if (!this.canRead(input.actorId, input.childId)) { + return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + } + // Deterministic attempt-1-slower simulation: the resumed attempt wins. + await sleep(input.attempt === 1 ? 60 : 15) + let st = this.captures.get(input.captureId) + if (st === undefined) { + st = { input, winnerAttempt: 0, winner: undefined, attempts: new Map(), revisions: [], publishedReceipt: undefined } + this.captures.set(input.captureId, st) + } + if (input.attempt < st.winnerAttempt) { + return { _tag: 'Superseded', captureId: input.captureId, attempt: input.attempt, winnerAttempt: st.winnerAttempt } + } + st.winnerAttempt = input.attempt + st.input = input + const record: AttemptRecord = { raw: input.raw, proposals: extract(input) } + st.winner = record + st.attempts.set(input.attempt, record) + return { _tag: 'Proposed', captureId: input.captureId, attempt: input.attempt, proposals: record.proposals, rawSha256: sha256Hex(input.raw) } + } + + async applyCorrection(input: CorrectionInput): Promise { + const st = this.captures.get(input.captureId) + if (st === undefined) { + return { _tag: 'UnresolvedReference', question: 'No capture found for that reference.' } + } + if (!this.canRead(input.authorId, st.input.childId)) { + return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + } + const live = st.winner?.proposals + if (live === undefined) { + return { _tag: 'UnresolvedReference', question: 'Capture has no proposals to correct.' } + } + const target = live.find((p) => p.localId === input.eventLocalId) + if (target === undefined) { + return { + _tag: 'UnresolvedReference', + question: input.viaStatement !== undefined ? `Which record does "${input.viaStatement}" refer to?` : 'Unknown record.', + } + } + const revision: RevisionRecord = { + revisionId: `rev-${String(this.revisionSeq++)}`, + eventLocalId: input.eventLocalId, + supersedes: input.eventLocalId, + byActorId: input.authorId, + original: { ...target }, + rejected: input.action._tag === 'Reject', + } + st.revisions.push(revision) + if (st.winner !== undefined) { + if (input.action._tag === 'Reject') { + st.winner = { ...st.winner, proposals: live.filter((p) => p.localId !== input.eventLocalId) } + } else if (input.action._tag === 'Patch') { + const fields = input.action.fields + st.winner = { + ...st.winner, + proposals: live.map((p) => (p.localId === input.eventLocalId ? { ...p, ...fields } : p)), + } + } + } + return { _tag: 'Revised', revisionId: revision.revisionId, supersedes: input.eventLocalId, originalPreserved: true, byActorId: input.authorId } + } + + async publishCapture(captureId: string, actorId: string): Promise { + const st = this.captures.get(captureId) + if (st === undefined) return { _tag: 'Refused', reason: 'nothing-proposed', disclosure: '' } + if (!this.canRead(actorId, st.input.childId)) return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + if (st.publishedReceipt !== undefined) { + return { _tag: 'AlreadyPublished', receipt: st.publishedReceipt, idempotentHit: true } + } + const proposals = st.winner?.proposals ?? [] + const receipt: PublishReceipt = { + entryId: `entry-${sha256Hex(captureId).slice(0, 12)}`, + proposalHash: sha256Hex(canonicalJson(proposals)), + wroteAt: st.input.capturedAt + 5 * 60000, + authoredBy: st.input.actorId, + } + st.publishedReceipt = receipt + proposals.forEach((p, i) => { + this.events.push({ + eventId: `${receipt.entryId}-ev${String(i)}`, + entryId: receipt.entryId, + childId: st.input.childId, + category: p.category, + occurredAt: p.occurredAt ?? receipt.wroteAt, + ...(p.quantity !== undefined ? { quantity: p.quantity } : {}), + ...(p.note !== undefined ? { note: p.note } : {}), + authorId: st.input.actorId, + audience: 'household', + }) + }) + this.entrySeq += 1 + return { _tag: 'Published', receipt, idempotentHit: false } + } + + private historyFor(actorId: string, childId: string, from: number, to: number): HistoryResult { + const visible = this.events.filter( + (ev) => ev.childId === childId && ev.occurredAt >= from && ev.occurredAt <= to && this.visibleTo(actorId, ev), + ) + const days = new Set() + for (const ev of visible) days.add(localDayKey(ev.occurredAt, 'America/New_York')) + const gaps: HistoryGap[] = [] + const seenDays = new Set() + const lastKey = localDayKey(to, 'America/New_York') + for (let t = from; t <= to; t += 86_400_000) { + const key = localDayKey(t, 'America/New_York') + if (seenDays.has(key) || key > lastKey) continue + seenDays.add(key) + if (!days.has(key)) gaps.push({ _tag: 'HistoryGap', day: key, reason: 'no-data' }) + } + if (!seenDays.has(lastKey) && !days.has(lastKey)) { + gaps.push({ _tag: 'HistoryGap', day: lastKey, reason: 'no-data' }) + seenDays.add(lastKey) + } + const events: HistoryEvent[] = visible + .slice() + .sort((a, b) => a.occurredAt - b.occurredAt) + .map((ev) => ({ + _tag: 'Event', + eventId: ev.eventId, + entryId: ev.entryId, + category: ev.category, + occurredAt: ev.occurredAt, + ...(ev.quantity !== undefined ? { quantity: ev.quantity } : {}), + ...(ev.note !== undefined ? { note: ev.note } : {}), + authorId: ev.authorId, + })) + return { _tag: 'History', events, gaps, windowRespected: true } + } + + async readHistory(query: HistoryQuery): Promise { + if (!this.canRead(query.actorId, query.childId)) { + return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + } + return this.historyFor(query.actorId, query.childId, query.from, query.to) + } + + async askQuestion(input: AskInput): Promise { + if (!this.canRead(input.actorId, input.childId)) { + return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + } + if (/^log |log that /i.test(input.question)) { + // Read-only surface: a write request is answered, never executed (G4). + return { + _tag: 'Answer', + answer: "Questions are read-only — I can't log that from here. Use capture to record it.", + citations: [], + wrote: false, + } + } + const window = { from: T.MINI_HISTORY_WINDOW.from, to: input.askAt } + const visible = this.events.filter( + (ev) => ev.childId === input.childId && ev.occurredAt >= window.from && ev.occurredAt <= window.to && this.visibleTo(input.actorId, ev), + ) + const cite = (ev: InternalEvent): { entryId: string; eventId: string } => ({ entryId: ev.entryId, eventId: ev.eventId }) + if (/eat|food|meal/i.test(input.question)) { + const meals = visible.filter((ev) => ev.category === 'meal').sort((a, b) => a.occurredAt - b.occurredAt) + const lines = meals.map((ev) => { + const qty = ev.quantity !== undefined ? ` ${String(ev.quantity.value)} ${ev.quantity.unit}` : '' + return `${localDayKey(ev.occurredAt, 'America/New_York')} ${String(ev.occurredAt)}: ${ev.category}${qty}` + }) + return { _tag: 'Answer', answer: `Meals on file: ${lines.join('; ')}`, citations: meals.map(cite), wrote: false } + } + if (/nap|sleep/i.test(input.question)) { + const last = visible.filter((ev) => ev.category === 'sleep').sort((a, b) => a.occurredAt - b.occurredAt).at(-1) + if (last === undefined) { + return { _tag: 'Answer', answer: 'No naps on file in the window.', citations: [], wrote: false } + } + return { _tag: 'Answer', answer: `Last nap: ${String(last.occurredAt)}${last.quantity !== undefined ? ` (${String(last.quantity.value)} ${last.quantity.unit})` : ''}`, citations: [cite(last)], wrote: false } + } + return { + _tag: 'Answer', + answer: `${String(visible.length)} events on file in the window.`, + citations: visible.slice(0, 5).map(cite), + wrote: false, + } + } + + async readLineage(query: LineageQuery): Promise { + const st = this.captures.get(query.captureId) + if (st === undefined) return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + if (!this.canRead(query.actorId, st.input.childId)) return { _tag: 'Refused', reason: 'unauthorized', disclosure: '' } + return { + _tag: 'Lineage', + captureId: query.captureId, + revisions: st.revisions.map((r) => ({ + revisionId: r.revisionId, + eventLocalId: r.eventLocalId, + action: r.rejected ? ({ _tag: 'Reject' } as const) : { _tag: 'Patch', fields: {} }, + supersedes: r.supersedes, + byActorId: r.byActorId, + original: r.original, + })), + originalRaw: st.winner?.raw ?? st.input.raw, + } + } +} + +export function createAdapter(): HomeCandidateAdapter { + return new HomeExampleAdapter() +} diff --git a/evaluation/src/home/match.ts b/evaluation/src/home/match.ts new file mode 100644 index 0000000..aae68b2 --- /dev/null +++ b/evaluation/src/home/match.ts @@ -0,0 +1,184 @@ +/** + * Matcher for the journey fixtures: compares an adapter's proposed events + * against the expected block in utterances.json. Extends the merged + * harness matcher with window expectations (unpinned expressions) and + * occurrence counts. Reuses canonicalJson/deepEqual/sha256Hex/normalizeNote + * from the existing harness match module. + */ +import type { EventCategory, ProposedEvent } from './adapter.ts' +import { canonicalJson, deepEqual, normalizeNote } from '../match.ts' + +export interface WindowExpectation { + readonly from: number + readonly to: number +} + +export interface EventExpectation { + readonly category: EventCategory + readonly occurrence: 'exactly-one' | 'at-least-one' + readonly occurredAt?: number + readonly toleranceMs?: number + readonly occurredAtWindow?: WindowExpectation + readonly occurredAtAbsentAllowed?: boolean + readonly quantity?: { readonly value: number; readonly unit: string } + readonly noteContains?: readonly string[] +} + +export interface ExpectedBlock { + readonly events: readonly EventExpectation[] + /** Extra proposals allowed beyond the expected occurrences. */ + readonly extrasAllow?: readonly { readonly category: EventCategory }[] + /** When true, ANY extra proposal is a failure. */ + readonly exactly?: boolean + /** No proposal may carry a note containing any of these (figure-of-speech guard, G5). */ + readonly forbiddenNoteSubstrings?: readonly string[] +} + +function occurredAtMismatches(p: ProposedEvent, exp: EventExpectation): string[] { + const bad: string[] = [] + const hasOccurredAt = p.occurredAt !== undefined + if (p.occurredAt === null as unknown as undefined) bad.push('occurredAt: explicit null — omit the field instead') + + if (exp.occurredAt !== undefined) { + if (!hasOccurredAt) { + bad.push(`occurredAt: pinned expectation ${String(exp.occurredAt)} but field absent`) + } else { + const tolerance = exp.toleranceMs ?? 0 + const drift = Math.abs(p.occurredAt - exp.occurredAt) + if (drift > tolerance) { + bad.push( + `occurredAt: expected ${String(exp.occurredAt)}, got ${String(p.occurredAt)} ` + + `(drift ${String(drift)}ms > tolerance ${String(tolerance)}ms)`, + ) + } + } + return bad + } + + if (exp.occurredAtWindow !== undefined) { + if (!hasOccurredAt) { + if (exp.occurredAtAbsentAllowed === true) return bad + bad.push('occurredAt: absent but unresolved instants are not allowed for this expectation') + return bad + } + const w = exp.occurredAtWindow + if (p.occurredAt < w.from || p.occurredAt > w.to) { + bad.push( + `occurredAt ${String(p.occurredAt)} outside window [${String(w.from)}, ${String(w.to)}] — ` + + 'unpinned expressions must not gain false precision (G5)', + ) + } + return bad + } + + // No time expectation at all: any value (or absence) is acceptable. + return bad +} + +function proposalMismatches(p: ProposedEvent, exp: EventExpectation): string[] { + const bad: string[] = [] + if (p._tag !== 'ProposedEvent') bad.push(`_tag must be "ProposedEvent", got ${JSON.stringify(p._tag)}`) + if (p.category !== exp.category) { + bad.push(`category: expected "${String(exp.category)}", got ${JSON.stringify(p.category)}`) + } + bad.push(...occurredAtMismatches(p, exp)) + + if (exp.quantity !== undefined) { + if (p.quantity === undefined) { + bad.push(`quantity: expected ${canonicalJson(exp.quantity)}, absent`) + } else if (!deepEqual(p.quantity, exp.quantity)) { + bad.push(`quantity: expected ${canonicalJson(exp.quantity)}, got ${canonicalJson(p.quantity)}`) + } + } + if (p.quantity !== undefined && (typeof p.quantity.value !== 'number' || !Number.isFinite(p.quantity.value))) { + bad.push('quantity.value must be a finite number when present') + } + + if (typeof p.confidence !== 'number' || !Number.isFinite(p.confidence) || p.confidence < 0 || p.confidence > 1) { + bad.push(`confidence must be finite within [0,1], got ${JSON.stringify(p.confidence)}`) + } + if (p.note !== undefined && (typeof p.note !== 'string' || p.note.length === 0)) { + bad.push('note must be a non-empty string when present') + } + if (exp.noteContains !== undefined) { + const note = typeof p.note === 'string' ? normalizeNote(p.note) : '' + for (const needle of exp.noteContains) { + if (!note.includes(normalizeNote(needle))) { + bad.push(`note: expected it to contain "${needle}", got ${JSON.stringify(p.note ?? null)}`) + } + } + } + return bad +} + +/** + * Match proposed events against an expected block. Returns mismatch + * descriptions (empty array = match). Category groups are matched per + * expectation; occurrence counts must hold; extras are failures unless + * allowed (or the block declares `exactly`). + */ +export function expectedSetMismatches( + proposals: readonly ProposedEvent[], + expected: ExpectedBlock, +): string[] { + const bad: string[] = [] + const consumed = new Set() + + for (const exp of expected.events) { + const group = proposals.filter((p) => p.category === exp.category && !consumed.has(p.localId)) + if (exp.occurrence === 'exactly-one' && group.length !== 1) { + bad.push(`category "${String(exp.category)}": expected exactly 1, got ${String(group.length)}`) + continue + } + if (exp.occurrence === 'at-least-one' && group.length < 1) { + bad.push(`category "${String(exp.category)}": expected at least 1, got 0`) + continue + } + for (const p of group) { + for (const m of proposalMismatches(p, exp)) { + bad.push(`${String(exp.category)}[${p.localId}]: ${m}`) + } + consumed.add(p.localId) + } + } + + const extras = proposals.filter((p) => !consumed.has(p.localId)) + if (extras.length > 0) { + if (expected.exactly === true) { + bad.push( + `exactly-block violated: ${String(extras.length)} extra proposal(s) ` + + `[${extras.map((p) => p.category).join(', ')}] — figures of speech must not become events (G5)`, + ) + } else { + const allowed = expected.extrasAllow ?? [] + for (const extra of extras) { + if (!allowed.some((a) => a.category === extra.category)) { + bad.push(`unexpected extra proposal of category "${String(extra.category)}"`) + } + } + } + } + + if (expected.forbiddenNoteSubstrings !== undefined) { + for (const p of proposals) { + const hay = normalizeNote(`${typeof p.note === 'string' ? p.note : ''} ${typeof p.sourceSpan === 'string' ? p.sourceSpan : ''}`) + for (const forbidden of expected.forbiddenNoteSubstrings) { + if (hay.includes(normalizeNote(forbidden))) { + bad.push(`forbidden substring "${forbidden}" appears in ${p.category}[${p.localId}] — figure-of-speech leak (G5)`) + } + } + } + } + return bad +} + +/** Corpus convention check for u1: "twice" = two events, or one event with quantity 2. */ +export function pottyCoverageMismatches(proposals: readonly ProposedEvent[]): string[] { + const potty = proposals.filter((p) => p.category === 'potty') + const twoEvents = potty.length >= 2 + const quantityTwo = potty.some((p) => p.quantity?.value === 2) + if (!twoEvents && !quantityTwo) { + return [`potty "twice" unresolved: ${String(potty.length)} event(s), none with quantity 2 (G5)`] + } + return [] +} diff --git a/evaluation/src/home/run.ts b/evaluation/src/home/run.ts new file mode 100644 index 0000000..294018e --- /dev/null +++ b/evaluation/src/home/run.ts @@ -0,0 +1,386 @@ +/** + * CLI entry: `bun src/home/run.ts [--adapter=] [--expect-failure]` + * + * Parent-home journey protocols P1-P6 (rubric art_uWsp80Fg §8), executed + * against a HomeCandidateAdapter. Mirrors the merged harness CLI: + * + * --adapter= Module exporting `createAdapter(): HomeCandidateAdapter`. + * Defaults to the worked example adapter. + * --expect-failure Invert the exit code: exit 0 iff the run found failures + * (used by the broken-adapter negative control). + * + * Evidence label for every run: DETERMINISTIC TEST DOUBLE — no live LLM, no + * device, no cloud. Pinned instants come from timestamps.ts and are asserted + * at import. + */ +import { readFileSync } from 'node:fs' +import type { AskInput, AskResult, CaptureInput, HistoryQuery, HistoryResult, HomeCandidateAdapter, ProposedEvent, SubmitResult } from './adapter.ts' +import { deepEqual, sha256Hex } from '../match.ts' +import { expectedSetMismatches, pottyCoverageMismatches, type ExpectedBlock } from './match.ts' +import * as T from './timestamps.ts' + +interface UtteranceFixture { + readonly captureId: string + readonly mode: 'voice-transcript' | 'text' + readonly actorId: string + readonly childId: string + readonly capturedAt: number + readonly timezone: string + readonly raw: string + readonly expected?: { readonly events: readonly never[] } & ExpectedBlock + readonly parityWith?: string + readonly stale?: { + readonly fragmentRaw: string + readonly fragmentAttempt: number + readonly resumedRaw: string + readonly resumedAttempt: number + readonly expectedWinnerEvents: number + } +} + +interface UtterancesFixture { + readonly captures: Readonly> + readonly corrections: { + readonly u4: { readonly viaStatement: string; readonly targetCaptureId: string; readonly targetCategory: string; readonly expectPatch: { readonly quantity: { readonly value: number; readonly unit: string } }; readonly byActorId: string } + readonly noTargetStatement: string + readonly rejectFromCaptureId: string + readonly rejectTargetCategory: string + readonly rejectByActorId: string + } + readonly questions: { + readonly q1: AskInput & { readonly text: string } + readonly q2: AskInput & { readonly text: string } + readonly q3WriteAttempt: AskInput & { readonly text: string } + readonly privateEntryId: string + readonly askAt: number + } + readonly authorization: { readonly nonMemberActorId: string; readonly outOfScopeChildId: string } +} + +interface Check { + readonly protocol: string + readonly ok: boolean + readonly failures: readonly string[] +} + +function loadFixture(): UtterancesFixture { + const url = new URL('../../fixtures/home/utterances.json', import.meta.url) + return JSON.parse(readFileSync(url, 'utf8')) as UtterancesFixture +} + +function captureOf(u: UtteranceFixture, raw: string, attempt: number, capturedAt: number): CaptureInput { + return { + captureId: u.captureId, + attempt, + mode: u.mode, + raw, + actorId: u.actorId, + childId: u.childId, + capturedAt, + timezone: u.timezone, + } +} + +function submitTag(r: SubmitResult): string { + return r._tag +} + +/** All refusals must disclose nothing — asserted on every Refused result. */ +function disclosureFailures(tag: string, r: { readonly _tag: string; readonly disclosure?: string }): string[] { + if (r._tag === 'Refused' && r.disclosure !== '') { + return [`${tag}: Refused carried a non-empty disclosure (leak): ${JSON.stringify(r.disclosure ?? null)}`] + } + return [] +} + +export async function runProtocols(adapter: HomeCandidateAdapter): Promise { + const fixtures = loadFixture() + const u1 = fixtures.captures['u1'] + const u2 = fixtures.captures['u2'] + const u1b = fixtures.captures['u1b'] + const u3 = fixtures.captures['u3'] + if (u1 === undefined || u2 === undefined || u1b === undefined || u3 === undefined || u1.expected === undefined) { + throw new Error('utterances.json is missing u1/u2/u1b/u3 or u1.expected') + } + const expectedU1: ExpectedBlock = u1.expected + const checks: Check[] = [] + + // ---- P1: capture parity (voice u1 vs text u2) ---------------------------- + const p1Proposals = new Map() + for (const [id, u, exp] of [ + ['P1/voice', u1, expectedU1], + ['P1/text', u2, expectedU1], // parityWith: shared expected block + ] as const) { + const r = await adapter.submitCapture(captureOf(u, u.raw, 1, u.capturedAt)) + const failures: string[] = [] + if (r._tag !== 'Proposed') failures.push(`submit returned ${submitTag(r)}, expected Proposed`) + if (r._tag === 'Proposed') { + if (r.rawSha256 !== sha256Hex(u.raw)) failures.push('rawSha256 does not match the raw utterance — raw not stored byte-for-byte (G3)') + failures.push(...expectedSetMismatches(r.proposals, exp)) + if (id === 'P1/voice') failures.push(...pottyCoverageMismatches(r.proposals)) + p1Proposals.set(u.captureId, r.proposals) + } + failures.push(...disclosureFailures(id, r)) + checks.push({ protocol: id, ok: failures.length === 0, failures }) + } + + // ---- P5: stale-result suppression (u1b interruption) --------------------- + { + const st = u1b.stale + if (st === undefined) throw new Error('u1b.stale missing') + // Both attempts in flight; attempt 1 must lose to attempt 2 when it + // resolves late. A fully-synchronous adapter may resolve attempt 1 first + // (nothing stale yet) — final-state equality still proves no clobber. + const p1 = adapter.submitCapture(captureOf(u1b, st.fragmentRaw, st.fragmentAttempt, T.MOM_CAPTURED_AT)) + const p2 = adapter.submitCapture(captureOf(u1b, st.resumedRaw, st.resumedAttempt, T.U1B_CAPTURED_AT)) + const r2 = await p2 + const r1 = await p1 + const failures: string[] = [] + if (r2._tag !== 'Proposed') failures.push(`resumed attempt returned ${submitTag(r2)}, expected Proposed`) + if (r1._tag === 'Proposed' && r2._tag === 'Proposed') { + failures.push('older attempt resolved as Proposed after the newer attempt was submitted — expected Superseded (P5)') + } + if (r1._tag === 'Superseded' && r1.winnerAttempt !== st.resumedAttempt) { + failures.push(`Superseded names winnerAttempt ${String(r1.winnerAttempt)}, expected ${String(st.resumedAttempt)}`) + } + // The winner's proposals stand; the fragment produced none. + if (r2._tag === 'Proposed' && r2.proposals.length !== st.expectedWinnerEvents) { + failures.push(`winner proposals: expected ${String(st.expectedWinnerEvents)} events, got ${String(r2.proposals.length)}`) + } + // Publish the interrupted capture: zero events must persist; nothing from the fragment. + const pub = await adapter.publishCapture(u1b.captureId, u1b.actorId) + if (pub._tag !== 'Published' && pub._tag !== 'AlreadyPublished') { + failures.push(`publish of zero-proposal capture returned ${pub._tag} — extraction must never block capture`) + } + const hist = await adapter.readHistory({ actorId: u1b.actorId, childId: u1b.childId, from: T.MONTH_WINDOW_NOV.from, to: T.MONTH_WINDOW_NOV.to }) + if (hist._tag === 'History') { + for (const ev of hist.events) { + const hay = `${ev.note ?? ''}`.toLowerCase() + if (hay.includes('wore her')) failures.push(`fragment leak in history: event ${ev.eventId} note ${JSON.stringify(ev.note)}`) + } + } + checks.push({ protocol: 'P5/stale', ok: failures.length === 0, failures }) + } + + // ---- P2: idempotent publish (u1) ----------------------------------------- + { + const proposals = p1Proposals.get(u1.captureId) + const r1 = await adapter.publishCapture(u1.captureId, u1.actorId) + const r2 = await adapter.publishCapture(u1.captureId, u1.actorId) + const failures: string[] = [] + if (r1._tag !== 'Published') failures.push(`first publish returned ${r1._tag}, expected Published`) + if (r2._tag !== 'AlreadyPublished') failures.push(`second publish returned ${r2._tag}, expected AlreadyPublished (G2)`) + if (r1._tag === 'Published' && r2._tag === 'AlreadyPublished') { + if (!deepEqual(r1.receipt, r2.receipt)) failures.push('receipts differ across replays — idempotent path returned a different receipt (G2)') + } + const hist = await adapter.readHistory({ actorId: u1.actorId, childId: u1.childId, from: T.MONTH_WINDOW_NOV.from, to: T.MONTH_WINDOW_NOV.to }) + if (hist._tag === 'History' && r1._tag === 'Published' && proposals !== undefined) { + const entryEvents = hist.events.filter((e) => e.entryId === r1.receipt.entryId) + if (entryEvents.length !== proposals.length) { + failures.push(`history has ${String(entryEvents.length)} events for the entry, expected ${String(proposals.length)} — duplicate write (G2)`) + } + for (const ev of entryEvents) { + if (ev.authorId !== u1.actorId) failures.push(`event ${ev.eventId} author ${ev.authorId}, expected ${u1.actorId}`) + } + } + failures.push(...disclosureFailures('P2', r1), ...disclosureFailures('P2', r2)) + checks.push({ protocol: 'P2/idempotent-publish', ok: failures.length === 0, failures }) + } + + // ---- P3: no-write proof + audience (ruth) -------------------------------- + { + const q = fixtures.questions + const window: HistoryQuery = { actorId: 'actor-ruth', childId: u1.childId, from: T.MINI_HISTORY_WINDOW.from, to: T.MONTH_WINDOW_NOV.to } + const before: HistoryResult = await adapter.readHistory(window) + const failures: string[] = [] + if (before._tag !== 'History') failures.push(`ruth readHistory returned ${before._tag} — granted caregiver must read`) + if (before._tag === 'History') { + if (before.windowRespected !== true) failures.push('windowRespected not asserted') + if (!before.gaps.some((g) => g.day === '2026-10-29')) failures.push(`missed day 2026-10-29 not disclosed as a gap, gaps: ${JSON.stringify(before.gaps)}`) + for (const ev of before.events) { + if (ev.occurredAt < window.from || ev.occurredAt > window.to) failures.push(`event ${ev.eventId} outside window (G7)`) + if (ev.entryId === q.privateEntryId) failures.push(`private entry ${q.privateEntryId} visible to ruth (G1)`) + } + // Ruth (caregiver) may correct/see household events but the private-audience + // event must be absent entirely — no leak surface. + if (before.events.some((e) => e.eventId === 'ev-o31-private-note')) { + failures.push('private-audience event present in ruth history (G1)') + } + } + for (const [tag, question] of [['P3/q1', q.q1], ['P3/q2', q.q2], ['P3/q3-write', q.q3WriteAttempt]] as const) { + const a: AskResult = await adapter.askQuestion({ ...question, askAt: q.askAt }) + if (a._tag === 'Answer') { + if (a.wrote !== false) failures.push(`${tag}: answer surface declared a write (G4)`) + if (tag !== 'P3/q3-write' && a.citations.length === 0) failures.push(`${tag}: answer has no source citations`) + for (const c of a.citations) { + if (c.entryId === q.privateEntryId) failures.push(`${tag}: citation leaks private entry (G1)`) + } + } + failures.push(...disclosureFailures(tag, a)) + } + const after: HistoryResult = await adapter.readHistory(window) + if (!deepEqual(before, after)) { + failures.push('history state changed across questions — the question surface wrote (G4)') + } + // Author visibility control: mom sees her own private entry. + const momHist = await adapter.readHistory({ actorId: 'actor-mom', childId: u1.childId, from: T.MINI_HISTORY_WINDOW.from, to: T.MONTH_WINDOW_NOV.to }) + if (momHist._tag === 'History' && !momHist.events.some((e) => e.entryId === q.privateEntryId)) { + failures.push('private entry invisible even to its author — audience filter is over-blocking') + } + checks.push({ protocol: 'P3/no-write+audience', ok: failures.length === 0, failures }) + } + + // ---- P4: correction lineage (u4 patch, reject, unresolved) --------------- + { + const corr = fixtures.corrections + const failures: string[] = [] + // Submit u3 (dad) for the NL-correction target. + const r3 = await adapter.submitCapture(captureOf(u3, u3.raw, 1, u3.capturedAt)) + if (r3._tag !== 'Proposed') failures.push(`u3 submit returned ${submitTag(r3)}, expected Proposed`) + if (r3._tag === 'Proposed') { + const sleep = r3.proposals.find((p) => p.category === corr.u4.targetCategory) + if (sleep === undefined) failures.push('u3 has no sleep proposal to correct') + else { + const patched = await adapter.applyCorrection({ + captureId: corr.u4.targetCaptureId, + eventLocalId: sleep.localId, + action: { _tag: 'Patch', fields: { quantity: corr.u4.expectPatch.quantity } }, + authorId: corr.u4.byActorId, + viaStatement: corr.u4.viaStatement, + }) + if (patched._tag !== 'Revised') failures.push(`u4 patch returned ${patched._tag}, expected Revised`) + if (patched._tag === 'Revised') { + if (patched.originalPreserved !== true) failures.push('patch did not preserve the original (G6)') + if (patched.supersedes !== sleep.localId) failures.push(`patch supersedes ${patched.supersedes}, expected ${sleep.localId}`) + if (patched.byActorId !== corr.u4.byActorId) failures.push(`reviser attribution ${patched.byActorId}, expected ${corr.u4.byActorId}`) + } + const lineage = await adapter.readLineage({ captureId: corr.u4.targetCaptureId, actorId: corr.u4.byActorId }) + if (lineage._tag !== 'Lineage') failures.push(`readLineage returned ${lineage._tag}, expected Lineage`) + if (lineage._tag === 'Lineage') { + if (lineage.originalRaw !== u3.raw) failures.push('lineage originalRaw differs from the raw utterance — raw not preserved verbatim (G3)') + const rev = lineage.revisions.find((rv) => rv.supersedes === sleep.localId) + if (rev === undefined) failures.push('patch revision missing from lineage') + else if (rev.original.quantity?.value !== 45) { + failures.push(`lineage original quantity ${JSON.stringify(rev.original.quantity ?? null)}, expected the original 45 minutes (G6)`) + } + } + } + // Reject the u1 potty (dad, non-author) — revision, not hard delete. + const u1Props = p1Proposals.get(u1.captureId) + const potty = u1Props?.find((p) => p.category === corr.rejectTargetCategory) + if (potty === undefined) failures.push('u1 has no potty proposal to reject') + else { + const rej = await adapter.applyCorrection({ + captureId: corr.rejectFromCaptureId, + eventLocalId: potty.localId, + action: { _tag: 'Reject' }, + authorId: corr.rejectByActorId, + }) + if (rej._tag !== 'Revised') failures.push(`reject returned ${rej._tag}, expected Revised (rejection is a revision, G6)`) + const lineage1 = await adapter.readLineage({ captureId: corr.rejectFromCaptureId, actorId: corr.rejectByActorId }) + if (lineage1._tag === 'Lineage') { + const rev = lineage1.revisions.find((rv) => rv.eventLocalId === potty.localId) + if (rev === undefined) failures.push('rejected event left no revision record — hard delete (G6)') + else if (rev.original.category !== corr.rejectTargetCategory) failures.push('revision original does not hold the rejected record') + } + } + // Unresolved reference must come back as a question, never a guess. + const noTarget = await adapter.applyCorrection({ + captureId: corr.rejectFromCaptureId, + eventLocalId: 'no-such-event', + action: { _tag: 'Patch', fields: { note: 'three waffles' } }, + authorId: corr.u4.byActorId, + viaStatement: corr.noTargetStatement, + }) + if (noTarget._tag !== 'UnresolvedReference') failures.push(`bogus-target correction returned ${noTarget._tag}, expected UnresolvedReference (no invented targets)`) + } + checks.push({ protocol: 'P4/lineage', ok: failures.length === 0, failures }) + } + + // ---- P6: authorization boundary ------------------------------------------ + { + const authz = fixtures.authorization + const failures: string[] = [] + const marcusCapture = await adapter.submitCapture({ + captureId: 'cap-marcus-probe', + attempt: 1, + mode: 'text', + raw: 'anything', + actorId: authz.nonMemberActorId, + childId: u1.childId, + capturedAt: T.MOM_CAPTURED_AT, + timezone: 'America/New_York', + }) + if (marcusCapture._tag !== 'Refused') failures.push(`non-member submit returned ${marcusCapture._tag}, expected Refused (G1)`) + const marcusHist = await adapter.readHistory({ actorId: authz.nonMemberActorId, childId: u1.childId, from: T.MINI_HISTORY_WINDOW.from, to: T.MONTH_WINDOW_NOV.to }) + if (marcusHist._tag !== 'Refused') failures.push(`non-member readHistory returned ${marcusHist._tag}, expected Refused (G1)`) + const marcusAsk = await adapter.askQuestion({ actorId: authz.nonMemberActorId, childId: u1.childId, question: 'What did she eat today?', askAt: T.MOM_CAPTURED_AT }) + if (marcusAsk._tag !== 'Refused') failures.push(`non-member ask returned ${marcusAsk._tag}, expected Refused (G1)`) + const marcusCorr = await adapter.applyCorrection({ captureId: u1.captureId, eventLocalId: 'any', action: { _tag: 'Reject' }, authorId: authz.nonMemberActorId }) + if (marcusCorr._tag !== 'Refused') failures.push(`non-member correction returned ${marcusCorr._tag}, expected Refused (G1)`) + const theo = await adapter.readHistory({ actorId: 'actor-ruth', childId: authz.outOfScopeChildId, from: T.MINI_HISTORY_WINDOW.from, to: T.MONTH_WINDOW_NOV.to }) + if (theo._tag !== 'Refused') failures.push(`out-of-scope child read returned ${theo._tag}, expected Refused (G1)`) + // Positive control: authorized reads still work after the refusals. + const dad = await adapter.readHistory({ actorId: 'actor-dad', childId: u1.childId, from: T.MINI_HISTORY_WINDOW.from, to: T.MONTH_WINDOW_NOV.to }) + if (dad._tag !== 'History') failures.push(`authorized dad readHistory returned ${dad._tag} — adapter must not refuse everyone`) + for (const [tag, r] of [ + ['P6/submit', marcusCapture], + ['P6/history', marcusHist], + ['P6/ask', marcusAsk], + ['P6/correct', marcusCorr], + ['P6/scope', theo], + ] as const) { + failures.push(...disclosureFailures(tag, r)) + } + checks.push({ protocol: 'P6/authorization', ok: failures.length === 0, failures }) + } + + return checks +} + +async function loadHomeAdapter(adapterArg: string | undefined): Promise { + if (adapterArg === undefined) { + const mod = await import('./example/home-example-adapter.ts') + return mod.createAdapter() + } + const { pathToFileURL, fileURLToPath } = await import('node:url') + const { resolve } = await import('node:path') + const packageRoot = fileURLToPath(new URL('../..', import.meta.url)) + const modulePath = pathToFileURL(resolve(packageRoot, adapterArg)).href + const mod = (await import(modulePath)) as { createAdapter?: unknown } + if (typeof mod.createAdapter !== 'function') { + throw new Error(`adapter module ${adapterArg} must export a named factory: createAdapter(): HomeCandidateAdapter`) + } + const adapter = (mod.createAdapter as () => HomeCandidateAdapter)() + if (typeof adapter?.name !== 'string' || adapter.name.length === 0) { + throw new Error(`adapter factory in ${adapterArg} returned an object without a non-empty "name"`) + } + return adapter +} + +async function main(): Promise { + const args = process.argv.slice(2) + const adapterArg = args.find((a) => a.startsWith('--adapter='))?.slice('--adapter='.length) + const expectFailure = args.includes('--expect-failure') + + const adapter = await loadHomeAdapter(adapterArg) + const checks = await runProtocols(adapter) + + console.log(`\nParent-home journey protocols — adapter: ${adapter.name}`) + console.log('Evidence label: DETERMINISTIC TEST DOUBLE (no live LLM, no device, no cloud)') + console.log('='.repeat(72)) + for (const c of checks) { + console.log(` ${c.ok ? 'PASS' : 'FAIL'} ${c.protocol}`) + for (const f of c.failures) console.log(` - ${f}`) + } + console.log('='.repeat(72)) + const failed = checks.filter((c) => !c.ok).length + console.log(`${String(checks.length - failed)}/${String(checks.length)} protocols passed`) + if (expectFailure) { + console.log(failed > 0 ? 'Negative control: failures detected as expected.' : 'Negative control: adapter passed unexpectedly — it should violate the gates.') + return failed > 0 ? 0 : 1 + } + return failed === 0 ? 0 : 1 +} + +process.exitCode = await main() diff --git a/evaluation/src/home/timestamps.ts b/evaluation/src/home/timestamps.ts new file mode 100644 index 0000000..db4b111 --- /dev/null +++ b/evaluation/src/home/timestamps.ts @@ -0,0 +1,139 @@ +/** + * Pinned epochs for the parent-home journey fixtures — SINGLE SOURCE OF TRUTH. + * + * Fixtures land on 2026-11-01, the US DST fall-back day (America/New_York): + * 02:00 EDT becomes 01:00 EST, so all Nov 1 instants are EST (UTC-5) while + * seed days Oct 28–31 are EDT (UTC-4). Every value below was derived by + * machine (fixed-offset method: UTC = local + offset) — never by hand + * arithmetic — and re-asserted at import time: importing this module with a + * wrong constant aborts the run. + * + * Narrative timeline (all America/New_York): + * Oct 28–31 seed mini-history (published entries; Oct 29 has no data — the gap) + * Nov 1 08:00 breakfast (u1 narrates it in the evening) + * Nov 1 10:30–11:15 Lena's nap per dad (ended "about an hour" before his 11:15 log) + * Nov 1 12:15 dad logs u3 (pasta lunch around noon, three-heads fiction) + * Nov 1 20:00 mom's evening retrospective u1 (u2 is its text twin, same instant) + * Nov 1 20:15 mom's resumed interrupted capture u1b + * Nov 2 09:00 Ruth (grandma) asks her catch-up questions + */ +import { deepEqual } from '../match.ts' + +// --- seed window (Oct, EDT = UTC-4) ------------------------------------ +export const MINI_HISTORY_WINDOW = { + from: 1793160000000, // Oct 28 00:00 EDT + to: 1793505599999, // Oct 31 23:59:59.999 EDT +} as const + +/** Local Nov 1 begins here (00:00 EDT = 04:00Z); seed events must stay below it. */ +export const NOV_1_MIDNIGHT_EDT = 1793505600000 + +/** November 2026 local month window (Nov 1 00:00 local .. Nov 30 23:59:59.999 EST). */ +export const MONTH_WINDOW_NOV = { from: 1793505600000, to: 1796101199999 } as const + +// --- capture instants (Nov 1, EST = UTC-5) ------------------------------ +export const MOM_CAPTURED_AT = 1793581200000 // Nov 1 20:00 EST — evening retrospective +export const U1B_CAPTURED_AT = 1793582100000 // Nov 1 20:15 EST — resume, 15 min later +export const DAD_CAPTURED_AT = 1793553300000 // Nov 1 12:15 EST — midday log +export const RUTH_ASK_AT = 1793628000000 // Nov 2 09:00 EST — next-morning catch-up + +// --- u1/u2 event anchors (Nov 1, EST) ----------------------------------- +export const U1_MEAL_AT_8 = 1793538000000 // 08:00 EST +export const U1_NAP_START = 1793554200000 // 12:30 EST +export const U1_NAP_END = 1793559600000 // 14:00 EST (90-minute nap, both EST — wall == elapsed) +export const U1_NAP_MINUTES = 90 +export const U1_MOOD_WINDOW = { from: 1793538000000, to: 1793554200000 } as const // [breakfast, nap) +export const U1_POTTY_WINDOW = { from: 1793559600000, to: 1793574000000 } as const // [nap end, 18:00) — "this afternoon" follows the nap + +// --- u3 event anchors (Nov 1, EST; corpus convention) ------------------- +// "woke up from her nap about an hour ago; that nap was 45 minutes" pins: +// nap end = capturedAt - 1h +// nap start = nap end - 45m +export const U3_NAP_END = 1793549700000 // 11:15 EST == DAD_CAPTURED_AT - 1h +export const U3_NAP_START = 1793547000000 // 10:30 EST == U3_NAP_END - 45m +export const U3_MEAL_WINDOW = { from: 1793547000000, to: 1793553300000 } as const // "lunch around noon" + +// --- windows & cross-anchor --------------------------------------------- +/** End of the local capture day — P3's history window stops here (no Nov 2 bleed). */ +export const HISTORY_WINDOW_NOV1_END = 1793595599999 // Nov 1 23:59:59.999 EST +/** Cross-anchor shared with the merged extraction corpus: Nov 1 18:00 EST. */ +export const NOV1_18_00_EST = 1793574000000 + +// --- machine self-assertions (abort the run on any drift) ---------------- +function assertEqual(actual: number, expected: number, label: string): void { + if (actual !== expected) { + throw new Error(`timestamps.ts: ${label} is ${actual}, expected ${expected}`) + } +} + +/** Offset of `timezone` at `epoch`, in minutes east of UTC (e.g. -300 for EST). */ +function zonedOffsetMinutes(epoch: number, timezone: string): number { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: timezone, + hour12: false, + hourCycle: 'h23', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }).formatToParts(new Date(epoch)) + const get = (t: string): number => Number(parts.find((p) => p.type === t)?.value) + const asUtc = Date.UTC(get('year'), get('month') - 1, get('day'), get('hour'), get('minute')) + return Math.round((asUtc - epoch) / 60000) +} + +// Zone offsets: EST = -300 for every Nov 1 instant (fold is at 02:00, long past). +for (const [label, epoch] of [ + ['MOM_CAPTURED_AT offset', MOM_CAPTURED_AT], + ['U1B_CAPTURED_AT offset', U1B_CAPTURED_AT], + ['DAD_CAPTURED_AT offset', DAD_CAPTURED_AT], + ['RUTH_ASK_AT offset', RUTH_ASK_AT], + ['U1_MEAL_AT_8 offset', U1_MEAL_AT_8], + ['U1_NAP_START offset', U1_NAP_START], + ['U3_NAP_END offset', U3_NAP_END], +] as const) { + assertEqual(zonedOffsetMinutes(epoch, 'America/New_York'), -300, `${label} (must be EST -300)`) +} +// Seed days are EDT = -240. +assertEqual(zonedOffsetMinutes(MINI_HISTORY_WINDOW.from, 'America/New_York'), -240, 'MINI_HISTORY_WINDOW.from offset (must be EDT -240)') + +// Wall-clock mappings (fixed-offset derivation re-check). +assertEqual(U1_MEAL_AT_8, Date.UTC(2026, 10, 1, 8) + 300 * 60000, 'U1_MEAL_AT_8 (08:00 EST)') +assertEqual(U1_NAP_START, Date.UTC(2026, 10, 1, 12, 30) + 300 * 60000, 'U1_NAP_START (12:30 EST)') +assertEqual(U1_NAP_END, Date.UTC(2026, 10, 1, 14) + 300 * 60000, 'U1_NAP_END (14:00 EST)') +assertEqual(MOM_CAPTURED_AT, Date.UTC(2026, 10, 1, 20) + 300 * 60000, 'MOM_CAPTURED_AT (20:00 EST)') +assertEqual(U1B_CAPTURED_AT, Date.UTC(2026, 10, 1, 20, 15) + 300 * 60000, 'U1B_CAPTURED_AT (20:15 EST)') +assertEqual(DAD_CAPTURED_AT, Date.UTC(2026, 10, 1, 12, 15) + 300 * 60000, 'DAD_CAPTURED_AT (12:15 EST)') +assertEqual(RUTH_ASK_AT, Date.UTC(2026, 10, 2, 9) + 300 * 60000, 'RUTH_ASK_AT (Nov 2 09:00 EST)') +assertEqual(MINI_HISTORY_WINDOW.from, Date.UTC(2026, 9, 28) + 240 * 60000, 'MINI_HISTORY_WINDOW.from (Oct 28 00:00 EDT)') +assertEqual(MINI_HISTORY_WINDOW.to, Date.UTC(2026, 10, 1) + 240 * 60000 - 1, 'MINI_HISTORY_WINDOW.to (Oct 31 23:59:59.999 EDT)') +assertEqual(NOV_1_MIDNIGHT_EDT, Date.UTC(2026, 10, 1) + 240 * 60000, 'NOV_1_MIDNIGHT_EDT (Nov 1 00:00 local)') +assertEqual(MONTH_WINDOW_NOV.from, NOV_1_MIDNIGHT_EDT, 'MONTH_WINDOW_NOV.from (Nov 1 00:00 local)') +assertEqual(MONTH_WINDOW_NOV.to, Date.UTC(2026, 11, 1) + 300 * 60000 - 1, 'MONTH_WINDOW_NOV.to (Nov 30 23:59:59.999 EST)') +assertEqual(HISTORY_WINDOW_NOV1_END, Date.UTC(2026, 10, 2) + 300 * 60000 - 1, 'HISTORY_WINDOW_NOV1_END (Nov 1 23:59:59.999 EST)') + +// Durations and the u3 corpus convention chain. +assertEqual(U1_NAP_END - U1_NAP_START, U1_NAP_MINUTES * 60000, 'u1 nap duration (90 min)') +assertEqual(U3_NAP_END, DAD_CAPTURED_AT - 3_600_000, 'U3 "an hour ago" convention') +assertEqual(U3_NAP_START, U3_NAP_END - 45 * 60000, 'u3 nap duration (45 min)') + +// Ordering: narrative causality holds. +function assertTrue(cond: boolean, label: string): void { + if (!cond) throw new Error(`timestamps.ts: ${label}`) +} +assertTrue(U1_MEAL_AT_8 < U1_MOOD_WINDOW.to, 'breakfast before mood window end') +assertTrue(U1_NAP_END < U1_POTTY_WINDOW.to, 'nap before potty window end') +assertEqual(U1_MOOD_WINDOW.to, U1_NAP_START, 'mood window ends at nap start') +assertTrue(U1_POTTY_WINDOW.from >= U1_NAP_END, 'potty window starts at/after nap end') +assertTrue(DAD_CAPTURED_AT < MOM_CAPTURED_AT, 'dad logs before mom') +assertTrue(MOM_CAPTURED_AT < U1B_CAPTURED_AT, 'resume after first capture') +assertEqual(U1B_CAPTURED_AT - MOM_CAPTURED_AT, 15 * 60000, 'resume is 15 minutes later') +assertTrue(RUTH_ASK_AT > U1B_CAPTURED_AT, 'Ruth asks the morning after the captures') +assertTrue(MINI_HISTORY_WINDOW.to < NOV_1_MIDNIGHT_EDT, 'seed window ends before capture day') + +// Cross-anchor with the merged extraction corpus. +assertEqual(NOV1_18_00_EST, Date.UTC(2026, 10, 1, 18) + 300 * 60000, 'NOV1_18_00_EST (Nov 1 18:00 EST)') +if (!deepEqual(U1_MOOD_WINDOW, { from: U1_MEAL_AT_8, to: U1_NAP_START })) { + throw new Error('timestamps.ts: U1_MOOD_WINDOW drift') +} diff --git a/evaluation/src/home/validate.ts b/evaluation/src/home/validate.ts new file mode 100644 index 0000000..f570b24 --- /dev/null +++ b/evaluation/src/home/validate.ts @@ -0,0 +1,253 @@ +/** + * Structural validator for the parent-home journey fixtures. + * `bun src/home/validate.ts` — exit 1 on any failure. + * + * Re-asserts every pinned epoch in the JSON fixtures against timestamps.ts + * (the single source of truth), so a hand-edited fixture can never drift, + * and checks structure: enums, occurrence kinds, references, window bounds, + * and the deliberate fixture properties (one gap day, one private entry, + * one pre-seeded correction chain, empty Nov 1). + */ +import { readFileSync } from 'node:fs' +import * as T from './timestamps.ts' +import { deepEqual } from '../match.ts' + +interface Check { + readonly label: string + readonly ok: boolean + readonly failures: readonly string[] +} + +const CATEGORIES = new Set(['potty', 'meal', 'sleep', 'mood', 'milestone', 'school']) +const OCCURRENCES = new Set(['exactly-one', 'at-least-one']) +const GATES = new Set(['G1', 'G2', 'G3', 'G4', 'G5', 'G6', 'G7']) +const DIMS = new Set(['S1', 'S2', 'S3', 'S4', 'S5', 'S6', 'S7']) +const PROTOCOLS = new Set(['P1', 'P2', 'P3', 'P4', 'P5', 'P6']) + +function loadJson(rel: string): unknown { + return JSON.parse(readFileSync(new URL(rel, import.meta.url), 'utf8')) as unknown +} + +function isRecord(v: unknown): v is Record { + return typeof v === 'object' && v !== null && !Array.isArray(v) +} + +/** Read a numeric field that must equal a pinned timestamp. */ +function epochFailures(rec: Record, field: string, expected: number, label: string): string[] { + const actual = rec[field] + if (typeof actual !== 'number' || actual !== expected) { + return [`${label}.${field} is ${String(actual)}, pinned value is ${String(expected)} (timestamps.ts)`] + } + return [] +} + +function check(label: string, failures: readonly string[]): Check { + return { label, ok: failures.length === 0, failures } +} + +function validateUtterances(u: Record): Check { + const bad: string[] = [] + const captures = u.captures + if (!isRecord(captures)) return check('utterances.captures', ['captures missing']) + + const u1 = captures['u1'] + const u2 = captures['u2'] + const u1b = captures['u1b'] + const u3 = captures['u3'] + for (const [k, v] of [['u1', u1], ['u2', u2], ['u1b', u1b], ['u3', u3]] as const) { + if (!isRecord(v)) bad.push(`capture ${k} missing`) + } + if (!isRecord(u1) || !isRecord(u2) || !isRecord(u1b) || !isRecord(u3)) return check('utterances', bad) + + bad.push(...epochFailures(u1, 'capturedAt', T.MOM_CAPTURED_AT, 'u1')) + bad.push(...epochFailures(u2, 'capturedAt', T.MOM_CAPTURED_AT, 'u2')) + bad.push(...epochFailures(u1b, 'capturedAt', T.U1B_CAPTURED_AT, 'u1b')) + bad.push(...epochFailures(u3, 'capturedAt', T.DAD_CAPTURED_AT, 'u3')) + + const u1Exp = u1['expected'] + if (!isRecord(u1Exp)) { + bad.push('u1.expected missing') + } else { + const events = u1Exp['events'] + if (!Array.isArray(events)) bad.push('u1.expected.events must be an array') + else { + const meal = events.find((e) => isRecord(e) && e['category'] === 'meal') + if (isRecord(meal)) bad.push(...epochFailures(meal, 'occurredAt', T.U1_MEAL_AT_8, 'u1.meal')) + else bad.push('u1 meal expectation missing') + const sleep = events.find((e) => isRecord(e) && e['category'] === 'sleep') + if (isRecord(sleep)) bad.push(...epochFailures(sleep, 'occurredAt', T.U1_NAP_START, 'u1.sleep')) + else bad.push('u1 sleep expectation missing') + const mood = events.find((e) => isRecord(e) && e['category'] === 'mood') + if (isRecord(mood)) { + const w = mood['occurredAtWindow'] + if (!isRecord(w) || !deepEqual(w, T.U1_MOOD_WINDOW)) bad.push(`u1.mood window ${JSON.stringify(mood['occurredAtWindow'] ?? null)} != pinned`) + } else bad.push('u1 mood expectation missing') + const potty = events.find((e) => isRecord(e) && e['category'] === 'potty') + if (isRecord(potty)) { + const w = potty['occurredAtWindow'] + if (!isRecord(w) || !deepEqual(w, T.U1_POTTY_WINDOW)) bad.push(`u1.potty window ${JSON.stringify(potty['occurredAtWindow'] ?? null)} != pinned`) + } else bad.push('u1 potty expectation missing') + for (const e of events) { + if (!isRecord(e)) continue + if (typeof e['category'] !== 'string' || !CATEGORIES.has(e['category'])) bad.push(`unknown category ${JSON.stringify(e['category'] ?? null)}`) + if (typeof e['occurrence'] !== 'string' || !OCCURRENCES.has(e['occurrence'])) bad.push(`unknown occurrence ${JSON.stringify(e['occurrence'] ?? null)}`) + } + } + } + + const u3Exp = u3['expected'] + if (!isRecord(u3Exp)) bad.push('u3.expected missing') + else { + const events = u3Exp['events'] + if (!Array.isArray(events)) bad.push('u3.expected.events must be an array') + else { + const sleep = events.find((e) => isRecord(e) && e['category'] === 'sleep') + if (isRecord(sleep)) { + const w = sleep['occurredAtWindow'] + const pinned = { from: T.U3_NAP_START, to: T.U3_NAP_END } + if (!isRecord(w) || !deepEqual(w, pinned)) bad.push(`u3.sleep window ${JSON.stringify(sleep['occurredAtWindow'] ?? null)} != pinned [${String(T.U3_NAP_START)}, ${String(T.U3_NAP_END)}]`) + } else bad.push('u3 sleep expectation missing') + const meal = events.find((e) => isRecord(e) && e['category'] === 'meal') + if (isRecord(meal)) { + const w = meal['occurredAtWindow'] + if (!isRecord(w) || !deepEqual(w, T.U3_MEAL_WINDOW)) bad.push(`u3.meal window != pinned`) + } else bad.push('u3 meal expectation missing') + } + } + + const u1bStale = u1b['stale'] + if (!isRecord(u1bStale)) bad.push('u1b.stale missing') + else { + if (u1bStale['fragmentAttempt'] !== 1 || u1bStale['resumedAttempt'] !== 2) { + bad.push('u1b stale attempts must be 1 (fragment) and 2 (resumed)') + } + if (u1bStale['expectedWinnerEvents'] !== 0) bad.push('u1b winner must yield zero events (extraction never blocks capture)') + } + + // Parity: u2 must declare parityWith u1 and share semantic content. + if (u2['parityWith'] !== 'u1') bad.push('u2.parityWith must be "u1"') + if (typeof u1['raw'] !== 'string' || typeof u2['raw'] !== 'string') bad.push('u1/u2 raw must be strings') + + return check('utterances', bad) +} + +function validateMiniHistory(m: Record): Check { + const bad: string[] = [] + const window = m['window'] + if (!isRecord(window) || !deepEqual(window, T.MINI_HISTORY_WINDOW)) { + bad.push(`mini-history window ${JSON.stringify(window ?? null)} != pinned MINI_HISTORY_WINDOW`) + } + const entries = m['entries'] + if (!Array.isArray(entries)) return check('mini-history.entries', ['entries must be an array']) + let sawPrivate = false + let sawCorrectionChainEvent = false + for (const entry of entries) { + if (!isRecord(entry)) { + bad.push('entry must be an object') + continue + } + const createdAt = entry['createdAt'] + if (typeof createdAt !== 'number' || createdAt < T.MINI_HISTORY_WINDOW.from || createdAt > T.MINI_HISTORY_WINDOW.to) { + bad.push(`entry ${String(entry['entryId'] ?? '?')} createdAt outside window`) + } + if (entry['audience'] === 'private') { + sawPrivate = true + if (entry['authorId'] !== 'actor-mom') bad.push('private entry must be authored by mom for the visibility control') + } + const events = entry['events'] + if (!Array.isArray(events)) { + bad.push(`entry ${String(entry['entryId'] ?? '?')} events must be an array`) + continue + } + for (const ev of events) { + if (!isRecord(ev)) continue + if (typeof ev['category'] !== 'string' || !CATEGORIES.has(ev['category'])) bad.push(`unknown category ${JSON.stringify(ev['category'] ?? null)}`) + const at = ev['occurredAt'] + if (typeof at !== 'number' || at < T.MINI_HISTORY_WINDOW.from || at >= T.NOV_1_MIDNIGHT_EDT) { + bad.push(`event ${String(ev['eventId'] ?? '?')} outside seed window or on capture day (Nov 1 must start empty)`) + } + if (ev['eventId'] === 'ev-o30-meal') sawCorrectionChainEvent = true + } + } + if (!sawPrivate) bad.push('no private-audience entry (audience check needs one)') + if (!sawCorrectionChainEvent) bad.push('correction-chain target event ev-o30-meal missing') + + const gaps = m['gaps'] + if (!Array.isArray(gaps) || !gaps.some((g) => isRecord(g) && g['day'] === '2026-10-29')) { + bad.push('gap day 2026-10-29 missing (missed-day disclosure needs it)') + } + const corrections = m['corrections'] + if (!Array.isArray(corrections) || corrections.length === 0) { + bad.push('pre-seeded correction chain missing') + } else { + const c0 = corrections[0] + if (!isRecord(c0)) bad.push('correction[0] must be an object') + else if (!isRecord(c0['supersedes']) || !isRecord(c0['patch'])) { + bad.push('correction[0] must carry supersedes (original) and patch — original must stay retrievable (G6)') + } + } + return check('mini-history', bad) +} + +function validateJourneys(j: Record): Check { + const bad: string[] = [] + const journeys = j['journeys'] + if (!Array.isArray(journeys)) return check('journeys.journeys', ['journeys must be an array']) + const actorIds = new Set(['actor-mom', 'actor-dad', 'actor-ruth', 'actor-marcus']) + for (const jr of journeys) { + if (!isRecord(jr)) continue + const id = jr['id'] + if (typeof id !== 'string' || !/^J[1-5]$/.test(id)) bad.push(`journey id ${JSON.stringify(id ?? null)} invalid`) + if (typeof jr['actorId'] !== 'string' || !actorIds.has(jr['actorId'])) bad.push(`journey ${String(id ?? '?')} actorId unknown`) + const steps = jr['steps'] + if (!Array.isArray(steps) || steps.length === 0) bad.push(`journey ${String(id ?? '?')} has no steps`) + else { + let expectedStep = 1 + for (const st of steps) { + if (!isRecord(st)) continue + if (st['step'] !== expectedStep) bad.push(`journey ${String(id ?? '?')} steps not sequential`) + expectedStep += 1 + if (typeof st['action'] !== 'string' || typeof st['expect'] !== 'string') bad.push(`journey ${String(id ?? '?')} step needs action + expect`) + for (const key of ['protocols', 'gates', 'dims'] as const) { + const arr = st[key] + if (!Array.isArray(arr)) continue + const known = key === 'protocols' ? PROTOCOLS : key === 'gates' ? GATES : DIMS + for (const v of arr) { + if (typeof v !== 'string' || !known.has(v)) bad.push(`journey ${String(id ?? '?')} step ${String(expectedStep - 1)}: unknown ${key} entry ${JSON.stringify(v ?? null)}`) + } + } + } + } + const evidence = jr['evidence'] + if (!Array.isArray(evidence) || evidence.length === 0) bad.push(`journey ${String(id ?? '?')} must name its evidence`) + } + if (journeys.length !== 5) bad.push(`expected 5 journeys, got ${String(journeys.length)}`) + return check('journeys', bad) +} + +function main(): number { + const checks: Check[] = [] + const utt = loadJson('../../fixtures/home/utterances.json') + if (isRecord(utt)) checks.push(validateUtterances(utt)) + else checks.push(check('utterances', ['not an object'])) + + const mini = loadJson('../../fixtures/home/mini-history.json') + if (isRecord(mini)) checks.push(validateMiniHistory(mini)) + else checks.push(check('mini-history', ['not an object'])) + + const journeys = loadJson('../../fixtures/home/journeys.json') + if (isRecord(journeys)) checks.push(validateJourneys(journeys)) + else checks.push(check('journeys', ['not an object'])) + + console.log('\nParent-home fixture validation (timestamps.ts assertions ran at import)') + let failed = 0 + for (const c of checks) { + console.log(` ${c.ok ? 'PASS' : 'FAIL'} ${c.label}`) + for (const f of c.failures) console.log(` - ${f}`) + if (!c.ok) failed += 1 + } + console.log('='.repeat(72)) + return failed === 0 ? 0 : 1 +} + +process.exitCode = main()