From 414bbfd3daae32713ef435a09d59df029fbfa278 Mon Sep 17 00:00:00 2001 From: Franz Hemmer <8227352+HemSoft@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:26:47 -0400 Subject: [PATCH 1/4] Give an exported provider session a bounded shutdown grace (#264) PollingController.Close now signals a private shutdown drain carried in the active invocation's context. A native launch holds that drain just before it exports auth, and no launch exports once the drain has begun. Only when a launch holds an export does the first Close wait, at most ten seconds (providerShutdownGrace) or its own earlier deadline, for the invocation to end before cancelling it as before. Every other phase is still cancelled at once, and a retried Close gets no second grace. During the grace the held launch closes its provider bridge, which closes the private provider stream and any in-flight request. The guest's existing client proxy stops its CLI on the failed exchange, and guest PID1 returns auth and the source audit in its ordinary result record. The host still proves whole-unit stop before any auth step and commits auth only through the durable stopped return. A failure after the drain reports cancelled, unless the guard latched a refusal or exhaustion, so the stopped job is requeued with its consumed budget. The engine starts no further phase once the drain began. A guest that does not return in time takes the existing stop-pending, fenced path. PollingController.Run's deferred Close budget becomes the grace plus its earlier ten seconds, inside GitReviewerService.Close's 30-second ceiling. The E2E close-provider branch where Close proves completion now expects the requeued job to rerun three phases and expire its captured budget, since the interrupted session stays charged, instead of a published review. Recovery of a lease whose stopped receipt is missing is unchanged and stays fenced pending the owner decision in #264. Co-Authored-By: Claude Opus 5.5 --- reviewer/e2e_qualification_linux_test.go | 184 +++++++--- reviewer/engine.go | 4 + reviewer/native_vm_provider.go | 4 + reviewer/native_vm_reviewer_linux.go | 62 ++++ .../native_vm_shutdown_grace_linux_test.go | 237 +++++++++++++ reviewer/polling.go | 29 +- reviewer/shutdown_grace.go | 95 ++++++ reviewer/shutdown_grace_test.go | 317 ++++++++++++++++++ 8 files changed, 870 insertions(+), 62 deletions(-) create mode 100644 reviewer/native_vm_shutdown_grace_linux_test.go create mode 100644 reviewer/shutdown_grace.go create mode 100644 reviewer/shutdown_grace_test.go diff --git a/reviewer/e2e_qualification_linux_test.go b/reviewer/e2e_qualification_linux_test.go index a32f269d..9811e56a 100644 --- a/reviewer/e2e_qualification_linux_test.go +++ b/reviewer/e2e_qualification_linux_test.go @@ -3217,13 +3217,37 @@ func e2eNativeSessionVerdict(units []e2eNativeUnit, jobs []*r.DurableJob, extra } // e2eProviderReopenVerdicts judges close-provider after a Close that proved -// completion and a reopen with the same assertions as every other completed -// review: one confirmed intent, one POST and the expected remote COMMENT, -// stopped source receipts, and one completed native launch per artifact. The -// interrupted launch is the only other launch allowed. +// completion (the shutdown grace returned the interrupted launch's auth) and +// a reopen. The requeued job keeps that launch's charged session, so its +// captured four-session budget leaves three: it reruns Codex R1, Claude R1 +// and Codex R2, each in one completed launch holding the canned report, then +// expires before Claude R2, unleased, with no intent, POST or remote review. +// Its source receipts must still be stopped, and the interrupted launch, +// bound to its recorded attempt, is the only other launch allowed. It returns +// PR 1's job, or nil when it is not the only job. func e2eProviderReopenVerdicts(c e2eCompletion, interrupted string) ([]e2eVerdict, *r.DurableJob) { - verdicts, jobs := e2eCompletedVerdicts("after reopen", c, []int{1}, interrupted) - return verdicts, jobs[0] + const label = "after reopen" + j := e2eJobFor(c.State, 1) + if j == nil || len(c.State.Jobs) != 1 { + return []e2eVerdict{{false, fmt.Sprintf("%s: PR 1's job is the only job (%d jobs)", label, len(c.State.Jobs))}}, nil + } + expected := e2eExpectedFor(c, 1, j) + phases := []string{} + for _, a := range j.Checkpoint.Artifacts { + phases = append(phases, fmt.Sprintf("%s R%d", a.Report.Provider, a.Report.Round)) + } + canned := e2eCannedArtifacts(expected) + v := []e2eVerdict{ + {r.Digest(j.Job) == r.Digest(expected.Job), fmt.Sprintf("%s: PR 1's job is exactly the one admission derives from its retained request and the harness's inputs", label)}, + {j.State == "incomplete_expired" && j.LeaseToken == "" && !j.Outstanding, fmt.Sprintf("%s: the requeued job expired its captured budget, unleased and with no outstanding reservation (state %s)", label, j.State)}, + {j.Checkpoint.ConsumedSessions == c.Config.MaxSessions() && j.Checkpoint.JobHash == r.Digest(j.Job), fmt.Sprintf("%s: the job consumed exactly its %d captured sessions, the interrupted one included (consumed %d)", label, c.Config.MaxSessions(), j.Checkpoint.ConsumedSessions)}, + {slices.Equal(phases, []string{"codex R1", "claude R1", "codex R2"}), fmt.Sprintf("%s: the job stored exactly Codex R1, Claude R1 and Codex R2 (%v)", label, phases)}, + {canned == nil, fmt.Sprintf("%s: the stored artifacts hold exactly the canned backend's reports for their phases: %v", label, canned)}, + {len(c.State.Writes) == 0, fmt.Sprintf("%s: the expired job prepared no write intent (%d)", label, len(c.State.Writes))}, + {c.Posts[1] == 0 && len(c.Reviews[1]) == 0, fmt.Sprintf("%s: PR 1 received no review POST and holds no remote review (%d POSTs, %d reviews)", label, c.Posts[1], len(c.Reviews[1]))}, + } + v = append(v, e2eSourceVerdicts(label, c, []*r.DurableJob{expected})...) + return append(v, e2eAttemptSetVerdict(c.Units, []*r.DurableJob{expected}, interrupted), e2eNativeSessionVerdict(c.Units, []*r.DurableJob{expected}, interrupted)), j } // e2eCompletedVerdicts judges a completed case's evidence for the reviewed @@ -3468,10 +3492,12 @@ func e2eCloseCapture(e *e2eEnv) { // close-provider: the service closes while the first native session's pinned // client waits for a withheld fake inference response. The unit must be -// proved stopped and the exported auth returned or fenced. If the attempt -// stays uncertain, its durable lease is retained and Close keeps reporting -// ErrWorkerStopPending; the root is then retained for close-provider-reopen, -// which a separate process runs as a restarted service. +// proved stopped and the exported auth returned or fenced. Within the +// shutdown grace the expected outcome is a returned auth and a Close that +// proves completion; the case then reopens and judges the requeued job. If +// the attempt stays uncertain, its durable lease is retained and Close keeps +// reporting ErrWorkerStopPending; the root is then retained for +// close-provider-reopen, which a separate process runs as a restarted service. func e2eCloseProvider(e *e2eEnv) { e.github.addPR(1, r.OwnerID, e.github.epoch.Add(time.Minute)) e.models[r.Codex].mu.Lock() @@ -3544,15 +3570,28 @@ func e2eCloseProvider(e *e2eEnv) { e.check((returned && auth[r.Codex] == "usable") || (!returned && auth[r.Codex] == "fenced"), "codex auth is usable exactly when its stopped return was committed, and fenced otherwise (returned %v, %s)", returned, auth[r.Codex]) e.check(e.github.reviewPosts(1) == 0, "nothing was published for the interrupted job") if !e.wedged { - e.check(j.LeaseToken == "" && j.State == "queued" && returned, "a completed Close leaves a queued job with returned auth (state %s)", j.State) - // Not yet observed on a host: the stopped receipt the job records for - // the interrupted attempt is the launch's audit evidence. + // The expected branch since the shutdown grace (#264): Close revoked + // the withheld session's provider intake, the guest stopped its client + // and returned auth, and the stopped launch was released and requeued + // with its charged session. Not yet observed on a host. + e.check(j.LeaseToken == "" && !j.Outstanding && j.State == "queued" && returned, "a completed Close leaves a queued, unleased job with no outstanding reservation and returned auth (state %s)", j.State) + failure := "" + if len(units) == 1 { + failure = units[0].Failure + } + e.check(len(units) == 1 && units[0].AuthExported && units[0].AuthReturned && failure == "cancelled", "the stopped audit records the auth as exported and returned and the session as ended by shutdown (failure %q)", failure) if len(units) == 1 { - _, err := e2eRecordedEvidence(expected, units[0]) - e.check(err == nil, "the job durably records the interrupted launch's complete stopped audit evidence: %v", err) + _, err := e2eInterruptedLaunch(units[0], expected) + e.check(err == nil, "the job durably records the interrupted launch as its attempt, with its complete stopped audit evidence and the source receipt its result returned: %v", err) } e.open("reopen") - e.wait("PR 1 review after reopen", 6*time.Minute, func(d r.DurableState) bool { return e2eConfirmed(d, 1) }) + // The requeued job keeps the interrupted session's charge, so its + // captured four-session budget cannot finish the adversarial + // protocol: it reruns three phases and expires before Claude R2. + e.wait("PR 1 job settles after reopen", 6*time.Minute, func(d r.DurableState) bool { + j := e2eJobFor(d, 1) + return j != nil && j.LeaseToken == "" && !j.Outstanding && j.State != "queued" && j.State != "running" + }) e.snapshotJobs("after reopen") // The one allowed extra launch is the Codex R1 session the expected // job derives, not whichever session the interrupted launch recorded. @@ -3561,13 +3600,16 @@ func e2eCloseProvider(e *e2eEnv) { interrupted = r.Digest(session) } e.check(interrupted != "" && len(units) == 1 && units[0].SessionHash == interrupted, "the interrupted launch ran the Codex R1 session the expected job derives") - verdicts, reopened := e2eProviderReopenVerdicts(e.completion(), interrupted) + c := e.completion() + c.Heads[1] = e.github.commits["1"] + verdicts, reopened := e2eProviderReopenVerdicts(c, interrupted) e.checkAll(verdicts) - // Not yet observed on a host: the interrupted launch plus one completed - // launch per adversarial phase, restated rather than counted from the - // reopened job's artifacts, still one acquisition, both auth stores - // usable, and the review published. - e.expect = e2eExpect{SourceUnits: 1, NativeUnits: 1 + len(e2eAdversarialRounds), AuthUsable: true, Routes: e2ePublishedRoutes} + e.result.Requests = e2eRequests(c.State) + e.checkAll(e2eRequestVerdicts("after reopen", e.result.Requests, map[int]e2eRequest{1: e2eAdmittedOpened(1, j.RequestID)})) + // The interrupted launch plus the three rerun phases, restated rather + // than counted from the reopened job's artifacts, still one + // acquisition, both auth stores usable, and nothing published. + e.expect = e2eExpect{SourceUnits: 1, NativeUnits: 1 + 3, AuthUsable: true, Routes: e2eQualifiedRoutes} if reopened != nil { e.result.Jobs = append(e.result.Jobs, e2eJobSummary(reopened)) } @@ -5203,6 +5245,9 @@ func e2eLaunchFixture(j *r.DurableJob, session r.Session, launch, attempt string if returned { audit.Failure = "cancelled" } + if access != nil && !complete { + audit.Stage = "result-validation" + } if complete { audit.Stage, audit.Failure = "result-validation", "" if provider == r.Codex { @@ -5306,21 +5351,33 @@ func e2eAcquisition(rec r.E2ESourceRecord) []e2eGitExchange { } } -// e2eReopenFixture is a valid completion after close-provider's reopen: the -// completed review plus the interrupted launch of the first session, whose -// version 2 receipt names the attempt under which the job also records its -// stopped evidence, and its held backend check. The interrupted launch is the -// last unit. The interrupted phase returned its auth, read no result and so recorded no -// report and no source receipt. +// e2eReopenFixture is a valid end state of close-provider's reopen after a +// Close that proved completion: PR 1's only job, requeued with the +// interrupted launch's charged session, reran Codex R1, Claude R1 and Codex +// R2 and expired before Claude R2, with no intent, POST or remote review. The +// interrupted launch of the first session is the last unit. Its version 2 +// receipt names the attempt under which the job records its stopped evidence +// and source receipt; it returned its auth in the result record, so its +// audit ends at that record's validation and reports the shutdown. func e2eReopenFixture() (e2eCompletion, string) { c := e2eCompletedFixture() job := e2eJobFor(c.State, 1) + job.Checkpoint.Artifacts = job.Checkpoint.Artifacts[:3] + job.ProviderAttempts, job.SourceAttempts = job.ProviderAttempts[:3], job.SourceAttempts[:3] + last := job.Checkpoint.Artifacts[2] + job.AttemptProvider, job.AttemptRound, job.AttemptID, job.AttemptInputHash = r.Codex, 2, job.ProviderAttempts[2].ID, last.InputHash + job.State, job.Checkpoint.ConsumedSessions = "incomplete_expired", 4 + c.Units, c.Sessions = c.Units[:3], c.Sessions[:3] + c.State.Writes, c.Posts, c.Reviews = map[string]*r.WriteIntent{}, map[int]int{}, map[int][]map[string]any{} session := e2eFixtureSession(job, r.Codex, 1) first := r.Digest(session) evidence := e2eFixtureEvidence(session, false) interrupted := strings.Repeat("b", 32) + access := &r.SourceAccess{Version: 1, SessionHash: first, Reads: []r.ReadRecord{}, Coverage: []r.Coverage{}} + recorded := *access job.ProviderAttempts = append([]r.ProviderAttempt{{ID: interrupted, Evidence: evidence}}, job.ProviderAttempts...) - c.Units = append(c.Units, e2eLaunchFixture(job, session, strings.Repeat("9", 32), interrupted, evidence, r.Report{}, nil, false, true)) + job.SourceAttempts = append([]r.SourceAttempt{{ID: interrupted, Access: &recorded}}, job.SourceAttempts...) + c.Units = append(c.Units, e2eLaunchFixture(job, session, strings.Repeat("9", 32), interrupted, evidence, r.Report{}, access, false, true)) c.Sessions = append([]e2eModelSession{{JobID: job.Job.ID, Provider: r.Codex, Round: 1, PR: 1, Head: job.Job.Snapshot.Head, SessionHash: first, HeldInference: true}}, c.Sessions...) return c, first } @@ -5342,55 +5399,72 @@ func TestE2EProviderReopenVerdicts(t *testing.T) { return e2eFailures(verdicts) } if f := failures(e2eReopenFixture()); len(f) != 0 { - t.Fatal("completed reopen refused:", f) + t.Fatal("budget-expired reopen refused:", f) } + const last = 3 // the interrupted launch for name, edit := range map[string]func(*e2eCompletion){ - "duplicate unknown intent": func(c *e2eCompletion) { + "review intent prepared": func(c *e2eCompletion) { c.State.Writes["w2"] = &r.WriteIntent{ID: "w2", Kind: "review", Scope: e2eScope(1), State: "unknown"} }, - "second review POST": func(c *e2eCompletion) { c.Posts[1] = 2 }, - "remote review without inline": func(c *e2eCompletion) { c.Reviews[1][0]["inline"] = []string{} }, - "remote review on another head": func(c *e2eCompletion) { c.Reviews[1][0]["commit_id"] = strings.Repeat("9", 40) }, + "review POST": func(c *e2eCompletion) { c.Posts[1] = 1 }, + "remote review": func(c *e2eCompletion) { c.Reviews[1] = []map[string]any{{"id": int64(5001)}} }, + "job completed": func(c *e2eCompletion) { e2eJobFor(c.State, 1).State = "dual_complete" }, + "job requeued again": func(c *e2eCompletion) { e2eJobFor(c.State, 1).State = "queued" }, + "lease retained": func(c *e2eCompletion) { e2eJobFor(c.State, 1).LeaseToken = "lease" }, + "reservation outstanding": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Outstanding = true }, + "interrupted refunded": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.ConsumedSessions = 3 }, + "session beyond budget": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.ConsumedSessions = 5 }, + "missing Codex R2": func(c *e2eCompletion) { + j := e2eJobFor(c.State, 1) + j.Checkpoint.Artifacts = j.Checkpoint.Artifacts[:2] + }, + "second job": func(c *e2eCompletion) { c.State.Jobs["other"] = e2eJobFor(c.State, 1) }, + "artifact not canned": func(c *e2eCompletion) { + e2eJobFor(c.State, 1).Checkpoint.Artifacts[1].Report.Retractions = []string{"changed"} + }, "completed launch kept its auth": func(c *e2eCompletion) { c.Units[1].AuthReturned = false }, "completed launch audit unbound": func(c *e2eCompletion) { c.Units[2].AuditPresent = false }, "completed launch names an unknown attempt": func(c *e2eCompletion) { c.Units[2].AttemptID = strings.Repeat("f", 32) }, "completed launch names the interrupted attempt": func(c *e2eCompletion) { - c.Units[0].AttemptID, c.Units[0].EvidenceHash = c.Units[4].AttemptID, c.Units[4].EvidenceHash + c.Units[0].AttemptID, c.Units[0].EvidenceHash = c.Units[last].AttemptID, c.Units[last].EvidenceHash }, "completed launch names another session's attempt": func(c *e2eCompletion) { c.Units[1].AttemptID, c.Units[1].EvidenceHash = c.Units[2].AttemptID, c.Units[2].EvidenceHash }, - "completed launch repeats other evidence": func(c *e2eCompletion) { c.Units[3].EvidenceHash = c.Units[4].EvidenceHash }, + "completed launch repeats other evidence": func(c *e2eCompletion) { c.Units[2].EvidenceHash = c.Units[last].EvidenceHash }, "artifact attempt unrecorded": func(c *e2eCompletion) { j := e2eJobFor(c.State, 1) j.ProviderAttempts = slices.Delete(j.ProviderAttempts, 2, 3) }, "artifact without provider evidence": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.Artifacts[1].ProviderEvidence = nil }, - "second interrupted launch": func(c *e2eCompletion) { c.Units = append(c.Units, c.Units[4]) }, - "source attempt not stopped": func(c *e2eCompletion) { c.Records[0].Stopped = false }, - "source records unreadable": func(c *e2eCompletion) { c.SourceErr = r.ErrLiveBlocked }, - "interrupted session recharged": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.ConsumedSessions = 5 }, + "second interrupted launch": func(c *e2eCompletion) { c.Units = append(c.Units, c.Units[last]) }, + "launch for Claude R2": func(c *e2eCompletion) { + full := e2eCompletedFixture() + c.Units = append(c.Units, full.Units[3]) + }, + "source attempt not stopped": func(c *e2eCompletion) { c.Records[0].Stopped = false }, + "source records unreadable": func(c *e2eCompletion) { c.SourceErr = r.ErrLiveBlocked }, "interrupted launch completed": func(c *e2eCompletion) { // A completed launch with a foreign receipt must not pass as the // interrupted one. - c.Units[4].Complete = true + c.Units[last].Complete = true }, - "interrupted launch without its audit": func(c *e2eCompletion) { c.Units[4].AuditPresent = false }, + "interrupted launch without its audit": func(c *e2eCompletion) { c.Units[last].AuditPresent = false }, "interrupted launch names a completed attempt": func(c *e2eCompletion) { - c.Units[4].AttemptID, c.Units[4].EvidenceHash = c.Units[1].AttemptID, c.Units[1].EvidenceHash + c.Units[last].AttemptID, c.Units[last].EvidenceHash = c.Units[1].AttemptID, c.Units[1].EvidenceHash }, - "interrupted launch names an unknown attempt": func(c *e2eCompletion) { c.Units[4].AttemptID = strings.Repeat("f", 32) }, - "interrupted launch repeats other evidence": func(c *e2eCompletion) { c.Units[4].EvidenceHash = c.Units[1].EvidenceHash }, + "interrupted launch names an unknown attempt": func(c *e2eCompletion) { c.Units[last].AttemptID = strings.Repeat("f", 32) }, + "interrupted launch repeats other evidence": func(c *e2eCompletion) { c.Units[last].EvidenceHash = c.Units[1].EvidenceHash }, "interrupted attempt recorded twice": func(c *e2eCompletion) { j := e2eJobFor(c.State, 1) j.ProviderAttempts = append(j.ProviderAttempts, j.ProviderAttempts[0]) }, - // Round 10: the interrupted launch is the job's one interrupted - // attempt, and its audit records that attempt's request, no report - // and exactly the source receipt the job recorded for it (none). "interrupted attempt never recorded": func(c *e2eCompletion) { e2eJobFor(c.State, 1).ProviderAttempts = e2eJobFor(c.State, 1).ProviderAttempts[1:] }, + "interrupted source receipt never recorded": func(c *e2eCompletion) { + e2eJobFor(c.State, 1).SourceAttempts = e2eJobFor(c.State, 1).SourceAttempts[1:] + }, "second interrupted provider attempt": func(c *e2eCompletion) { j := e2eJobFor(c.State, 1) other := *j.ProviderAttempts[0].Evidence @@ -5398,15 +5472,15 @@ func TestE2EProviderReopenVerdicts(t *testing.T) { j.ProviderAttempts = append(j.ProviderAttempts, r.ProviderAttempt{ID: "interrupted-again", Evidence: &other}) }, "interrupted audit of a report": func(c *e2eCompletion) { - c.Units[len(c.Units)-1].Audit.ReportHash = r.Digest(e2eJobFor(c.State, 1).Checkpoint.Artifacts[0].Report) + c.Units[last].Audit.ReportHash = r.Digest(e2eJobFor(c.State, 1).Checkpoint.Artifacts[0].Report) }, - "interrupted audit of an unrecorded source receipt": func(c *e2eCompletion) { - c.Units[len(c.Units)-1].Audit.SourceHash = r.Digest(e2eJobFor(c.State, 1).Checkpoint.Artifacts[0].SourceAccess) + "interrupted audit of another source receipt": func(c *e2eCompletion) { + c.Units[last].Audit.SourceHash = r.Digest(e2eJobFor(c.State, 1).Checkpoint.Artifacts[0].SourceAccess) }, - "interrupted audit of another request": func(c *e2eCompletion) { c.Units[len(c.Units)-1].Audit.RequestHash = r.Digest("another request") }, - "interrupted audit at another stage": func(c *e2eCompletion) { c.Units[len(c.Units)-1].Audit.Stage = "auth-export" }, - "interrupted audit without a failure": func(c *e2eCompletion) { c.Units[len(c.Units)-1].Audit.Failure = "" }, - "interrupted unit never aborted": func(c *e2eCompletion) { c.Units[len(c.Units)-1].Aborted = false }, + "interrupted audit of another request": func(c *e2eCompletion) { c.Units[last].Audit.RequestHash = r.Digest("another request") }, + "interrupted audit before its result": func(c *e2eCompletion) { c.Units[last].Audit.Stage = "result" }, + "interrupted audit without a failure": func(c *e2eCompletion) { c.Units[last].Audit.Failure = "" }, + "interrupted unit never aborted": func(c *e2eCompletion) { c.Units[last].Aborted = false }, } { c, interrupted := e2eReopenFixture() edit(&c) diff --git a/reviewer/engine.go b/reviewer/engine.go index ea2c7ccb..268c8b96 100644 --- a/reviewer/engine.go +++ b/reviewer/engine.go @@ -223,6 +223,10 @@ func (e Engine) Run(ctx context.Context, job Job, checkpoint Checkpoint) (Outcom if err := ctx.Err(); err != nil { return fail("cancelled", err) } + // A shutdown drain lets only the in-flight phase finish. + if shutdownDraining(ctx) { + return fail("cancelled", context.Canceled) + } // Serialization gives each invocation its own frozen peer data. peerBytes, _ := json.Marshal(peers) var frozen []Report diff --git a/reviewer/native_vm_provider.go b/reviewer/native_vm_provider.go index a57e72f9..9a9963d8 100644 --- a/reviewer/native_vm_provider.go +++ b/reviewer/native_vm_provider.go @@ -145,6 +145,10 @@ func startNativeProviderBridge(ctx context.Context, s Session, policy EgressPoli return bridge, nil } +// revoke ends provider intake early: it closes the private stream and any +// in-flight request. It attests nothing; close remains the only receipt. +func (b *nativeProviderBridge) revoke() { b.server.Close() } + // close waits for both HTTP intake and all guarded upstream/body work. This is // an HTTP receipt, never a substitute for the native unit's whole-VM stop proof. func (b *nativeProviderBridge) close(ctx context.Context) (*ProviderEvidence, error) { diff --git a/reviewer/native_vm_reviewer_linux.go b/reviewer/native_vm_reviewer_linux.go index 6d0f688b..109df5bd 100644 --- a/reviewer/native_vm_reviewer_linux.go +++ b/reviewer/native_vm_reviewer_linux.go @@ -110,6 +110,11 @@ type nativeVMHostPhase struct { // attempt is the durable attempt ID bound into the stopped receipt, or // empty outside a durable attempt. attempt string + // drain is the controller invocation's shutdown drain this launch holds + // from auth export until its stopped receipt; see holdShutdownGrace. + drain *shutdownDrain + releaseGrace func() + stopGrace func() bool } var _ attemptAuditedReviewer = (*NativeVMReviewer)(nil) @@ -249,6 +254,10 @@ func (r *NativeVMReviewer) reviewTaskAudited(ctx context.Context, s Session, con if nativeCheckLeasedAuth(ctx, phase.profile, nativeVMExportGate{supervisor, manifest, request, ready}) != nil { return nativeVMTaskReport{}, nil, nil, PhaseError{State: "auth_failed", BeforeWork: true} } + // Once service shutdown drains this invocation no auth is exported. + if !phase.holdShutdownGrace(ctx) { + return nativeVMTaskReport{}, nil, nil, workerLauncherError(context.Canceled, true) + } phase.stage = "auth-export" seed, err := phase.profile.ExportAuth(ctx, nativeVMExportGate{supervisor, manifest, request, ready}) if err != nil { @@ -287,6 +296,11 @@ func (p *nativeVMHostPhase) finish(ctx context.Context, report Report, primary e return result.review, access, evidence, err } func (p *nativeVMHostPhase) finishTask(ctx context.Context, report nativeVMTaskReport, primary error) (nativeVMTaskReport, *SourceAccess, *ProviderEvidence, error) { + // The grace hold lasts until the stopped receipt below is written. + if p.releaseGrace != nil { + defer p.releaseGrace() + } + drained := p.shutdownDrained() if p.profile != nil { defer p.profile.Close() } @@ -395,6 +409,13 @@ func (p *nativeVMHostPhase) finishTask(ctx context.Context, report nativeVMTaskR } } + // A failure after shutdown revoked provider intake reports the shutdown, + // so the controller may requeue the stopped job under its consumed + // budget. A latched route or response refusal, or exhaustion, keeps its + // own classification. + if primary != nil && drained && !nativeProviderRefused(evidence) { + primary = errors.Join(primary, context.Canceled) + } if p.started && p.persistAudit(cleanup, report.value(), access, evidence, primary) != nil { primary = errors.Join(primary, ErrProviderAuditPending) } @@ -410,6 +431,47 @@ func (p *nativeVMHostPhase) finishTask(ctx context.Context, report nativeVMTaskR return report, access, evidence, nil } +// holdShutdownGrace registers this launch with its controller invocation's +// shutdown drain before auth export, and reports false once that drain has +// begun. While the hold lasts, a drain revokes provider intake: the private +// provider stream and any in-flight request close, the guest's client proxy +// stops its CLI on the failed exchange, and guest PID1 returns auth in its +// ordinary finished record. Nothing else changes. The host still reads that +// record only under the session deadline and Close's later cancellation, +// proves whole-unit stop first, and commits auth only through its stopped +// return. Launches outside a controller invocation hold nothing. +func (p *nativeVMHostPhase) holdShutdownGrace(ctx context.Context) bool { + drain := shutdownDrainOf(ctx) + if drain == nil { + return true + } + release, ok := drain.hold() + if !ok { + return false + } + p.drain, p.releaseGrace = drain, release + bridge := p.bridge + p.stopGrace = context.AfterFunc(drain.signal, func() { + if bridge != nil { + bridge.revoke() + } + }) + return !drain.draining() +} + +// shutdownDrained ends the revocation hook and reports whether shutdown had +// begun draining this held launch. +func (p *nativeVMHostPhase) shutdownDrained() bool { + if p.stopGrace != nil { + p.stopGrace() + } + return p.releaseGrace != nil && p.drain.draining() +} + +func nativeProviderRefused(evidence *ProviderEvidence) bool { + return evidence != nil && (evidence.Access.RouteDenials > 0 || evidence.Access.ResponseDenials > 0 || evidence.Access.Exhausted) +} + // A stopped CLI can return its old file after the provider already rotated it. // Keep the exported lease pending if any refresh lacks a complete, nonfailed // provider receipt, or if a refresh did not produce changed local auth. A valid diff --git a/reviewer/native_vm_shutdown_grace_linux_test.go b/reviewer/native_vm_shutdown_grace_linux_test.go new file mode 100644 index 00000000..92811655 --- /dev/null +++ b/reviewer/native_vm_shutdown_grace_linux_test.go @@ -0,0 +1,237 @@ +//go:build linux + +package reviewer + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +// A drained, held launch revokes provider intake while the pinned client +// waits for a withheld inference response. The guest's existing client proxy +// sees its private exchange fail and stops its CLI, which is what lets guest +// PID1 return auth. The host receipt records the aborted request as failed. +func TestNativeVMShutdownDrainRevokesWithheldInferenceAndStopsGuestClient(t *testing.T) { + p, _ := nativeHostPhaseFixture(t) + guest, host := vmSocketPair(t) + reached, released := make(chan struct{}), make(chan error, 1) + var err error + p.bridge, err = startNativeProviderBridge(context.Background(), p.request.Session, p.request.Policy, egressRoundTrip(func(q *http.Request) (*http.Response, error) { + close(reached) + <-q.Context().Done() + released <- q.Context().Err() + return nil, q.Context().Err() + }), host) + if err != nil { + t.Fatal(err) + } + drain := newShutdownDrain() + if !p.holdShutdownGrace(withShutdownDrain(context.Background(), drain)) { + t.Fatal("an undrained launch was refused its export") + } + transport := nativeProviderTransport(guest) + defer transport.CloseIdleConnections() + var class atomic.Value + proxy := httptest.NewServer(nativeGuestProviderHandlerObserved(p.request, transport, func(c string) { class.Store(c) })) + // Cleanups run last first: a failed run closes the private stream before + // the proxy waits for its handler. + t.Cleanup(proxy.Close) + t.Cleanup(func() { guest.Close(); host.Close() }) + raw, _ := json.Marshal(map[string]any{"model": p.request.Session.Identity.Model}) + q, _ := http.NewRequest("POST", proxy.URL+"/infer", strings.NewReader(string(raw))) + q.Header.Set("Authorization", "Bearer synthetic-native-access") + q.Header.Set("Content-Type", "application/json") + answered := make(chan int, 1) + go func() { + r, err := proxy.Client().Do(q) + if err != nil { + answered <- 0 + return + } + io.Copy(io.Discard, r.Body) + r.Body.Close() + answered <- r.StatusCode + }() + select { + case <-reached: + case <-time.After(5 * time.Second): + t.Fatal("inference never reached the withheld upstream") + } + if !drain.begin() { + t.Fatal("a held export was not granted the grace") + } + select { + case status := <-answered: + if status != http.StatusBadGateway || class.Load() != "provider-transport" { + t.Fatal("guest proxy did not stop its client on revoked intake", status, class.Load()) + } + case <-time.After(5 * time.Second): + t.Fatal("revoked intake left the guest's request waiting") + } + select { + case err := <-released: + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("withheld upstream request survived revocation") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + e, err := p.bridge.close(ctx) + if err != nil || !e.Access.Stopped || !e.Access.Failed || e.Access.Forwarded != 1 || e.Access.Responses != 0 || nativeProviderRefused(e) { + t.Fatal("revoked intake did not leave a stopped failed receipt", e, err) + } + if !p.shutdownDrained() { + t.Fatal("held launch lost its drain") + } + p.releaseGrace() + if drain.held != 0 { + t.Fatal("hold not released") + } + // A launch reaching export after the drain began exports nothing. + late, _ := nativeHostPhaseFixture(t) + if late.holdShutdownGrace(withShutdownDrain(context.Background(), drain)) || late.releaseGrace != nil { + t.Fatal("a launch exported auth after shutdown began") + } + // Outside a controller invocation nothing is held or drained. + free, _ := nativeHostPhaseFixture(t) + if !free.holdShutdownGrace(context.Background()) || free.shutdownDrained() { + t.Fatal("an evaluation launch was tied to a controller drain") + } +} + +func nativeDrainedPhase(t *testing.T, drain bool) (*nativeVMHostPhase, *AuthStore, *shutdownDrain, AuthTransferInfo) { + t.Helper() + p, _ := nativeHostPhaseFixture(t) + store := authTestStore(t, Codex) + profile, err := store.LeaseProfile(context.Background(), p.request.Binding) + if err != nil { + t.Fatal(err) + } + p.profile = profile + attachIdleNativeProviderBridge(t, p) + var d *shutdownDrain + if drain { + d = newShutdownDrain() + if !p.holdShutdownGrace(withShutdownDrain(context.Background(), d)) { + t.Fatal("hold refused") + } + } + seed, err := profile.ExportAuth(context.Background(), authExportCheck(func(context.Context, AuthBinding) error { return nil })) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { seed.Close() }) + p.exported = true + if d != nil && !d.begin() { + t.Fatal("held export denied the grace") + } + info, err := seed.Info() + if err != nil { + t.Fatal(err) + } + return p, store, d, info +} + +func nativeReturnedFinished(t *testing.T, p *nativeVMHostPhase, info AuthTransferInfo, payload string) { + t.Helper() + returned, err := NewAuthReturn(info, []byte(payload)) + if err != nil { + t.Fatal(err) + } + p.finished = &nativeVMFinished{Version: 1, Binding: p.request.Binding, RequestHash: Digest(p.request), State: "provider_unavailable", AuthReturn: authWire(t, returned), SourceAudit: nativeHostAuditFixture(t, p.request)} +} + +type nativeDrainReceipt struct { + Complete, AuthExported, AuthReturned bool + Failure string +} + +func readNativeDrainReceipt(t *testing.T, p *nativeVMHostPhase) nativeDrainReceipt { + t.Helper() + raw, err := os.ReadFile(filepath.Join(p.supervisor.supervisor.root, p.request.Binding.LaunchID, "native-audit.json")) + if err != nil { + t.Fatal("stopped receipt missing", err) + } + var r nativeDrainReceipt + if err = json.Unmarshal(raw, &r); err != nil { + t.Fatal(err) + } + return r +} + +// The guest returned its finished record within the grace: the host proves +// stop, commits the stopped auth return, writes the stopped receipt and +// reports the shutdown, which the controller requeues. +func TestNativeVMDrainedSessionCommitsStoppedReturnAsShutdown(t *testing.T) { + p, store, drain, info := nativeDrainedPhase(t, true) + nativeReturnedFinished(t, p, info, "synthetic-returned-after-drain") + _, _, evidence, err := p.finish(context.Background(), Report{}, PhaseError{State: "provider_unavailable"}) + if !errors.Is(err, context.Canceled) || phaseState(err) != "cancelled" || errors.Is(err, ErrWorkerStopPending) || errors.Is(err, ErrAuthRecoveryRequired) || errors.Is(err, ErrSourceAuditPending) || errors.Is(err, ErrProviderAuditPending) { + t.Fatal("drained stopped session was not reported as a proved shutdown", err) + } + var pe PhaseError + if !errors.As(err, &pe) || pe.BeforeWork { + t.Fatal("exported session was refunded as before-work", err) + } + if !p.authReturned || evidence == nil || !evidence.Access.Stopped { + t.Fatal("drained session lost its stopped return or receipt", p.authReturned, evidence) + } + state, err := os.ReadFile(filepath.Join(store.directory, "state.json")) + if err != nil || string(state) != "synthetic-returned-after-drain" { + t.Fatal("stopped return not committed", err) + } + if r := readNativeDrainReceipt(t, p); r.Complete || !r.AuthExported || !r.AuthReturned || r.Failure != "cancelled" { + t.Fatal("stopped receipt does not record the returned drained session", r) + } + if drain.held != 0 { + t.Fatal("hold outlived the stopped receipt") + } + // The same result without a drain keeps its own state. + q, _, _, qInfo := nativeDrainedPhase(t, false) + nativeReturnedFinished(t, q, qInfo, "synthetic-returned-without-drain") + if _, _, _, err = q.finish(context.Background(), Report{}, PhaseError{State: "provider_unavailable"}); errors.Is(err, context.Canceled) || phaseState(err) != "provider_unavailable" { + t.Fatal("undrained failure was reclassified", err) + } +} + +// A latched route refusal during the grace stays a refusal, not a shutdown. +func TestNativeVMDrainedRefusalKeepsItsClassification(t *testing.T) { + p, _, _, info := nativeDrainedPhase(t, true) + nativeReturnedFinished(t, p, info, "synthetic-returned-after-refusal") + p.bridge.guard.denyRoute() + if _, _, _, err := p.finish(context.Background(), Report{}, PhaseError{State: "provider_unavailable"}); errors.Is(err, context.Canceled) || phaseState(err) != "provider_unavailable" { + t.Fatal("a provider refusal was hidden as shutdown", err) + } +} + +// The guest never returned within the grace: Close cancelled, the unit was +// still proved stopped, and the exported auth stays fenced with the lease +// retained for source audit, exactly as without a grace. +func TestNativeVMDrainedSessionWithoutReturnStaysFenced(t *testing.T) { + p, store, drain, _ := nativeDrainedPhase(t, true) + _, _, _, err := p.finish(context.Background(), Report{}, PhaseError{State: "provider_unavailable"}) + if !errors.Is(err, ErrAuthRecoveryRequired) || !errors.Is(err, ErrSourceAuditPending) { + t.Fatal("a missing return after the grace was not kept for recovery", err) + } + if r := readNativeDrainReceipt(t, p); r.AuthReturned || !r.AuthExported { + t.Fatal("receipt claims a return that never happened", r) + } + if _, err = store.LeaseProfile(context.Background(), p.request.Binding); !errors.Is(err, ErrAuthRecoveryRequired) { + t.Fatal("next phase reused a possibly refreshed seed", err) + } + if drain.held != 0 { + t.Fatal("hold outlived the stopped receipt") + } +} diff --git a/reviewer/polling.go b/reviewer/polling.go index e7c46989..48bb2254 100644 --- a/reviewer/polling.go +++ b/reviewer/polling.go @@ -68,6 +68,7 @@ type pollWork struct { job string snapshot Snapshot cancel context.CancelFunc + drain *shutdownDrain done chan struct{} err error qualificationInterrupted bool @@ -88,6 +89,8 @@ type PollingController struct { closed bool active *pollWork pollCancel context.CancelFunc + // after is a private fake-clock seam for the shutdown grace timer. + after func(time.Duration) <-chan time.Time } func OpenPollingController(ctx context.Context, o PollOptions) (*PollingController, error) { @@ -122,7 +125,7 @@ func OpenPollingController(ctx context.Context, o PollOptions) (*PollingControll o.Publisher.Now = o.Now tick := make(chan struct{}, 1) tick <- struct{}{} - return &PollingController{options: o, owner: owner, tick: tick, observing: make(chan struct{}, 1)}, nil + return &PollingController{options: o, owner: owner, tick: tick, observing: make(chan struct{}, 1), after: time.After}, nil } func (c *PollingController) now() time.Time { @@ -874,18 +877,20 @@ func (c *PollingController) Poll(ctx context.Context) (result PollResult, err er w := c.selectWork(d, remote) if w != nil { worker, cancel := context.WithCancel(context.Background()) - w.cancel = cancel + w.cancel, w.drain = cancel, newShutdownDrain() c.active = w result.Active = true result.RecoveryRequired = false - go c.runWork(worker, w) + go c.runWork(withShutdownDrain(worker, w.drain), w) } return result, nil } func (c *PollingController) Run(ctx context.Context) (result error) { defer func() { - cleanup, cancel := context.WithTimeout(context.Background(), 10*time.Second) + // Close may first spend the provider shutdown grace, then keeps the + // earlier ten seconds for the cancelled worker's stop. + cleanup, cancel := context.WithTimeout(context.Background(), providerShutdownGrace+10*time.Second) defer cancel() result = errors.Join(result, c.Close(cleanup)) }() @@ -923,6 +928,11 @@ func (c *PollingController) Run(ctx context.Context) (result error) { // Close cancels/drains the active invocation before releasing controller // ownership. An uncertain physical worker keeps its durable lease for explicit // supervisor recovery. A Close timeout keeps the owner lock held for a retry. +// +// Close first signals the invocation's shutdown drain. Only when a provider +// session already holds exported auth does the first Close wait, at most +// providerShutdownGrace or its own deadline, for the invocation to end before +// cancelling it. Every other phase is cancelled at once, as before. func (c *PollingController) Close(ctx context.Context) error { c.mu.Lock() c.closed = true @@ -930,11 +940,16 @@ func (c *PollingController) Close(ctx context.Context) error { c.pollCancel() } active := c.active - if active != nil { - active.cancel() - } c.mu.Unlock() if active != nil { + if active.drain != nil && active.drain.begin() { + select { + case <-active.done: + case <-c.after(providerShutdownGrace): + case <-ctx.Done(): + } + } + active.cancel() select { case <-active.done: case <-ctx.Done(): diff --git a/reviewer/shutdown_grace.go b/reviewer/shutdown_grace.go new file mode 100644 index 00000000..7fa4d53f --- /dev/null +++ b/reviewer/shutdown_grace.go @@ -0,0 +1,95 @@ +package reviewer + +import ( + "context" + "sync" + "time" +) + +// providerShutdownGrace bounds how long PollingController.Close waits, once, +// for an in-flight provider session that has already exported auth. Close +// revokes that session's provider intake, so the guest's client ends and the +// guest can return auth and the host can stop the unit and write its stopped +// receipt. After the grace, or the caller's earlier deadline, Close cancels as +// before. The grace never resets: a second Close or a hanging guest gets none. +// +// Ten seconds covers the guest's own bounded cleanup after its client ends +// (role stop and auth read, at most twelve seconds, normally well under one) +// and the result record. Together with the native host's fifteen-second stop +// path after a late cancellation it stays inside GitReviewerService.Close's +// thirty-second ceiling. A session's own deadline is never extended. +const providerShutdownGrace = 10 * time.Second + +// shutdownDrain is the private signal from PollingController.Close to its one +// active invocation, carried in that invocation's context. A provider session +// holds it before exporting auth; only a held drain earns the grace. +type shutdownDrain struct { + mu sync.Mutex + held int + begun bool + signal context.Context + fire context.CancelFunc +} + +func newShutdownDrain() *shutdownDrain { + signal, fire := context.WithCancel(context.Background()) + return &shutdownDrain{signal: signal, fire: fire} +} + +type shutdownDrainKey struct{} + +func withShutdownDrain(ctx context.Context, d *shutdownDrain) context.Context { + return context.WithValue(ctx, shutdownDrainKey{}, d) +} + +// shutdownDrainOf returns the invocation's drain, or nil outside a controller +// invocation (evaluation runs, fixtures), which never get a grace. +func shutdownDrainOf(ctx context.Context) *shutdownDrain { + if ctx == nil { + return nil + } + d, _ := ctx.Value(shutdownDrainKey{}).(*shutdownDrain) + return d +} + +// hold registers a session about to export auth. It refuses once the drain +// has begun, so no auth is exported after Close started. release is +// idempotent. +func (d *shutdownDrain) hold() (release func(), ok bool) { + d.mu.Lock() + defer d.mu.Unlock() + if d.begun { + return nil, false + } + d.held++ + return sync.OnceFunc(func() { + d.mu.Lock() + d.held-- + d.mu.Unlock() + }), true +} + +// begin starts the drain and fires its signal. It reports true only to the +// first caller and only while a session holds exported auth: that caller +// alone may wait the grace. +func (d *shutdownDrain) begin() bool { + d.mu.Lock() + defer d.mu.Unlock() + first := !d.begun + d.begun = true + d.fire() + return first && d.held > 0 +} + +func (d *shutdownDrain) draining() bool { + if d == nil { + return false + } + d.mu.Lock() + defer d.mu.Unlock() + return d.begun +} + +// shutdownDraining reports whether Close has begun draining ctx's invocation. +// The engine starts no further phase once it has. +func shutdownDraining(ctx context.Context) bool { return shutdownDrainOf(ctx).draining() } diff --git a/reviewer/shutdown_grace_test.go b/reviewer/shutdown_grace_test.go new file mode 100644 index 00000000..84ca21f2 --- /dev/null +++ b/reviewer/shutdown_grace_test.go @@ -0,0 +1,317 @@ +package reviewer + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +func TestShutdownDrainGrantsOneGraceOnlyToAHeldExport(t *testing.T) { + d := newShutdownDrain() + if d.begin() { + t.Fatal("a drain without a held export granted a grace") + } + if _, ok := d.hold(); ok { + t.Fatal("an export was admitted after shutdown began") + } + d = newShutdownDrain() + release, ok := d.hold() + if !ok || d.draining() { + t.Fatal("hold refused before shutdown", ok) + } + if !d.begin() || !d.draining() || d.signal.Err() == nil { + t.Fatal("a held export did not get the grace and its signal") + } + if d.begin() { + t.Fatal("a second Close got a second grace") + } + release() + release() + if d.held != 0 { + t.Fatal("release is not idempotent", d.held) + } + if shutdownDrainOf(context.Background()) != nil || shutdownDraining(context.Background()) { + t.Fatal("an unrelated context carries a drain") + } + if shutdownDrainOf(withShutdownDrain(context.Background(), d)) != d { + t.Fatal("the invocation context lost its drain") + } +} + +// graceClock is the fake shutdown timer: it records each requested grace and +// fires only when the test says so. +type graceClock struct { + mu sync.Mutex + requested []time.Duration + fire chan time.Time +} + +func newGraceClock(c *PollingController) *graceClock { + g := &graceClock{fire: make(chan time.Time)} + c.after = func(d time.Duration) <-chan time.Time { + g.mu.Lock() + defer g.mu.Unlock() + g.requested = append(g.requested, d) + return g.fire + } + return g +} + +func (g *graceClock) calls() []time.Duration { + g.mu.Lock() + defer g.mu.Unlock() + return append([]time.Duration(nil), g.requested...) +} + +// heldSession is a fake provider session that holds the drain like an +// exported native launch: it reports when shutdown signals it and whether its +// context was cancelled by then. +type heldSession struct { + held, signalled chan struct{} + cancelledEarly bool +} + +func newHeldSession() *heldSession { + return &heldSession{held: make(chan struct{}), signalled: make(chan struct{})} +} + +func (h *heldSession) hold(t *testing.T, ctx context.Context) func() { + t.Helper() + d := shutdownDrainOf(ctx) + if d == nil { + t.Error("controller invocation carries no shutdown drain") + return func() {} + } + release, ok := d.hold() + if !ok { + t.Error("drain refused a session before Close") + return func() {} + } + close(h.held) + select { + case <-d.signal.Done(): + case <-ctx.Done(): + } + h.cancelledEarly = d.signal.Err() == nil || ctx.Err() != nil + close(h.signalled) + return release +} + +func awaitClosed(t *testing.T, ch chan struct{}, what string) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal(what) + } +} + +func closeAsync(c *PollingController, ctx context.Context) chan error { + done := make(chan error, 1) + go func() { done <- c.Close(ctx) }() + return done +} + +// A held session that returns auth after its intake is revoked ends within +// the grace: Close proves completion and the stopped job is requeued with its +// consumed session. No later phase starts. +func TestPollingControllerShutdownGraceLetsAHeldSessionReturn(t *testing.T) { + h := newHeldSession() + c, f, calls := pollControllerFixture(t, func(ctx context.Context, s Session) (Report, error) { + release := h.hold(t, ctx) + defer release() + // The stopped native launch reports the shutdown with its own state. + return Report{}, errors.Join(PhaseError{State: "provider_unavailable"}, context.Canceled) + }) + g := newGraceClock(c) + if _, err := pollOnce(t, c); err != nil { + t.Fatal(err) + } + awaitClosed(t, h.held, "session never held the drain") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := c.Close(ctx); err != nil { + t.Fatal("Close did not prove a session that ended within the grace", err) + } + if h.cancelledEarly { + t.Fatal("the held session was cancelled instead of signalled") + } + if got := g.calls(); len(got) != 1 || got[0] != providerShutdownGrace { + t.Fatal("Close did not bound its wait by the documented grace", got) + } + d, _ := f.store.Snapshot(context.Background()) + for _, j := range d.Jobs { + if j.State != "queued" || j.LeaseToken != "" || j.Outstanding || j.Checkpoint.ConsumedSessions != 1 || len(j.Checkpoint.Artifacts) != 0 { + t.Fatal("drained session lost its stop, charge or requeue", j.State, j.LeaseToken, j.Outstanding, j.Checkpoint.ConsumedSessions) + } + } + if calls.Load() != 1 { + t.Fatal("drain admitted another phase", calls.Load()) + } +} + +// A session that completes during the grace keeps its artifact; the engine +// then starts no further phase and the job is requeued. +func TestPollingControllerShutdownGraceStartsNoFurtherPhase(t *testing.T) { + h := newHeldSession() + c, f, calls := pollControllerFixture(t, func(ctx context.Context, s Session) (Report, error) { + release := h.hold(t, ctx) + defer release() + return fakeReport(s), nil + }) + newGraceClock(c) + if _, err := pollOnce(t, c); err != nil { + t.Fatal(err) + } + awaitClosed(t, h.held, "session never held the drain") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := c.Close(ctx); err != nil { + t.Fatal(err) + } + d, _ := f.store.Snapshot(context.Background()) + for _, j := range d.Jobs { + if j.State != "queued" || j.LeaseToken != "" || j.Checkpoint.ConsumedSessions != 1 || len(j.Checkpoint.Artifacts) != 1 { + t.Fatal("completed drained phase was lost or followed by another", j.State, j.Checkpoint.ConsumedSessions, len(j.Checkpoint.Artifacts)) + } + } + if calls.Load() != 1 { + t.Fatal("drain admitted another phase", calls.Load()) + } +} + +// A guest that never returns: Close waits exactly until the grace timer +// fires, never longer, then cancels; the uncertain stop keeps the lease and +// reports ErrWorkerStopPending as before. +func TestPollingControllerShutdownGraceExpiresIntoStopPending(t *testing.T) { + h := newHeldSession() + stopped := make(chan struct{}) + c, f, _ := pollControllerFixture(t, func(ctx context.Context, s Session) (Report, error) { + release := h.hold(t, ctx) + defer release() + <-ctx.Done() + close(stopped) + return Report{}, errors.Join(ErrWorkerStopPending, ctx.Err()) + }) + g := newGraceClock(c) + if _, err := pollOnce(t, c); err != nil { + t.Fatal(err) + } + awaitClosed(t, h.held, "session never held the drain") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + done := closeAsync(c, ctx) + awaitClosed(t, h.signalled, "held session was never signalled") + select { + case err := <-done: + t.Fatal("Close returned before the grace expired", err) + case <-stopped: + t.Fatal("the session was cancelled before the grace expired") + case <-time.After(50 * time.Millisecond): + } + if got := g.calls(); len(got) != 1 || got[0] != providerShutdownGrace { + t.Fatal("grace not requested once at its documented bound", got) + } + g.fire <- time.Now() + awaitClosed(t, stopped, "grace expiry did not cancel the session") + if err := <-done; !errors.Is(err, ErrWorkerStopPending) { + t.Fatal("an unproved stop was hidden", err) + } + d, _ := f.store.Snapshot(context.Background()) + for _, j := range d.Jobs { + if j.LeaseToken == "" || !j.Outstanding { + t.Fatal("uncertain session lost its lease or reservation", j.State) + } + } + // A retry gets no second grace. + retry, cancelRetry := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancelRetry() + if err := c.Close(retry); !errors.Is(err, ErrWorkerStopPending) { + t.Fatal(err) + } + if got := g.calls(); len(got) != 1 { + t.Fatal("a retried Close waited a second grace", got) + } +} + +// The caller's earlier deadline bounds the grace too. +func TestPollingControllerShutdownGraceNeverOutlastsCloseDeadline(t *testing.T) { + h := newHeldSession() + stopped := make(chan struct{}) + c, _, _ := pollControllerFixture(t, func(ctx context.Context, s Session) (Report, error) { + release := h.hold(t, ctx) + defer release() + <-ctx.Done() + close(stopped) + return Report{}, errors.Join(ErrWorkerStopPending, ctx.Err()) + }) + newGraceClock(c) // never fires + if _, err := pollOnce(t, c); err != nil { + t.Fatal(err) + } + awaitClosed(t, h.held, "session never held the drain") + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Millisecond) + defer cancel() + started := time.Now() + if err := c.Close(ctx); err == nil { + t.Fatal("Close proved an uncertain session") + } + if elapsed := time.Since(started); elapsed > 2*time.Second { + t.Fatal("Close outlasted its deadline", elapsed) + } + awaitClosed(t, stopped, "deadline did not cancel the held session") +} + +// Close outside a held provider export cancels at once and never starts the +// grace timer. +func TestPollingControllerShutdownOutsideProviderSessionHasNoGrace(t *testing.T) { + entered := make(chan struct{}) + c, f, calls := pollControllerFixture(t, func(ctx context.Context, s Session) (Report, error) { + close(entered) + <-ctx.Done() + return Report{}, ctx.Err() + }) + c.after = func(time.Duration) <-chan time.Time { + t.Error("a session without exported auth started the grace") + return make(chan time.Time) + } + if _, err := pollOnce(t, c); err != nil { + t.Fatal(err) + } + awaitClosed(t, entered, "worker never started") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := c.Close(ctx); err != nil { + t.Fatal(err) + } + d, _ := f.store.Snapshot(context.Background()) + for _, j := range d.Jobs { + if j.State != "queued" || j.LeaseToken != "" { + t.Fatal("unheld shutdown changed the existing requeue", j.State) + } + } + if calls.Load() != 1 { + t.Fatal(calls.Load()) + } + + // Nor during factory work before any session. + c2, _, _ := pollControllerFixture(t, nil) + c2.after = c.after + factory := make(chan struct{}) + c2.options.Runtime = func(ctx context.Context, _ Job, _ *JobEvidence) (Engine, error) { + close(factory) + <-ctx.Done() + return Engine{}, ctx.Err() + } + if _, err := pollOnce(t, c2); err != nil { + t.Fatal(err) + } + awaitClosed(t, factory, "factory never started") + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + if err := c2.Close(ctx2); err != nil { + t.Fatal(err) + } +} From 11395042b1e850455473e8d0b261cf537ecedd0f Mon Sep 17 00:00:00 2001 From: Franz Hemmer <8227352+HemSoft@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:34:09 -0400 Subject: [PATCH 2/4] Document the bounded provider shutdown grace (#264) GIT-STARTUP.md gains a shutdown grace section: what holds the drain, what Close waits for and for how long, why ten seconds fits the existing guest cleanup, native stop path and 30-second Close ceiling, and that stop proof and auth rules are unchanged. NATIVE-VM.md, POLLING.md and CONTROLLER.md describe the host revocation and controller behaviour. E2E-QUALIFICATION.md now names the Close-proves-completion branch of close-provider as the expected one, with the requeued job expiring its captured budget after three reruns, and keeps the fenced branch for a guest that does not return in time. Recovery of a lease whose stopped receipt is missing remains an open owner decision in #264 and stays fenced. Co-Authored-By: Claude Opus 5.5 --- reviewer/CONTROLLER.md | 4 +++ reviewer/E2E-QUALIFICATION.md | 50 ++++++++++++++++++++--------- reviewer/GIT-STARTUP.md | 60 +++++++++++++++++++++++++++++++---- reviewer/NATIVE-VM.md | 18 +++++++++++ reviewer/POLLING.md | 20 ++++++++---- 5 files changed, 123 insertions(+), 29 deletions(-) diff --git a/reviewer/CONTROLLER.md b/reviewer/CONTROLLER.md index 25b97d90..8b03a9b8 100644 --- a/reviewer/CONTROLLER.md +++ b/reviewer/CONTROLLER.md @@ -59,6 +59,10 @@ sessions begin, including jobs queued on earlier dates. Only one lease may exist. Expiration does not authorize stealing it. A separately qualified supervisor must prove the worker has stopped before `ReleaseStopped`. +Service shutdown gives a provider session that already exported auth a bounded +grace to stop and return it before cancellation +([POLLING.md](POLLING.md#shutdown-and-recovery)). The lease and stop rules are +unchanged, and a session that does not return in time keeps its lease. Drivers supplied to `RunStopped` must return only after worker termination. Broker-equipped drivers also return trusted per-attempt source receipts, kept diff --git a/reviewer/E2E-QUALIFICATION.md b/reviewer/E2E-QUALIFICATION.md index 48b5da76..b75e2e26 100644 --- a/reviewer/E2E-QUALIFICATION.md +++ b/reviewer/E2E-QUALIFICATION.md @@ -94,19 +94,34 @@ ledger, and a restore probe. | `review` | PR 1 automatic request through capture, four sessions and one confirmed COMMENT. | 1 / 4 | 1 | both usable | | `isolation` | PR 1 (automatic) and PR 2 (owner comment) pending together. | 2 / 8 | 2 | both usable | | `close-capture` | Close the whole service after capture and admission, inside the first session's host check, before any VM launch. Reopen and finish. | 1 / 4 | 1 | both usable | -| `close-provider` | Close the whole service while the first Codex session waits for a withheld inference response. | 1 / 1 | lifecycle order only | Codex fenced, Claude usable | +| `close-provider` | Close the whole service while the first Codex session waits for a withheld inference response. With the shutdown grace, reopen and run the requeued job. | 1 / 4 (grace) or 1 / 1 (fenced) | 1 (grace) or lifecycle order only (fenced) | both usable (grace), or Codex fenced and Claude usable | | `close-provider-reopen` | A second process attached to the root retained by `close-provider` (`SFL_E2E_ATTACH`), as a restarted service. | 0 / 0 | 1 (retained) | Codex fenced, Claude usable | | `close-provider-recover` | Instead of `close-provider-reopen`: a restarted process recovers the retained attempt through the operator API, publishes PR 1's `auth_failed` status, then reviews and publishes the next request. | 1 / 4 | 2 (1 retained) | both usable (Codex re-provisioned) | | `close-publish` | The fake GitHub commits the review POST but withholds its response; close the whole service; reopen and reconcile. | 1 / 4 | 1 | both usable | -If `close-provider`'s `Close` proves completion instead of reporting -`ErrWorkerStopPending`, the case reopens and expects one source unit, one -native unit for the interrupted launch plus one per artifact of the reopened -job, and both auth stores usable. The job must record the interrupted -launch's complete stopped audit evidence. After the reopen it runs the same -completed-review checks as the other successful cases (see Evidence bindings), -with the interrupted launch, stopped, audited and incomplete, as the only -other launch. No host run has taken this branch. +Since the [shutdown grace](GIT-STARTUP.md#shutdown-grace) ([#264](https://github.com/hemsoft-dev/set-it-free-loop/issues/264)), +the expected `close-provider` branch is a `Close` that proves completion. +`Close` revokes the withheld session's provider intake, the guest's client +proxy stops the CLI, and the guest returns auth in its result record within +the ten-second grace. The case then requires a queued, unleased job with one +charged session and no artifact, and Codex auth usable. The stopped audit must +record the auth as exported and returned, end at the result record's +validation and fail as `cancelled`. The job must record that launch as its +attempt, with its complete stopped evidence and the source receipt its result +returned. The case then reopens. The requeued job keeps that charged session, +so its captured four-session budget leaves three. It reruns Codex R1, Claude +R1 and Codex R2, each in one completed launch holding the canned report, and +expires (`incomplete_expired`) before Claude R2, unleased, with no intent, +POST or remote review. The case therefore expects one source unit, four +native units (the interrupted launch and three reruns), both auth stores +usable and only the qualified routes. The interrupted launch is the only +launch outside the job's artifacts. No host run has taken this branch. Its +earlier form expected a published review after the reopen, which the +captured budget cannot allow. + +If the guest does not return within the grace, `Close` cancels as before and +the case takes the fenced branch below. Only that branch retains a root for +`close-provider-reopen` and `close-provider-recover`. Every case also observes `systemctl --user list-units 'sfl-alpha-*'` every 200 ms, excluding units that existed before the run. At the end it first stops @@ -533,10 +548,11 @@ flock -o ~/.cache/sfl-vm.lock env GOMAXPROCS=2 \ timeout 600 nice -n 10 "$OUT/reviewer.test" -test.run '^TestE2EQualification$' -test.v -test.timeout 590s ``` -For `close-provider`, run `close-provider-reopen` next in a new process with -`SFL_E2E_ATTACH` set to the `Root` recorded in `close-provider.json`, inside -the same lock hold. State lives under a fresh `/tmp/sfl-e2e-` root, -which a passing run removes. `SFL_E2E_KEEP_FAILED=1` retains a failed root. +When `close-provider` took the fenced branch and retained its root, run +`close-provider-reopen` next in a new process with `SFL_E2E_ATTACH` set to the +`Root` recorded in `close-provider.json`, inside the same lock hold. State +lives under a fresh `/tmp/sfl-e2e-` root, which a passing run removes. +`SFL_E2E_KEEP_FAILED=1` retains a failed root. ## Measured at `13251e3` @@ -716,7 +732,8 @@ and return auth. Both change security-relevant recovery behavior, so neither was implemented in the commits measured above. The operator path was added later and is qualified in [Operator recovery after a provider-session close](#operator-recovery-after-a-provider-session-close). -The bounded shutdown grace is not implemented. +The bounded [shutdown grace](GIT-STARTUP.md#shutdown-grace) was added after +that, and its `close-provider` branch has not run on a host. A shutdown before a session's work, while the host check runs, is different. The native reviewer reports a trusted before-work failure, the @@ -879,10 +896,13 @@ now waits for admission. [GIT-VM-SOURCE.md](GIT-VM-SOURCE.md), and native supervisor process loss by [NATIVE-VM-CRASH.md](NATIVE-VM-CRASH.md). A `SIGKILL` between session reservation and stopped receipt would leave the same durable fence as - `close-provider`, without the in-process stop proof. + `close-provider`, without the in-process stop proof. The shutdown grace + applies only to `Close`; a process kill gets none. - Operator recovery covers a provider attempt stopped with a durable stopped audit receipt (see the recovery section). An attempt without that receipt, such as after a host process kill mid-session, stays fenced. + Whether it may ever be recovered, and on what alternative evidence, is an + open owner decision in [#264](https://github.com/hemsoft-dev/set-it-free-loop/issues/264). - Isolation is between sequential jobs because admission is serial. Both jobs share the per-provider auth stores by design; each session exports and returns them, and the case checks returns but cannot observe a concurrent diff --git a/reviewer/GIT-STARTUP.md b/reviewer/GIT-STARTUP.md index d266f0f5..00f0bbbc 100644 --- a/reviewer/GIT-STARTUP.md +++ b/reviewer/GIT-STARTUP.md @@ -128,14 +128,57 @@ Operators must retain and investigate such artifacts; automatic crash-artifact collection is not implemented. This bound and retained artifacts belong in the full host disk/resource qualification. +## Shutdown grace + +`Close` gives an in-flight native provider session a bounded chance to return +its auth before the session is cancelled +([#264](https://github.com/hemsoft-dev/set-it-free-loop/issues/264)). +The controller's active invocation carries a private shutdown drain. A native launch holds it +immediately before exporting auth, and no launch exports once the drain has +begun. When `Close` begins the drain: + +- Without a held export, `Close` cancels at once, exactly as before. This + covers capture, admission, restoration, the host check, boot before export, + publication and work outside a controller invocation. +- With a held export, the launch closes its provider intake: the private + provider stream and any in-flight request close. The guest's client proxy + already stops its owned CLI when a private exchange fails, so the client + ends, and guest PID1 stops its roles and returns auth and the source audit + in its ordinary result record. The first `Close` waits for the invocation + to end for at most `providerShutdownGrace` (ten seconds) or its own earlier + deadline, then cancels as before. The grace is granted once; a retried + `Close` cancels at once. The session's own deadline is never extended. + +The stop proof and the auth rules do not change. The host still reads the +result record only under the session deadline and the later cancellation, +proves whole-unit stop before any auth step, and commits auth only through the +durable stopped return. A refresh-uncertain return is still fenced. A session +that fails after the drain reports `cancelled`, so the stopped job is requeued +with its artifacts and consumed budget; a latched route or response refusal, +or exhaustion, keeps its own classification. A session that completes within +the grace keeps its artifact, and the engine starts no further phase. + +The ten seconds cover the guest's bounded cleanup after its client ends (at +most twelve seconds, normally well under one) and its result record. If the +grace expires, cancellation interrupts the result read and the native host's +fifteen-second stop path runs, so grace and stop stay inside the 30-second +`Close` ceiling. The deferred `Close` in `PollingController.Run` has the grace +plus its earlier ten seconds. When the guest does not return in time, the +outcome is the one described below: the unit is still proved stopped, the +stopped receipt records the unreturned export, the lease stays and `Close` +reports `ErrWorkerStopPending`. + +The grace applies only to `Close`. A host process kill gets none. + ## Stopped provider recovery -A shutdown or failure during a native provider session can leave the durable -job leased: the unit is proven stopped, but the guest never returned its -result record, so the source audit is missing and an exported provider auth -stays fenced. The controller then admits nothing, and `Close` keeps reporting -`ErrWorkerStopPending`. Nothing releases that lease automatically, on a timer -or from polling, GitHub or model input. +A failure during a native provider session, a shutdown whose session does not +return within the [grace](#shutdown-grace), or a host process kill can leave +the durable job leased: the unit is proven stopped, but the guest never +returned its result record, so the source audit is missing and an exported +provider auth stays fenced. The controller then admits nothing, and `Close` +keeps reporting `ErrWorkerStopPending`. Nothing releases that lease +automatically, on a timer or from polling, GitHub or model input. `RecoverStoppedProvider` is the explicit operator path. Call it after a process restart, on a service that was opened but not yet refreshed or run: @@ -165,7 +208,10 @@ It refuses, without changing the job or auth, unless all of these hold: and evidence, so the attempt ID is what rejects an earlier attempt's launch. A receipt without an attempt ID refuses. A host process kill before that receipt was written therefore cannot be recovered through - this path. + this path. Whether such a lease may ever be recovered, and on what + alternative evidence, is an open owner decision in + [#264](https://github.com/hemsoft-dev/set-it-free-loop/issues/264). + Until it is decided, that lease stays fenced. - The auth journal of that provider either belongs to this launch or does not exist, and is consistent with the receipt (`ErrAuthRecoveryRequired`). diff --git a/reviewer/NATIVE-VM.md b/reviewer/NATIVE-VM.md index 1305440c..26454dda 100644 --- a/reviewer/NATIVE-VM.md +++ b/reviewer/NATIVE-VM.md @@ -349,6 +349,24 @@ then stops the entire unit and keeps uncertain exported auth held. A blocked source read exposed a seventy-second wait for the original deadline; a stopped source channel alone did not wake the host's guest-result read. +A controller shutdown drains a launch that already exported auth before it +cancels it (see the [shutdown grace](GIT-STARTUP.md#shutdown-grace)). The +launch holds the drain from just before export until its stopped receipt is +written, and exports nothing once the drain has begun. The drain closes the +host provider bridge's HTTP server, which closes the private provider stream +and any in-flight request; the guard records an aborted request as failed. +The guest protocol is unchanged. Its client proxy already stops the owned CLI +when a private exchange fails (`provider-transport`), and guest PID1 then +returns auth and the source audit in its finished record. The host reads that +record under the session deadline until the controller's grace expires and +cancels it. Whole-unit stop, auth import, the conservative refresh rule and +the stopped receipt are unchanged; the receipt of a drained session that +failed reports `cancelled`. Offline tests cover the host bridge, the guest +proxy handler and the host result handling with fakes. Whether the guest sees +the closed virtio port as end of stream promptly is a host property they do +not exercise; if it does not, the grace expires into the cancellation path +above. + ## Operator recovery of a stopped attempt `GitReviewerService.RecoverStoppedProvider` (see diff --git a/reviewer/POLLING.md b/reviewer/POLLING.md index 2b189fb2..6ac49dd6 100644 --- a/reviewer/POLLING.md +++ b/reviewer/POLLING.md @@ -155,15 +155,21 @@ App history consistency remains a separate qualification gate. ## Shutdown and recovery -`Close` cancels in-progress polling and the active invocation, waits for them, -then releases controller ownership. A timeout retains the owner lock for a later -close attempt. An uncertain physical worker retains its original SQLite lease -and reports `ErrWorkerStopPending`. Expiration and process restart never permit -lease stealing; a separately qualified supervisor must prove stopped recovery. +`Close` cancels in-progress polling and signals the active invocation's +shutdown drain. Only when a provider session already holds exported auth does +the first `Close` wait, at most the ten-second provider shutdown grace or its +own deadline, for that invocation to end +([GIT-STARTUP.md](GIT-STARTUP.md#shutdown-grace)). It then cancels the +invocation, waits for it and releases controller ownership. A timeout retains +the owner lock for a later close attempt. An uncertain physical worker retains +its original SQLite lease and reports `ErrWorkerStopPending`. Expiration and +process restart never permit lease stealing; a separately qualified supervisor +must prove stopped recovery. Known-stopped work interrupted only by service shutdown may return to `queued` -with its exact artifacts and consumed budget. Owner cancellation, generation -changes and uncertain stop cannot use that path. Factory errors become durable +with its exact artifacts and consumed budget. A phase that completes during the +grace keeps its artifact, and the engine starts no further phase. Owner +cancellation, generation changes and uncertain stop cannot use that path. Factory errors become durable unavailable/blocked states. A failed qualification check cancels active work; all admitted paths, including discovery/restoration and pre-work qualification, share one finalization step. An otherwise current queued or cancelled job From b189e8587f5bcd73673d34a5bc42ba24e006ba5e Mon Sep 17 00:00:00 2001 From: Franz Hemmer <8227352+HemSoft@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:56:34 -0400 Subject: [PATCH 3/4] End a job its session budget can no longer finish without a session (#264) A job's captured session budget equals its schedule, one session per provider and round, so every charged session without a validated artifact leaves it one session short. After a shutdown grace the requeued job kept its interrupted session's charge and then reran three real sessions before expiring. sessionBudgetFits now states the rule from the job's configuration and checkpoint, as the engine plans phases, and such a job spends nothing: - the polling controller ends a retained queued or quota-waiting job that does not fit as incomplete_expired before restoring evidence, building a runtime or claiming it; - the claim ends it the same way before any lease, quota wait or session; - the engine starts no phase of a schedule that does not fit (ErrSessionBudgetExhausted); - ResumeUnavailable refuses to requeue it with ErrSessionBudgetExhausted, covering charged auth or provider failures and operator recovery. Its status names the cause: an incomplete_expired status reports the exhausted budget with its charged and resultless sessions, and an auth_failed or provider_unavailable status no longer promises a resume. The E2E close-provider branch where Close proves completion now expects the reopened job to end incomplete with no further launch and exactly one confirmed INCOMPLETE status matched to its remote COMMENT. Tests that relied on resuming a short budget now require the refusal. Co-Authored-By: Claude Opus 5.5 --- reviewer/durable.go | 42 ++++ reviewer/durable_test.go | 21 +- reviewer/e2e_export_linux_test.go | 9 +- reviewer/e2e_qualification_linux_test.go | 230 ++++++++++---------- reviewer/engine.go | 19 ++ reviewer/engine_test.go | 9 +- reviewer/git_service_recovery_linux_test.go | 10 +- reviewer/polling.go | 11 + reviewer/polling_test.go | 11 +- reviewer/publication.go | 2 +- reviewer/render.go | 21 +- reviewer/session_budget_test.go | 183 ++++++++++++++++ reviewer/status_polling_test.go | 2 +- 13 files changed, 430 insertions(+), 140 deletions(-) create mode 100644 reviewer/session_budget_test.go diff --git a/reviewer/durable.go b/reviewer/durable.go index a10c4e8c..5ed75a8e 100644 --- a/reviewer/durable.go +++ b/reviewer/durable.go @@ -645,6 +645,12 @@ func (s *Store) claimJob(ctx context.Context, id string, now time.Time) (*Durabl if j.State != "queued" && j.State != "quota_wait" { return ErrDeferred } + // A job whose remaining budget cannot finish its schedule ends here, + // before a lease, quota wait or session, as an expired wait does. + if !sessionBudgetFits(j.Job, j.Checkpoint) { + j.State = "incomplete_expired" + return nil + } daily := 0 for _, other := range d.Jobs { if chargedOn(other, now.UTC().Format("2006-01-02")) { @@ -1270,11 +1276,47 @@ func (s *Store) ResumeUnavailable(ctx context.Context, id string, now time.Time) if now.Sub(j.CreatedAt) > j.Job.Config.QuotaWait || j.Checkpoint.ConsumedSessions >= j.Job.Config.MaxSessions() { return ErrDeferred } + // A charged session without a result leaves too little budget for the + // schedule. Such a job is never requeued to spend more sessions. + if !sessionBudgetFits(j.Job, j.Checkpoint) { + return ErrSessionBudgetExhausted + } j.State = "queued" return nil }) } +var errBudgetUnchanged = errors.New("job budget unchanged") + +// expireUnfinishable ends a current, unleased queued or quota-waiting job as +// incomplete_expired when its remaining session budget can no longer finish +// its schedule (sessionBudgetFits). It starts nothing and reports whether it +// ended the job; the controller calls it before restoring evidence or +// building a runtime, so a certain incomplete spends no further work. +func (s *Store) expireUnfinishable(ctx context.Context, id string) (bool, error) { + err := s.update(ctx, func(d *DurableState) error { + j := d.Jobs[id] + if j == nil || j.LeaseToken != "" || j.Outstanding || (j.State != "queued" && j.State != "quota_wait") || sessionBudgetFits(j.Job, j.Checkpoint) { + return errBudgetUnchanged + } + p := d.PRs[j.Scope.key()] + if p == nil || p.Paused || p.Generation != j.Job.Generation || p.CurrentRequest != j.RequestID { + return errBudgetUnchanged + } + for _, other := range d.Jobs { + if other.LeaseToken != "" { + return errBudgetUnchanged + } + } + j.State = "incomplete_expired" + return nil + }) + if errors.Is(err, errBudgetUnchanged) { + return false, nil + } + return err == nil, err +} + func retireFencedJobs(d *DurableState, p *PRRecord) { for _, j := range d.Jobs { if j.Scope == p.Scope && (j.Job.Generation != p.Generation || j.RequestID != p.CurrentRequest) { diff --git a/reviewer/durable_test.go b/reviewer/durable_test.go index 0cf1a53b..87e0731d 100644 --- a/reviewer/durable_test.go +++ b/reviewer/durable_test.go @@ -336,8 +336,14 @@ func TestLeaseDoesNotStealExpiredWorkerAndCrashConsumesBudget(t *testing.T) { if err = s.ReleaseStopped(ctx, job.ID, token, "queued"); err != nil { t.Fatal(err) } - if _, err = s.Claim(ctx, job.ID, now); err != nil { - t.Fatal(err) + // The crashed session leaves three sessions for four phases, so the claim + // ends the job incomplete instead of leasing it for more sessions. + if _, err = s.Claim(ctx, job.ID, now); !errors.Is(err, ErrDeferred) { + t.Fatal("an unfinishable job was claimed", err) + } + d, _ = s.Snapshot(ctx) + if j := d.Jobs[job.ID]; j.State != "incomplete_expired" || j.LeaseToken != "" || j.Checkpoint.ConsumedSessions != 1 { + t.Fatal("unfinishable job not ended incomplete", j.State, j.LeaseToken, j.Checkpoint.ConsumedSessions) } } func TestSQLiteConcurrentAdmissionAcrossHandles(t *testing.T) { @@ -997,13 +1003,6 @@ func TestFailedRefundRetainsLeaseForExplicitDurableRecovery(t *testing.T) { s, job, _ := admittedFixture(t) ctx := context.Background() now := controlTime.Add(4 * time.Minute) - // Three earlier uncertain attempts left exactly one session available. - if err := s.update(ctx, func(d *DurableState) error { - d.Jobs[job.ID].Checkpoint.ConsumedSessions = 3 - return nil - }); err != nil { - t.Fatal(err) - } preflightErr := errors.New("fixture authentication failure before work") engine := testEngine(t, func(context.Context, Session) (Report, error) { _, err := s.db.Exec(`CREATE TRIGGER refund_fault BEFORE UPDATE ON state BEGIN SELECT RAISE(FAIL, 'refund_fault'); END`) @@ -1021,7 +1020,7 @@ func TestFailedRefundRetainsLeaseForExplicitDurableRecovery(t *testing.T) { t.Fatal(err) } j := d.Jobs[job.ID] - if j.LeaseToken == "" || !j.Outstanding || j.Checkpoint.ConsumedSessions != 4 || j.AttemptChargeDay == "" { + if j.LeaseToken == "" || !j.Outstanding || j.Checkpoint.ConsumedSessions != 1 || j.AttemptChargeDay == "" { t.Fatal("failed refund lost durable recovery metadata", j) } if _, err = s.db.Exec("DROP TRIGGER refund_fault"); err != nil { @@ -1060,7 +1059,7 @@ func TestFailedRefundRetainsLeaseForExplicitDurableRecovery(t *testing.T) { t.Fatal(err) } j = d.Jobs[job.ID] - if j.Checkpoint.ConsumedSessions != 3 || len(j.ChargeDays) != 0 || j.AttemptChargeDay != "" || j.Outstanding || j.LeaseToken != "" || j.State != "queued" { + if j.Checkpoint.ConsumedSessions != 0 || len(j.ChargeDays) != 0 || j.AttemptChargeDay != "" || j.Outstanding || j.LeaseToken != "" || j.State != "queued" { t.Fatal("explicit recovery did not restore the actual budget", j) } } diff --git a/reviewer/e2e_export_linux_test.go b/reviewer/e2e_export_linux_test.go index f3b70970..0cd5ed73 100644 --- a/reviewer/e2e_export_linux_test.go +++ b/reviewer/e2e_export_linux_test.go @@ -368,12 +368,13 @@ func E2ECompletedDocument(job Job, reports []Report, ledger []Entry, diff map[st // E2EStatusDocument renders the canonical status document production // publishes for job in non-complete state after completed validated phases: // production's renderer, the intent's marker appended as newIntent appends -// it, with no inline comment. It reads no durable state. -func E2EStatusDocument(job Job, state string, completed int, marker string) (Document, error) { - if !statusStates[state] || completed < 0 { +// it, with no inline comment. consumed is the job's charged sessions, which +// name a budget that can no longer finish. It reads no durable state. +func E2EStatusDocument(job Job, state string, completed, consumed int, marker string) (Document, error) { + if !statusStates[state] || completed < 0 || consumed < completed { return Document{}, ErrIgnored } - doc, err := renderOutcome(Outcome{Job: job, State: state, Artifacts: make([]Artifact, completed)}, nil) + doc, err := renderOutcome(Outcome{Job: job, State: state, Artifacts: make([]Artifact, completed), ConsumedSessions: consumed}, nil) if err != nil { return Document{}, err } diff --git a/reviewer/e2e_qualification_linux_test.go b/reviewer/e2e_qualification_linux_test.go index 9811e56a..08210030 100644 --- a/reviewer/e2e_qualification_linux_test.go +++ b/reviewer/e2e_qualification_linux_test.go @@ -2645,7 +2645,7 @@ func e2eStatusVerdicts(c e2eCompletion, pr int, j *r.DurableJob, state string) [ v := []e2eVerdict{{w != nil, fmt.Sprintf("PR %d has exactly one status intent and it is confirmed (%d writes)", pr, len(writes))}} if w != nil && j != nil { expect := e2eExpectedFor(c, pr, j).Job - want, err := r.E2EStatusDocument(expect, state, len(j.Checkpoint.Artifacts), w.Marker) + want, err := r.E2EStatusDocument(expect, state, len(j.Checkpoint.Artifacts), j.Checkpoint.ConsumedSessions, w.Marker) v = append(v, e2eVerdict{w.Status == state && j.State == state && w.Scope == j.Scope && w.RequestID == j.RequestID && w.Generation == j.Job.Generation && w.Snapshot == j.Job.Snapshot && w.JobHash == r.Digest(j.Job) && w.CheckpointHash == r.Digest(j.Checkpoint) && w.ReceiptID > 0, fmt.Sprintf("PR %d confirmed status reports %s and binds its job's request, generation, snapshot, job and checkpoint, and a receipt (status %s)", pr, state, w.Status)}, e2eVerdict{w.Marker == "" && w.BodyHash == r.Digest(w.Document) && len(w.Document.Inline) == 0, fmt.Sprintf("PR %d confirmed status's marker names its own ID, its body hash is its document's and it has no inline comment", pr)}, @@ -3218,36 +3218,37 @@ func e2eNativeSessionVerdict(units []e2eNativeUnit, jobs []*r.DurableJob, extra // e2eProviderReopenVerdicts judges close-provider after a Close that proved // completion (the shutdown grace returned the interrupted launch's auth) and -// a reopen. The requeued job keeps that launch's charged session, so its -// captured four-session budget leaves three: it reruns Codex R1, Claude R1 -// and Codex R2, each in one completed launch holding the canned report, then -// expires before Claude R2, unleased, with no intent, POST or remote review. -// Its source receipts must still be stopped, and the interrupted launch, -// bound to its recorded attempt, is the only other launch allowed. It returns -// PR 1's job, or nil when it is not the only job. +// a reopen. The requeued job keeps that launch's charged session without a +// result, so its captured budget can no longer finish the schedule: the +// reopened controller ends it incomplete_expired at once, unleased, with no +// artifact, no further session and no review, and publishes exactly one +// INCOMPLETE status for it, confirmed and held as one inline-free remote +// COMMENT (e2eStatusVerdicts). Its source receipts must still be stopped, and +// the interrupted launch, bound to its recorded attempt and returned auth, +// must be the only launch. It returns PR 1's job, or nil when it is not the +// only job. func e2eProviderReopenVerdicts(c e2eCompletion, interrupted string) ([]e2eVerdict, *r.DurableJob) { const label = "after reopen" + v := e2eDurableCountVerdicts(label, c.State, map[int]int{1: 1}, map[int]int{}, map[int]int{1: 1}) j := e2eJobFor(c.State, 1) if j == nil || len(c.State.Jobs) != 1 { - return []e2eVerdict{{false, fmt.Sprintf("%s: PR 1's job is the only job (%d jobs)", label, len(c.State.Jobs))}}, nil + return append(v, e2eVerdict{false, fmt.Sprintf("%s: PR 1's job is the only job (%d jobs)", label, len(c.State.Jobs))}), nil } expected := e2eExpectedFor(c, 1, j) - phases := []string{} - for _, a := range j.Checkpoint.Artifacts { - phases = append(phases, fmt.Sprintf("%s R%d", a.Report.Provider, a.Report.Round)) - } - canned := e2eCannedArtifacts(expected) - v := []e2eVerdict{ - {r.Digest(j.Job) == r.Digest(expected.Job), fmt.Sprintf("%s: PR 1's job is exactly the one admission derives from its retained request and the harness's inputs", label)}, - {j.State == "incomplete_expired" && j.LeaseToken == "" && !j.Outstanding, fmt.Sprintf("%s: the requeued job expired its captured budget, unleased and with no outstanding reservation (state %s)", label, j.State)}, - {j.Checkpoint.ConsumedSessions == c.Config.MaxSessions() && j.Checkpoint.JobHash == r.Digest(j.Job), fmt.Sprintf("%s: the job consumed exactly its %d captured sessions, the interrupted one included (consumed %d)", label, c.Config.MaxSessions(), j.Checkpoint.ConsumedSessions)}, - {slices.Equal(phases, []string{"codex R1", "claude R1", "codex R2"}), fmt.Sprintf("%s: the job stored exactly Codex R1, Claude R1 and Codex R2 (%v)", label, phases)}, - {canned == nil, fmt.Sprintf("%s: the stored artifacts hold exactly the canned backend's reports for their phases: %v", label, canned)}, - {len(c.State.Writes) == 0, fmt.Sprintf("%s: the expired job prepared no write intent (%d)", label, len(c.State.Writes))}, - {c.Posts[1] == 0 && len(c.Reviews[1]) == 0, fmt.Sprintf("%s: PR 1 received no review POST and holds no remote review (%d POSTs, %d reviews)", label, c.Posts[1], len(c.Reviews[1]))}, - } + v = append(v, + e2eVerdict{r.Digest(j.Job) == r.Digest(expected.Job), fmt.Sprintf("%s: PR 1's job is exactly the one admission derives from its retained request and the harness's inputs", label)}, + e2eVerdict{j.State == "incomplete_expired" && j.LeaseToken == "" && !j.Outstanding, fmt.Sprintf("%s: the requeued job that its budget could no longer finish ended incomplete, unleased and with no outstanding reservation (state %s)", label, j.State)}, + e2eVerdict{j.Checkpoint.ConsumedSessions == 1 && len(j.Checkpoint.Artifacts) == 0 && j.Checkpoint.JobHash == r.Digest(j.Job), fmt.Sprintf("%s: the job started no further session: its one charged session is the interrupted one and it holds no artifact (consumed %d, artifacts %d)", label, j.Checkpoint.ConsumedSessions, len(j.Checkpoint.Artifacts))}, + ) + v = append(v, e2eStatusVerdicts(c, 1, j, "incomplete_expired")...) v = append(v, e2eSourceVerdicts(label, c, []*r.DurableJob{expected})...) - return append(v, e2eAttemptSetVerdict(c.Units, []*r.DurableJob{expected}, interrupted), e2eNativeSessionVerdict(c.Units, []*r.DurableJob{expected}, interrupted)), j + only := len(c.Units) == 1 && c.Units[0].SessionHash == interrupted && c.Units[0].wholeStop() && c.Units[0].AuditPresent && !c.Units[0].Complete && c.Units[0].AuthExported && c.Units[0].AuthReturned + var bound error = errors.New("no single interrupted launch") + if only { + _, bound = e2eInterruptedLaunch(c.Units[0], expected) + } + v = append(v, e2eVerdict{only && bound == nil, fmt.Sprintf("%s: the only native launch is the interrupted Codex R1 launch, whole-unit stopped with its auth exported and returned, bound to the job's recorded attempt (%d launches): %v", label, len(c.Units), bound)}) + return append(v, e2eAttemptSetVerdict(c.Units, []*r.DurableJob{expected}, interrupted)), j } // e2eCompletedVerdicts judges a completed case's evidence for the reviewed @@ -3586,11 +3587,20 @@ func e2eCloseProvider(e *e2eEnv) { } e.open("reopen") // The requeued job keeps the interrupted session's charge, so its - // captured four-session budget cannot finish the adversarial - // protocol: it reruns three phases and expires before Claude R2. - e.wait("PR 1 job settles after reopen", 6*time.Minute, func(d r.DurableState) bool { + // captured budget can no longer finish the schedule: the reopened + // controller ends it incomplete at once, starts no session and + // publishes its one INCOMPLETE status. + e.wait("PR 1 job ended incomplete with its status confirmed after reopen", 2*time.Minute, func(d r.DurableState) bool { j := e2eJobFor(d, 1) - return j != nil && j.LeaseToken == "" && !j.Outstanding && j.State != "queued" && j.State != "running" + if j == nil || j.LeaseToken != "" || j.Outstanding || j.State == "queued" || j.State == "running" { + return false + } + for _, w := range d.Writes { + if w.Kind == "status" && w.State == "confirmed" { + return true + } + } + return false }) e.snapshotJobs("after reopen") // The one allowed extra launch is the Codex R1 session the expected @@ -3606,10 +3616,10 @@ func e2eCloseProvider(e *e2eEnv) { e.checkAll(verdicts) e.result.Requests = e2eRequests(c.State) e.checkAll(e2eRequestVerdicts("after reopen", e.result.Requests, map[int]e2eRequest{1: e2eAdmittedOpened(1, j.RequestID)})) - // The interrupted launch plus the three rerun phases, restated rather - // than counted from the reopened job's artifacts, still one - // acquisition, both auth stores usable, and nothing published. - e.expect = e2eExpect{SourceUnits: 1, NativeUnits: 1 + 3, AuthUsable: true, Routes: e2eQualifiedRoutes} + // Only the interrupted launch and the one acquisition, both auth + // stores usable, and the status published through the review POST + // and receipt routes. + e.expect = e2eExpect{SourceUnits: 1, NativeUnits: 1, AuthUsable: true, Routes: e2ePublishedRoutes} if reopened != nil { e.result.Jobs = append(e.result.Jobs, e2eJobSummary(reopened)) } @@ -5353,32 +5363,40 @@ func e2eAcquisition(rec r.E2ESourceRecord) []e2eGitExchange { // e2eReopenFixture is a valid end state of close-provider's reopen after a // Close that proved completion: PR 1's only job, requeued with the -// interrupted launch's charged session, reran Codex R1, Claude R1 and Codex -// R2 and expired before Claude R2, with no intent, POST or remote review. The -// interrupted launch of the first session is the last unit. Its version 2 -// receipt names the attempt under which the job records its stopped evidence -// and source receipt; it returned its auth in the result record, so its -// audit ends at that record's validation and reports the shutdown. +// interrupted launch's charged session and no artifact, ended +// incomplete_expired without another session and published its one +// INCOMPLETE status as an inline-free COMMENT. The interrupted launch of the +// first session is the only unit. Its version 2 receipt names the attempt +// under which the job records its stopped evidence and source receipt; it +// returned its auth in the result record, so its audit ends at that record's +// validation and reports the shutdown. func e2eReopenFixture() (e2eCompletion, string) { c := e2eCompletedFixture() job := e2eJobFor(c.State, 1) - job.Checkpoint.Artifacts = job.Checkpoint.Artifacts[:3] - job.ProviderAttempts, job.SourceAttempts = job.ProviderAttempts[:3], job.SourceAttempts[:3] - last := job.Checkpoint.Artifacts[2] - job.AttemptProvider, job.AttemptRound, job.AttemptID, job.AttemptInputHash = r.Codex, 2, job.ProviderAttempts[2].ID, last.InputHash - job.State, job.Checkpoint.ConsumedSessions = "incomplete_expired", 4 - c.Units, c.Sessions = c.Units[:3], c.Sessions[:3] - c.State.Writes, c.Posts, c.Reviews = map[string]*r.WriteIntent{}, map[int]int{}, map[int][]map[string]any{} session := e2eFixtureSession(job, r.Codex, 1) first := r.Digest(session) evidence := e2eFixtureEvidence(session, false) interrupted := strings.Repeat("b", 32) access := &r.SourceAccess{Version: 1, SessionHash: first, Reads: []r.ReadRecord{}, Coverage: []r.Coverage{}} recorded := *access - job.ProviderAttempts = append([]r.ProviderAttempt{{ID: interrupted, Evidence: evidence}}, job.ProviderAttempts...) - job.SourceAttempts = append([]r.SourceAttempt{{ID: interrupted, Access: &recorded}}, job.SourceAttempts...) - c.Units = append(c.Units, e2eLaunchFixture(job, session, strings.Repeat("9", 32), interrupted, evidence, r.Report{}, access, false, true)) - c.Sessions = append([]e2eModelSession{{JobID: job.Job.ID, Provider: r.Codex, Round: 1, PR: 1, Head: job.Job.Snapshot.Head, SessionHash: first, HeldInference: true}}, c.Sessions...) + job.Checkpoint.Artifacts = nil + job.ProviderAttempts = []r.ProviderAttempt{{ID: interrupted, Evidence: evidence}} + job.SourceAttempts = []r.SourceAttempt{{ID: interrupted, Access: &recorded}} + job.AuditedProviders = []r.Provider{r.Codex} + job.AttemptProvider, job.AttemptRound, job.AttemptID, job.AttemptInputHash = r.Codex, 1, interrupted, first + job.State, job.Checkpoint.ConsumedSessions = "incomplete_expired", 1 + c.Units = []e2eNativeUnit{e2eLaunchFixture(job, session, strings.Repeat("9", 32), interrupted, evidence, r.Report{}, access, false, true)} + c.Sessions = []e2eModelSession{{JobID: job.Job.ID, Provider: r.Codex, Round: 1, PR: 1, Head: job.Job.Snapshot.Head, SessionHash: first, HeldInference: true}} + id := strings.Repeat("e", 32) + marker := "" + document, err := r.E2EStatusDocument(job.Job, "incomplete_expired", 0, 1, marker) + if err != nil { + panic(err) + } + c.State.Writes = map[string]*r.WriteIntent{id: {ID: id, Kind: "status", Status: "incomplete_expired", Scope: job.Scope, RequestID: job.RequestID, Generation: job.Job.Generation, JobHash: r.Digest(job.Job), CheckpointHash: r.Digest(job.Checkpoint), Snapshot: job.Job.Snapshot, Document: document, BodyHash: r.Digest(document), Marker: marker, State: "confirmed", ReceiptID: 5002}} + remoteMarker, remoteHash := r.E2ERemoteReview(document.Summary, document.Inline) + c.Reviews = map[int][]map[string]any{1: {{"id": int64(5002), "state": "COMMENTED", "commit_id": job.Job.Snapshot.Head, "marker": remoteMarker, "document_hash": remoteHash, "inline": []string{}}}} + c.Posts = map[int]int{1: 1} return c, first } @@ -5399,88 +5417,68 @@ func TestE2EProviderReopenVerdicts(t *testing.T) { return e2eFailures(verdicts) } if f := failures(e2eReopenFixture()); len(f) != 0 { - t.Fatal("budget-expired reopen refused:", f) + t.Fatal("unfinishable reopen refused:", f) } - const last = 3 // the interrupted launch + status := func(c *e2eCompletion) *r.WriteIntent { return c.State.Writes[strings.Repeat("e", 32)] } for name, edit := range map[string]func(*e2eCompletion){ "review intent prepared": func(c *e2eCompletion) { c.State.Writes["w2"] = &r.WriteIntent{ID: "w2", Kind: "review", Scope: e2eScope(1), State: "unknown"} }, - "review POST": func(c *e2eCompletion) { c.Posts[1] = 1 }, - "remote review": func(c *e2eCompletion) { c.Reviews[1] = []map[string]any{{"id": int64(5001)}} }, - "job completed": func(c *e2eCompletion) { e2eJobFor(c.State, 1).State = "dual_complete" }, - "job requeued again": func(c *e2eCompletion) { e2eJobFor(c.State, 1).State = "queued" }, - "lease retained": func(c *e2eCompletion) { e2eJobFor(c.State, 1).LeaseToken = "lease" }, - "reservation outstanding": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Outstanding = true }, - "interrupted refunded": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.ConsumedSessions = 3 }, - "session beyond budget": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.ConsumedSessions = 5 }, - "missing Codex R2": func(c *e2eCompletion) { - j := e2eJobFor(c.State, 1) - j.Checkpoint.Artifacts = j.Checkpoint.Artifacts[:2] - }, - "second job": func(c *e2eCompletion) { c.State.Jobs["other"] = e2eJobFor(c.State, 1) }, - "artifact not canned": func(c *e2eCompletion) { - e2eJobFor(c.State, 1).Checkpoint.Artifacts[1].Report.Retractions = []string{"changed"} - }, - "completed launch kept its auth": func(c *e2eCompletion) { c.Units[1].AuthReturned = false }, - "completed launch audit unbound": func(c *e2eCompletion) { c.Units[2].AuditPresent = false }, - "completed launch names an unknown attempt": func(c *e2eCompletion) { c.Units[2].AttemptID = strings.Repeat("f", 32) }, - "completed launch names the interrupted attempt": func(c *e2eCompletion) { - c.Units[0].AttemptID, c.Units[0].EvidenceHash = c.Units[last].AttemptID, c.Units[last].EvidenceHash - }, - "completed launch names another session's attempt": func(c *e2eCompletion) { - c.Units[1].AttemptID, c.Units[1].EvidenceHash = c.Units[2].AttemptID, c.Units[2].EvidenceHash - }, - "completed launch repeats other evidence": func(c *e2eCompletion) { c.Units[2].EvidenceHash = c.Units[last].EvidenceHash }, - "artifact attempt unrecorded": func(c *e2eCompletion) { - j := e2eJobFor(c.State, 1) - j.ProviderAttempts = slices.Delete(j.ProviderAttempts, 2, 3) - }, - "artifact without provider evidence": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.Artifacts[1].ProviderEvidence = nil }, - "second interrupted launch": func(c *e2eCompletion) { c.Units = append(c.Units, c.Units[last]) }, - "launch for Claude R2": func(c *e2eCompletion) { - full := e2eCompletedFixture() - c.Units = append(c.Units, full.Units[3]) - }, + "second status intent": func(c *e2eCompletion) { + other := *status(c) + other.ID = "w3" + c.State.Writes["w3"] = &other + }, + "no status intent": func(c *e2eCompletion) { delete(c.State.Writes, strings.Repeat("e", 32)) }, + "status not confirmed": func(c *e2eCompletion) { status(c).State = "unknown" }, + "status of another state": func(c *e2eCompletion) { status(c).Status = "auth_failed" }, + "status of another document": func(c *e2eCompletion) { status(c).Document.Summary += "changed" }, + "status without its receipt": func(c *e2eCompletion) { status(c).ReceiptID = 0 }, + "second status POST": func(c *e2eCompletion) { c.Posts[1] = 2 }, + "no remote COMMENT": func(c *e2eCompletion) { c.Reviews[1] = nil }, + "remote COMMENT with inline": func(c *e2eCompletion) { c.Reviews[1][0]["inline"] = []string{"source.go:1"} }, + "remote COMMENT on another head": func(c *e2eCompletion) { + c.Reviews[1][0]["commit_id"] = strings.Repeat("9", 40) + }, + "remote COMMENT of another document": func(c *e2eCompletion) { c.Reviews[1][0]["document_hash"] = r.Digest("other") }, + "job requeued again": func(c *e2eCompletion) { e2eJobFor(c.State, 1).State = "queued" }, + "job ended another way": func(c *e2eCompletion) { e2eJobFor(c.State, 1).State = "auth_failed" }, + "lease retained": func(c *e2eCompletion) { e2eJobFor(c.State, 1).LeaseToken = "lease" }, + "reservation outstanding": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Outstanding = true }, + "further session charged": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.ConsumedSessions = 2 }, + "interrupted refunded": func(c *e2eCompletion) { e2eJobFor(c.State, 1).Checkpoint.ConsumedSessions = 0 }, + "artifact stored": func(c *e2eCompletion) { + full := e2eJobFor(e2eCompletedFixture().State, 1) + e2eJobFor(c.State, 1).Checkpoint.Artifacts = full.Checkpoint.Artifacts[:1] + }, + "second job": func(c *e2eCompletion) { c.State.Jobs["other"] = e2eJobFor(c.State, 1) }, + "launch for a rerun": func(c *e2eCompletion) { c.Units = append(c.Units, e2eCompletedFixture().Units[0]) }, + "second interrupted launch": func(c *e2eCompletion) { c.Units = append(c.Units, c.Units[0]) }, + "no launch": func(c *e2eCompletion) { c.Units = nil }, "source attempt not stopped": func(c *e2eCompletion) { c.Records[0].Stopped = false }, "source records unreadable": func(c *e2eCompletion) { c.SourceErr = r.ErrLiveBlocked }, "interrupted launch completed": func(c *e2eCompletion) { - // A completed launch with a foreign receipt must not pass as the - // interrupted one. - c.Units[last].Complete = true + c.Units[0].Complete = true }, - "interrupted launch without its audit": func(c *e2eCompletion) { c.Units[last].AuditPresent = false }, - "interrupted launch names a completed attempt": func(c *e2eCompletion) { - c.Units[last].AttemptID, c.Units[last].EvidenceHash = c.Units[1].AttemptID, c.Units[1].EvidenceHash + "interrupted launch kept its auth": func(c *e2eCompletion) { c.Units[0].AuthReturned = false }, + "interrupted launch without its audit": func(c *e2eCompletion) { c.Units[0].AuditPresent = false }, + "interrupted launch names an unknown attempt": func(c *e2eCompletion) { c.Units[0].AttemptID = strings.Repeat("f", 32) }, + "interrupted launch repeats other evidence": func(c *e2eCompletion) { + c.Units[0].EvidenceHash = e2eCompletedFixture().Units[0].EvidenceHash }, - "interrupted launch names an unknown attempt": func(c *e2eCompletion) { c.Units[last].AttemptID = strings.Repeat("f", 32) }, - "interrupted launch repeats other evidence": func(c *e2eCompletion) { c.Units[last].EvidenceHash = c.Units[1].EvidenceHash }, "interrupted attempt recorded twice": func(c *e2eCompletion) { j := e2eJobFor(c.State, 1) j.ProviderAttempts = append(j.ProviderAttempts, j.ProviderAttempts[0]) }, - "interrupted attempt never recorded": func(c *e2eCompletion) { - e2eJobFor(c.State, 1).ProviderAttempts = e2eJobFor(c.State, 1).ProviderAttempts[1:] - }, + "interrupted attempt never recorded": func(c *e2eCompletion) { e2eJobFor(c.State, 1).ProviderAttempts = nil }, "interrupted source receipt never recorded": func(c *e2eCompletion) { - e2eJobFor(c.State, 1).SourceAttempts = e2eJobFor(c.State, 1).SourceAttempts[1:] - }, - "second interrupted provider attempt": func(c *e2eCompletion) { - j := e2eJobFor(c.State, 1) - other := *j.ProviderAttempts[0].Evidence - other.Access.Responses-- - j.ProviderAttempts = append(j.ProviderAttempts, r.ProviderAttempt{ID: "interrupted-again", Evidence: &other}) - }, - "interrupted audit of a report": func(c *e2eCompletion) { - c.Units[last].Audit.ReportHash = r.Digest(e2eJobFor(c.State, 1).Checkpoint.Artifacts[0].Report) - }, - "interrupted audit of another source receipt": func(c *e2eCompletion) { - c.Units[last].Audit.SourceHash = r.Digest(e2eJobFor(c.State, 1).Checkpoint.Artifacts[0].SourceAccess) + e2eJobFor(c.State, 1).SourceAttempts = nil }, - "interrupted audit of another request": func(c *e2eCompletion) { c.Units[last].Audit.RequestHash = r.Digest("another request") }, - "interrupted audit before its result": func(c *e2eCompletion) { c.Units[last].Audit.Stage = "result" }, - "interrupted audit without a failure": func(c *e2eCompletion) { c.Units[last].Audit.Failure = "" }, - "interrupted unit never aborted": func(c *e2eCompletion) { c.Units[last].Aborted = false }, + "interrupted audit of another source receipt": func(c *e2eCompletion) { c.Units[0].Audit.SourceHash = r.Digest("other receipt") }, + "interrupted audit of another request": func(c *e2eCompletion) { c.Units[0].Audit.RequestHash = r.Digest("another request") }, + "interrupted audit before its result": func(c *e2eCompletion) { c.Units[0].Audit.Stage = "result" }, + "interrupted audit without a failure": func(c *e2eCompletion) { c.Units[0].Audit.Failure = "" }, + "interrupted unit never aborted": func(c *e2eCompletion) { c.Units[0].Aborted = false }, } { c, interrupted := e2eReopenFixture() edit(&c) @@ -6100,7 +6098,7 @@ func e2eRecoveredFixtureOf(edit1, edit2 func(*r.Job)) (e2eCompletion, *r.Durable marker := "" // An edited job admission cannot validate may not render; its status then // carries only the marker, which its other refusals do not depend on. - document, err := r.E2EStatusDocument(j1.Job, "auth_failed", len(j1.Checkpoint.Artifacts), marker) + document, err := r.E2EStatusDocument(j1.Job, "auth_failed", len(j1.Checkpoint.Artifacts), j1.Checkpoint.ConsumedSessions, marker) if err != nil { document = r.Document{Summary: "\n\n" + marker, Inline: []r.InlineComment{}} } diff --git a/reviewer/engine.go b/reviewer/engine.go index 268c8b96..7abf3311 100644 --- a/reviewer/engine.go +++ b/reviewer/engine.go @@ -163,6 +163,21 @@ func phaseState(err error) string { return "provider_unavailable" } +// ErrSessionBudgetExhausted reports a job whose remaining captured session +// budget can no longer run every phase it still needs. It ends the job before +// any session starts, so no provider usage is spent on a certain incomplete. +var ErrSessionBudgetExhausted = errors.New("captured session budget cannot complete the remaining phases") + +// sessionBudgetFits reports whether a job's remaining captured session budget +// can run every phase the engine still plans: one session per provider and +// round, less the validated artifacts it resumes. Every charged session +// without an artifact (an interrupted, failed or shutdown-ended phase) leaves +// the schedule one session short, because the budget equals the schedule. +func sessionBudgetFits(job Job, cp Checkpoint) bool { + phases := len(job.Config.Providers()) * job.Config.Rounds() + return job.Config.MaxSessions()-cp.ConsumedSessions >= phases-len(cp.Artifacts) +} + // Run executes a bounded sequential protocol. Valid artifacts can resume only // the exact captured job, input, configuration, release and snapshot. It performs // no publication, automatic transport retry or change of selected provider mode. @@ -203,6 +218,10 @@ func (e Engine) Run(ctx context.Context, job Job, checkpoint Checkpoint) (Outcom return fail("invalid_output", errors.New("resume artifacts are not a valid phase prefix")) } } + // Start no phase of a schedule the remaining budget cannot finish. + if !sessionBudgetFits(job, checkpoint) { + return fail("incomplete_expired", ErrSessionBudgetExhausted) + } var reports []Report for round := 1; round <= job.Config.Rounds(); round++ { var peers []Report diff --git a/reviewer/engine_test.go b/reviewer/engine_test.go index bb1757f1..7448076a 100644 --- a/reviewer/engine_test.go +++ b/reviewer/engine_test.go @@ -179,9 +179,14 @@ func TestFailedSessionsCannotExceedBudgetAcrossResume(t *testing.T) { t.Fatal("uncertain session was refunded") } fail = false + // The charged failure leaves two sessions for three phases: the resume + // starts none and ends incomplete at once. + calls := 0 + e.Reviewers[Codex] = reviewFunc(func(_ context.Context, s Session) (Report, error) { calls++; return protocolReport(s), nil }) + e.Reviewers[Claude] = e.Reviewers[Codex] next, err := e.Run(context.Background(), testJob(), out.Checkpoint()) - if err == nil || next.State != "incomplete_expired" || next.ConsumedSessions != 4 { - t.Fatal("session ceiling bypassed", err) + if !errors.Is(err, ErrSessionBudgetExhausted) || next.State != "incomplete_expired" || next.ConsumedSessions != 2 || next.ExecutedSessions != 0 || calls != 0 { + t.Fatal("an unfinishable resume spent sessions", next.State, next.ConsumedSessions, next.ExecutedSessions, calls, err) } } diff --git a/reviewer/git_service_recovery_linux_test.go b/reviewer/git_service_recovery_linux_test.go index f0134c0b..35d609f6 100644 --- a/reviewer/git_service_recovery_linux_test.go +++ b/reviewer/git_service_recovery_linux_test.go @@ -251,9 +251,13 @@ func TestProviderRecoveryRetiresLostAuthAndReleasesChargedLease(t *testing.T) { if _, err := f.auth.LeaseProfile(ctx, authTestBinding()); err == nil { t.Fatal("retired auth could still be leased") } - // Only an explicit resume can requeue the charged job; nothing reviewed it. - if err = f.store.ResumeUnavailable(ctx, f.job.ID, controlTime.Add(5*time.Minute)); err != nil { - t.Fatal("documented resume path refused the recovered job", err) + // The charged session produced no result, so the remaining budget cannot + // finish the schedule: even an explicit resume refuses to requeue it. + if err = f.store.ResumeUnavailable(ctx, f.job.ID, controlTime.Add(5*time.Minute)); !errors.Is(err, ErrSessionBudgetExhausted) { + t.Fatal("an unfinishable recovered job was requeued", err) + } + if d, _ = f.store.Snapshot(ctx); d.Jobs[f.job.ID].State != "auth_failed" { + t.Fatal("refused resume changed the recovered job", d.Jobs[f.job.ID].State) } if _, err = f.service.RecoverStoppedProvider(ctx, f.request()); !errors.Is(err, ErrFenced) { t.Fatal("repeated recovery was accepted", err) diff --git a/reviewer/polling.go b/reviewer/polling.go index 48bb2254..3d662beb 100644 --- a/reviewer/polling.go +++ b/reviewer/polling.go @@ -535,6 +535,17 @@ func (c *PollingController) beforeWork(ctx context.Context, w *pollWork) (bound func (c *PollingController) runWork(ctx context.Context, w *pollWork) { defer close(w.done) + // A retained job that can no longer finish its schedule ends incomplete + // before restoration, a runtime or a claim. A failed qualification instead + // takes the ordinary path below. + if w.job != "" && c.options.Check(ctx, c.options.Enrollment, c.options.Release) == nil { + if expired, err := c.options.Store.expireUnfinishable(ctx, w.job); err != nil || expired { + c.mu.Lock() + w.err = err + c.mu.Unlock() + return + } + } bound, job, err := c.beforeWork(ctx, w) refused := err == errCaptureRefused if refused { diff --git a/reviewer/polling_test.go b/reviewer/polling_test.go index 3b351bee..7ab17728 100644 --- a/reviewer/polling_test.go +++ b/reviewer/polling_test.go @@ -701,7 +701,16 @@ func TestPollingQualificationInterruptionIsExplicitlyRecoverable(t *testing.T) { if err != nil || result.Active { t.Fatal("automatically repaired qualification interruption", result, err) } - if err := f.store.ResumeUnavailable(context.Background(), original.ID, f.clock.Now()); err != nil { + err = f.store.ResumeUnavailable(context.Background(), original.ID, f.clock.Now()) + if !factory { + // The interrupted provider session stays charged without a + // result, so the budget cannot finish the schedule. + if !errors.Is(err, ErrSessionBudgetExhausted) { + t.Fatal("an unfinishable interrupted job was requeued", err) + } + return + } + if err != nil { t.Fatal("original request cannot resume", err) } d, _ = f.store.Snapshot(context.Background()) diff --git a/reviewer/publication.go b/reviewer/publication.go index ab91dd25..ddff719e 100644 --- a/reviewer/publication.go +++ b/reviewer/publication.go @@ -316,7 +316,7 @@ func (p Publisher) PrepareStatuses(ctx context.Context) error { if reported { continue } - doc, err := renderOutcome(Outcome{Job: j.Job, State: j.State, Artifacts: j.Checkpoint.Artifacts}, nil) + doc, err := renderOutcome(Outcome{Job: j.Job, State: j.State, Artifacts: j.Checkpoint.Artifacts, ConsumedSessions: j.Checkpoint.ConsumedSessions}, nil) if err != nil { return err } diff --git a/reviewer/render.go b/reviewer/render.go index e28e8e71..e2d5c678 100644 --- a/reviewer/render.go +++ b/reviewer/render.go @@ -143,7 +143,7 @@ func renderOutcome(out Outcome, diffLines map[string]map[int]bool) (Document, er } b.WriteString("\n\n\n") if !complete { - fmt.Fprintf(&b, "%s No substantive partial findings are published. Completed phases: %d of %d.\n", safeText(statusCause(out.Job, out.State)), len(out.Artifacts), out.Job.Config.MaxSessions()) + fmt.Fprintf(&b, "%s No substantive partial findings are published. Completed phases: %d of %d.\n", safeText(statusBudgetCause(out.Job, out.State, out.ConsumedSessions, out.Artifacts)), len(out.Artifacts), out.Job.Config.MaxSessions()) doc.Summary = b.String() return doc, nil } @@ -289,6 +289,25 @@ func statusCause(job Job, state string) string { return "INCOMPLETE: " + state + "." } +// statusBudgetCause is statusCause, except that a job whose remaining session +// budget can no longer finish its schedule says so: it cannot resume, and an +// expired job names the charged sessions that produced no result. +func statusBudgetCause(job Job, state string, consumed int, artifacts []Artifact) string { + if sessionBudgetFits(job, Checkpoint{ConsumedSessions: consumed, Artifacts: artifacts}) { + return statusCause(job, state) + } + short := fmt.Sprintf("%d of its %d captured sessions are charged, %d of them without a result, so the remaining sessions cannot complete every required phase.", consumed, job.Config.MaxSessions(), consumed-len(artifacts)) + switch state { + case "incomplete_expired": + return "INCOMPLETE: the captured session budget is exhausted. " + short + " It ended without a review and starts no further session." + case "auth_failed": + return "INCOMPLETE: provider authentication is unavailable. " + short + " This job cannot be resumed and ended without a review." + case "provider_unavailable": + return "INCOMPLETE: a selected provider is unavailable. " + short + " This job cannot be resumed and ended without a review." + } + return statusCause(job, state) +} + // Exact duplicate allegations share one conservative status and all history. // An unresolved objection to either ID blocks inline publication for the group. func renderGroups(entries []Entry) []Entry { diff --git a/reviewer/session_budget_test.go b/reviewer/session_budget_test.go new file mode 100644 index 00000000..462e049e --- /dev/null +++ b/reviewer/session_budget_test.go @@ -0,0 +1,183 @@ +package reviewer + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestSessionBudgetFitsFollowsTheEngineSchedule(t *testing.T) { + job := testJob() + artifact := Artifact{} + for _, c := range []struct { + consumed, artifacts int + fits bool + }{ + {0, 0, true}, {1, 1, true}, {3, 3, true}, {4, 4, true}, + {1, 0, false}, {2, 1, false}, {4, 3, false}, {4, 0, false}, + } { + cp := Checkpoint{ConsumedSessions: c.consumed} + for range c.artifacts { + cp.Artifacts = append(cp.Artifacts, artifact) + } + if sessionBudgetFits(job, cp) != c.fits { + t.Error("budget rule disagrees with the schedule", c) + } + } + // A paired third round widens schedule and budget alike. + job.Config.PeerRounds = 2 + if !sessionBudgetFits(job, Checkpoint{ConsumedSessions: 4, Artifacts: make([]Artifact, 4)}) || sessionBudgetFits(job, Checkpoint{ConsumedSessions: 5, Artifacts: make([]Artifact, 4)}) { + t.Fatal("budget rule ignores the configured rounds") + } +} + +// heldShutdownFixture runs the shutdown grace end to end on the fake +// controller: the first session holds the drain, returns its auth and reports +// the shutdown, and Close requeues the stopped job with that charged session. +func heldShutdownFixture(t *testing.T) (*PollingController, *pollingFixture, *atomic.Int32) { + t.Helper() + h := newHeldSession() + var started atomic.Bool + c, f, calls := pollControllerFixture(t, func(ctx context.Context, s Session) (Report, error) { + // Only the first session is the held one; any later session is + // counted by the fixture and must never run. + if started.Swap(true) { + return fakeReport(s), nil + } + release := h.hold(t, ctx) + defer release() + return Report{}, errors.Join(PhaseError{State: "provider_unavailable"}, context.Canceled) + }) + newGraceClock(c) + if _, err := pollOnce(t, c); err != nil { + t.Fatal(err) + } + awaitClosed(t, h.held, "session never held the drain") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := c.Close(ctx); err != nil { + t.Fatal(err) + } + return c, f, calls +} + +// After a shutdown that proved completion, the requeued job keeps a charged +// session without a result and can no longer finish its schedule. The +// restarted controller ends it incomplete without restoring evidence, +// building a runtime, claiming it or starting a session, and publishes one +// INCOMPLETE status that names the cause. +func TestPollingEndsAnUnfinishableJobWithoutFurtherSessions(t *testing.T) { + c, f, calls := heldShutdownFixture(t) + var restores, runtimes atomic.Int32 + options := c.options + restore, runtime := options.Restore, options.Runtime + options.Restore = func(ctx context.Context, job Job) (*JobEvidence, error) { + restores.Add(1) + return restore(ctx, job) + } + options.Runtime = func(ctx context.Context, job Job, bound *JobEvidence) (Engine, error) { + runtimes.Add(1) + return runtime(ctx, job, bound) + } + reopened, err := OpenPollingController(context.Background(), options) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { reopened.Close(context.Background()) }) + f.clock.Advance(2 * time.Second) + if _, err := pollOnce(t, reopened); err != nil { + t.Fatal(err) + } + settleWorker(t, reopened) + for range 3 { + pollAgain(t, reopened, f) + } + d, _ := f.store.Snapshot(context.Background()) + for _, j := range d.Jobs { + if j.State != "incomplete_expired" || j.LeaseToken != "" || j.Outstanding || j.Checkpoint.ConsumedSessions != 1 || len(j.Checkpoint.Artifacts) != 0 { + t.Fatal("unfinishable job not ended at once", j.State, j.Checkpoint.ConsumedSessions) + } + } + if calls.Load() != 1 || restores.Load() != 0 || runtimes.Load() != 0 { + t.Fatal("an unfinishable job spent work", calls.Load(), restores.Load(), runtimes.Load()) + } + statuses := statusWrites(t, f) + if len(statuses) != 1 || f.posts != 1 || statuses[0].Status != "incomplete_expired" || statuses[0].State != "confirmed" { + t.Fatal("not exactly one confirmed INCOMPLETE status", len(statuses), f.posts) + } + if !strings.Contains(statuses[0].Document.Summary, "INCOMPLETE: the captured session budget is exhausted. 1 of its 4 captured sessions are charged, 1 of them without a result") { + t.Fatal("status does not name the exhausted budget", statuses[0].Document.Summary) + } +} + +// A job that cannot finish never waits for quota: the claim gate ends it +// before the daily limit, without a lease or session. +func TestClaimEndsAnUnfinishableQuotaWaitBeforeTheDailyLimit(t *testing.T) { + s, job, _ := admittedFixture(t) + ctx := context.Background() + if err := s.update(ctx, func(d *DurableState) error { + j := d.Jobs[job.ID] + j.State, j.Checkpoint.ConsumedSessions, j.ChargeDays = "quota_wait", 1, []string{"2026-10-01"} + for i := range j.Job.Config.DailyReviews { + other := *j + other.Job.ID = job.ID + "-charged-" + string(rune('a'+i)) + other.State, other.ChargeDays = "dual_complete", []string{controlTime.UTC().Format("2006-01-02")} + d.Jobs[other.Job.ID] = &other + } + return nil + }); err != nil { + t.Fatal(err) + } + if _, err := s.Claim(ctx, job.ID, controlTime.Add(4*time.Minute)); !errors.Is(err, ErrDeferred) { + t.Fatal(err) + } + d, _ := s.Snapshot(ctx) + if j := d.Jobs[job.ID]; j.State != "incomplete_expired" || j.LeaseToken != "" || j.Checkpoint.ConsumedSessions != 1 { + t.Fatal("unfinishable job kept waiting for quota", j.State) + } +} + +// ResumeUnavailable never requeues a job its budget cannot finish, and still +// resumes one it can. +func TestResumeUnavailableRefusesAnUnfinishableJob(t *testing.T) { + for _, c := range []struct { + consumed, artifacts int + want error + state string + }{{2, 1, ErrSessionBudgetExhausted, "provider_unavailable"}, {1, 1, nil, "queued"}} { + s, job, _ := admittedFixture(t) + ctx := context.Background() + if err := s.update(ctx, func(d *DurableState) error { + j := d.Jobs[job.ID] + j.State, j.Checkpoint.ConsumedSessions, j.Checkpoint.Artifacts = "provider_unavailable", c.consumed, make([]Artifact, c.artifacts) + return nil + }); err != nil { + t.Fatal(err) + } + if err := s.ResumeUnavailable(ctx, job.ID, controlTime.Add(4*time.Minute)); !errors.Is(err, c.want) || (c.want == nil) != (err == nil) { + t.Fatal("resume decision", c, err) + } + d, _ := s.Snapshot(ctx) + if d.Jobs[job.ID].State != c.state { + t.Fatal("resume changed the wrong job state", c, d.Jobs[job.ID].State) + } + } +} + +// A resumable status of a job that can no longer finish does not promise a +// resume. +func TestStatusOfAnUnfinishableJobPromisesNoResume(t *testing.T) { + job := testJob() + for _, state := range []string{"auth_failed", "provider_unavailable"} { + cause := statusBudgetCause(job, state, 1, nil) + if !strings.Contains(cause, "1 of its 4 captured sessions are charged, 1 of them without a result") || !strings.Contains(cause, "cannot be resumed") || strings.Contains(cause, "explicitly resumes") { + t.Fatal("unfinishable status still promises a resume", state, cause) + } + if statusBudgetCause(job, state, 1, make([]Artifact, 1)) != statusCause(job, state) { + t.Fatal("a finishable job's status changed", state) + } + } +} diff --git a/reviewer/status_polling_test.go b/reviewer/status_polling_test.go index 3e6a3586..2d3553cc 100644 --- a/reviewer/status_polling_test.go +++ b/reviewer/status_polling_test.go @@ -55,7 +55,7 @@ func TestPollingPublishesOneStatusForEachNonCompleteOutcome(t *testing.T) { "auth_failed": "INCOMPLETE: provider authentication is unavailable.", "provider_unavailable": "INCOMPLETE: a selected provider is unavailable.", "invalid_output": "INCOMPLETE: a provider returned output that failed validation.", - "incomplete_expired": "INCOMPLETE: this job expired before every required phase completed.", + "incomplete_expired": "INCOMPLETE: the captured session budget is exhausted. 2 of its 4 captured sessions are charged, 1 of them without a result", } { t.Run(state, func(t *testing.T) { c, f, _ := pollControllerFixture(t, func(_ context.Context, s Session) (Report, error) { From 3dc6a4da9aa786d435a5b820ac5feaa28a4fb204 Mon Sep 17 00:00:00 2001 From: Franz Hemmer <8227352+HemSoft@users.noreply.github.com> Date: Fri, 23 Oct 2026 23:56:34 -0400 Subject: [PATCH 4/4] Document the session budget rule and the reopened close-provider outcome (#264) CONTROLLER.md states when a job's budget can no longer finish its schedule and every point that ends or refuses it, and the status wording that names the cause. GIT-STARTUP.md, POLLING.md and E2E-QUALIFICATION.md describe the requeued job after a shutdown grace ending incomplete with one status, and recovery's resume refusal. Co-Authored-By: Claude Opus 5.5 --- reviewer/CONTROLLER.md | 32 +++++++++++++++++++++++++++----- reviewer/E2E-QUALIFICATION.md | 26 +++++++++++++++----------- reviewer/GIT-STARTUP.md | 13 ++++++++++--- reviewer/POLLING.md | 6 +++++- 4 files changed, 57 insertions(+), 20 deletions(-) diff --git a/reviewer/CONTROLLER.md b/reviewer/CONTROLLER.md index 8b03a9b8..f96f526a 100644 --- a/reviewer/CONTROLLER.md +++ b/reviewer/CONTROLLER.md @@ -144,6 +144,28 @@ budget. It does no inference, is never called by configuration comments, and rejects expired/exhausted, cancelled, superseded or paused jobs. Unknown provider work remains charged; this recovery cannot create additional session capacity. +A job's captured session budget equals its schedule: one session per provider +and round. Every charged session without a validated artifact (an +interrupted, failed or shutdown-ended phase) therefore leaves the schedule one +session short, and the job can never complete. `sessionBudgetFits` states that +rule from the job's configuration and checkpoint exactly as the engine plans +phases. Such a job spends no further provider session: + +- The polling controller ends a retained queued or quota-waiting job that does + not fit as `incomplete_expired` before it restores evidence, builds a + runtime or claims it. +- The claim ends it the same way before any lease, quota wait or session, so + a direct `RunStopped` caller cannot spend one either. +- The engine refuses to start a phase of a schedule that does not fit + (`ErrSessionBudgetExhausted`). +- `ResumeUnavailable` refuses to requeue it with `ErrSessionBudgetExhausted` + and leaves its state unchanged. This covers an `auth_failed` or + `provider_unavailable` job after a charged failure and the job that + [operator recovery](GIT-STARTUP.md#stopped-provider-recovery) releases. + +A before-work refund keeps the budget whole, so `quota_wait` and other +refunded attempts still fit. + A lease claim returns its captured job/checkpoint from the claiming transaction. There is no fallible post-commit read that can hide the committed lease nonce. `RunStopped` uses that atomic result directly. Superseded/cancelled generations @@ -177,15 +199,15 @@ published states and their leading words are: | Job state | Status | | --- | --- | | `quota_wait` | INCOMPLETE: daily quota exhausted; resumes by itself until the configured quota wait expires | -| `auth_failed` | INCOMPLETE: provider authentication unavailable; stopped until an operator repairs it and resumes the job | -| `provider_unavailable` | INCOMPLETE: a selected provider is unavailable; stopped until an operator resumes the job | +| `auth_failed` | INCOMPLETE: provider authentication unavailable; stopped until an operator repairs it and resumes the job, or, when its budget no longer fits, cannot be resumed | +| `provider_unavailable` | INCOMPLETE: a selected provider is unavailable; stopped until an operator resumes the job, or, when its budget no longer fits, cannot be resumed | | `invalid_output` | INCOMPLETE: provider output failed validation; ended without a review | -| `incomplete_expired` | INCOMPLETE: expired before every required phase completed | +| `incomplete_expired` | INCOMPLETE: expired before every required phase completed, or the captured session budget is exhausted, naming the charged sessions without a result | | `cancelled` | CANCELLED: the request was cancelled before the job completed | | `capture_refused` | INCOMPLETE: missing history, ambiguous merge base or over the 128-file inventory limit; the request ended without a review | -The text is fixed controller wording keyed by the durable state and the -captured configuration. It passes through `safeText` and known-secret +The text is fixed controller wording keyed by the durable state, the captured +configuration and the checkpoint's session counts. It passes through `safeText` and known-secret redaction like every document, names the reviewed revisions and the completed phase count, and never includes a finding. `queued` and `running` are transient. `blocked` is the operator qualification gate, not a review diff --git a/reviewer/E2E-QUALIFICATION.md b/reviewer/E2E-QUALIFICATION.md index b75e2e26..1b302b80 100644 --- a/reviewer/E2E-QUALIFICATION.md +++ b/reviewer/E2E-QUALIFICATION.md @@ -94,7 +94,7 @@ ledger, and a restore probe. | `review` | PR 1 automatic request through capture, four sessions and one confirmed COMMENT. | 1 / 4 | 1 | both usable | | `isolation` | PR 1 (automatic) and PR 2 (owner comment) pending together. | 2 / 8 | 2 | both usable | | `close-capture` | Close the whole service after capture and admission, inside the first session's host check, before any VM launch. Reopen and finish. | 1 / 4 | 1 | both usable | -| `close-provider` | Close the whole service while the first Codex session waits for a withheld inference response. With the shutdown grace, reopen and run the requeued job. | 1 / 4 (grace) or 1 / 1 (fenced) | 1 (grace) or lifecycle order only (fenced) | both usable (grace), or Codex fenced and Claude usable | +| `close-provider` | Close the whole service while the first Codex session waits for a withheld inference response. With the shutdown grace, reopen: the requeued job ends incomplete and publishes one status. | 1 / 1 | 1 (grace) or lifecycle order only (fenced) | both usable (grace), or Codex fenced and Claude usable | | `close-provider-reopen` | A second process attached to the root retained by `close-provider` (`SFL_E2E_ATTACH`), as a restarted service. | 0 / 0 | 1 (retained) | Codex fenced, Claude usable | | `close-provider-recover` | Instead of `close-provider-reopen`: a restarted process recovers the retained attempt through the operator API, publishes PR 1's `auth_failed` status, then reviews and publishes the next request. | 1 / 4 | 2 (1 retained) | both usable (Codex re-provisioned) | | `close-publish` | The fake GitHub commits the review POST but withholds its response; close the whole service; reopen and reconcile. | 1 / 4 | 1 | both usable | @@ -108,16 +108,20 @@ charged session and no artifact, and Codex auth usable. The stopped audit must record the auth as exported and returned, end at the result record's validation and fail as `cancelled`. The job must record that launch as its attempt, with its complete stopped evidence and the source receipt its result -returned. The case then reopens. The requeued job keeps that charged session, -so its captured four-session budget leaves three. It reruns Codex R1, Claude -R1 and Codex R2, each in one completed launch holding the canned report, and -expires (`incomplete_expired`) before Claude R2, unleased, with no intent, -POST or remote review. The case therefore expects one source unit, four -native units (the interrupted launch and three reruns), both auth stores -usable and only the qualified routes. The interrupted launch is the only -launch outside the job's artifacts. No host run has taken this branch. Its -earlier form expected a published review after the reopen, which the -captured budget cannot allow. +returned. The case then reopens. The requeued job keeps that charged session +without a result, so its captured four-session budget can no longer finish +the schedule. The reopened controller ends it `incomplete_expired` before +restoring evidence or starting a session, unleased, with one charged session +and no artifact. It publishes exactly one status for PR 1, which must be +confirmed, bound to the job and checkpoint, carry the document production +renders for that state and checkpoint (the exhausted-budget wording) and +match its one inline-free remote COMMENT on the head (`e2eStatusVerdicts`). +There is no review intent and no other write. The case therefore expects one +source unit, one native unit (the interrupted launch, the only launch), both +auth stores usable, and the publishing route set: the qualified routes plus +the publisher's review POST and receipt read. No host run has taken this +branch. Its earlier forms expected a published review, then three reruns +that expired, which spent sessions on a certain incomplete. If the guest does not return within the grace, `Close` cancels as before and the case takes the fenced branch below. Only that branch retains a root for diff --git a/reviewer/GIT-STARTUP.md b/reviewer/GIT-STARTUP.md index 00f0bbbc..925d38de 100644 --- a/reviewer/GIT-STARTUP.md +++ b/reviewer/GIT-STARTUP.md @@ -156,7 +156,11 @@ durable stopped return. A refresh-uncertain return is still fenced. A session that fails after the drain reports `cancelled`, so the stopped job is requeued with its artifacts and consumed budget; a latched route or response refusal, or exhaustion, keeps its own classification. A session that completes within -the grace keeps its artifact, and the engine starts no further phase. +the grace keeps its artifact, and the engine starts no further phase. A +requeued job whose interrupted session left no artifact can no longer finish +its schedule, so the next controller ends it `incomplete_expired` without +another session and publishes one INCOMPLETE status naming the exhausted +budget ([CONTROLLER.md](CONTROLLER.md)). The ten seconds cover the guest's bounded cleanup after its client ends (at most twelve seconds, normally well under one) and its result record. If the @@ -227,8 +231,11 @@ becomes `auth_failed` when the exported auth was lost and retired. It becomes that a cancellation or newer request fenced while it held the lease stays `cancelled` or `stale`; the result and the receipt report that final state. None of these states is a review. Only an explicit `ResumeUnavailable` can -requeue the same unfenced request, within its remaining session budget. -Other requests proceed immediately. +requeue the same unfenced request, and only while its remaining session +budget can still finish the schedule. The recovered session stays charged +without a result, so for a job with no spare session, which the default +configuration is, it refuses with `ErrSessionBudgetExhausted` and the status +says the job cannot be resumed. Other requests proceed immediately. A retired provider has no canonical auth. Its sessions fail before work as `auth_failed` until the operator establishes new auth outside the reviewer. diff --git a/reviewer/POLLING.md b/reviewer/POLLING.md index 6ac49dd6..e41d9614 100644 --- a/reviewer/POLLING.md +++ b/reviewer/POLLING.md @@ -169,7 +169,11 @@ must prove stopped recovery. Known-stopped work interrupted only by service shutdown may return to `queued` with its exact artifacts and consumed budget. A phase that completes during the grace keeps its artifact, and the engine starts no further phase. Owner -cancellation, generation changes and uncertain stop cannot use that path. Factory errors become durable +cancellation, generation changes and uncertain stop cannot use that path. +Before restoring evidence for a retained job, the controller ends one whose +remaining session budget can no longer finish its schedule as +`incomplete_expired`, with no runtime, claim or session +([CONTROLLER.md](CONTROLLER.md)). Factory errors become durable unavailable/blocked states. A failed qualification check cancels active work; all admitted paths, including discovery/restoration and pre-work qualification, share one finalization step. An otherwise current queued or cancelled job