From e8c0746e6ee117801de5ca28fda4f5d90947989c Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Wed, 7 Oct 2026 13:51:24 +1100 Subject: [PATCH] feat(skillgen): run the Harness on GLM 5.3 through OpenCode Go Skillgen runs up to 96 model calls per job. GLM 5.3 on OpenCode Go lists at $1.40 per million input tokens and $4.40 per million output tokens, well under Sonnet 4.6, so each run costs less. The gateway gains an opencode-go route with the same guarantees as the Anthropic and Google routes: Worker-held credential, forced model, 8,192 output token cap, hosted execution blocked, and the job call limit. Extra choices and service tiers are removed because both raise the price. The pinned OpenCode 1.18.31 bootstrap lists opencode-go/glm-5.3 from its bundled snapshot with model fetches disabled and no network, so the runner needs no explicit provider declaration. --- docs/runbooks/skill-harness-proof.md | 15 ++- workers/skill-harness/.dev.vars.example | 1 + workers/skill-harness/src/gateway.ts | 76 ++++++++++++--- workers/skill-harness/src/index.ts | 6 +- workers/skill-harness/test/gateway.test.ts | 93 +++++++++++++++++++ .../skill-harness/worker-configuration.d.ts | 9 +- workers/skill-harness/wrangler.jsonc | 2 +- 7 files changed, 178 insertions(+), 24 deletions(-) diff --git a/docs/runbooks/skill-harness-proof.md b/docs/runbooks/skill-harness-proof.md index 3f10919f..c3477267 100644 --- a/docs/runbooks/skill-harness-proof.md +++ b/docs/runbooks/skill-harness-proof.md @@ -1,10 +1,10 @@ # Skill generation on Cloudflare -This proof runs `skilld-harness` against an exact public npm package version. +This proof runs `skilld-harness` against an exact public [npm](https://npmjs.com) package version. It uses an existing Skill as the update baseline. The result includes generated files, output checks, review findings, source attempts, and model call counts. -The proof produces a file bundle. GitHub tag handling and pull request publication follow this runtime proof. +The proof produces a file bundle. [GitHub](https://github.com) tag handling and pull request publication follow this runtime proof. ## Deploy the proof @@ -18,11 +18,17 @@ To deploy a branch by hand, dispatch the workflow with `target=harness-proof`. gh workflow run deploy-cloudflare.yml --ref BRANCH --field target=harness-proof ``` -Configure `ANTHROPIC_API_KEY` and `PROOF_TOKEN` as Worker secrets. +By default, OpenCode runs GLM 5.3 through OpenCode Go: `PROVIDER=opencode-go` and `MODEL=glm-5.3`. +Configure `OPENCODE_API_KEY` and `PROOF_TOKEN` as Worker secrets. +For Anthropic, set `PROVIDER=anthropic`, a supported model, and `ANTHROPIC_API_KEY`. For Google, set `PROVIDER=google`, a supported model, and `GOOGLE_GENERATIVE_AI_API_KEY`. Use a temporary proof token. Do not put either secret in a repository file or container environment. The container receives a placeholder. Its outbound gateway adds the model credential inside the Worker. +The runner disables model list fetches. The model must exist in the model snapshot of the pinned OpenCode. +After a bootstrap update, run `OPENCODE_DISABLE_MODELS_FETCH=true opencode models opencode-go` with that OpenCode version. +If the model is missing, declare it in `openCodeConfig` in `runner/run.mjs`. + ## Run an update Set `SKILL_HARNESS_PROOF_TOKEN` in the calling shell. @@ -52,6 +58,7 @@ Durable Object alarms check progress and enforce the fifteen minute deadline. The proof allows one active job and 96 model calls. Generation targets 35 turns. Independent review targets 20 turns. These turn targets guide the model. The gateway enforces the total call limit. +OpenCode Go calls allow 8,192 output tokens. The gateway caps `max_tokens` and `max_completion_tokens`. Anthropic calls allow 8,192 output tokens, including a 2,048-token thinking budget. Google calls allow 4,096 output tokens. It limits each request to 512 KiB and each result to 1 MiB. @@ -64,6 +71,8 @@ These bounds constrain usage. They do not constitute a measured dollar budget. The outbound gateway allows HTTPS npm retrieval, GitHub source archives, and the configured model provider. It blocks other destinations, hosted model tools, credential overrides, and model conversation reuse. +For OpenCode Go, only chat completions pass. Every tool must have the `function` type. +The gateway removes extra choices and service tier requests, because both raise the price. External documentation requests are blocked. Source files must carry the evidence for this proof. If generation fails, read `state.result.detail` and `state.result.generation` in the saved result. diff --git a/workers/skill-harness/.dev.vars.example b/workers/skill-harness/.dev.vars.example index 08d14ad2..e284ab89 100644 --- a/workers/skill-harness/.dev.vars.example +++ b/workers/skill-harness/.dev.vars.example @@ -1,6 +1,7 @@ GOOGLE_GENERATIVE_AI_API_KEY=example PROOF_TOKEN=example ANTHROPIC_API_KEY=example +OPENCODE_API_KEY=example GITHUB_APP_ID=example GITHUB_APP_PRIVATE_KEY_PKCS8=example GITHUB_APP_WEBHOOK_SECRET=example diff --git a/workers/skill-harness/src/gateway.ts b/workers/skill-harness/src/gateway.ts index c5e09320..61e29e74 100644 --- a/workers/skill-harness/src/gateway.ts +++ b/workers/skill-harness/src/gateway.ts @@ -1,9 +1,32 @@ import { MAX_REQUEST_BYTES, parseJson, readBoundedBody } from './contracts' +export type ModelProvider = 'google' | 'anthropic' | 'opencode-go' + +const OPENCODE_GO_CHAT_URL = 'https://opencode.ai/zen/go/v1/chat/completions' +const OPENCODE_GO_MAX_OUTPUT_TOKENS = 8192 +// Web search, hosted plugins, live search, and remote MCP run on the provider at a separate cost. +const OPENAI_COMPATIBLE_HOSTED_FIELDS = ['web_search_options', 'plugins', 'search_parameters', 'mcp_servers'] + +async function readModelInput(request: Request): Promise<{ _tag: 'Ok', value: Record } | { _tag: 'Err', response: Response }> { + const body = await readBoundedBody(request, MAX_REQUEST_BYTES) + if (body === undefined) + return { _tag: 'Err', response: Response.json({ code: 'REQUEST_TOO_LARGE' }, { status: 413 }) } + const parsed = parseJson(body) + if (parsed._tag === 'Err' || !parsed.value || typeof parsed.value !== 'object' || Array.isArray(parsed.value)) + return { _tag: 'Err', response: Response.json({ code: 'INVALID_MODEL_REQUEST' }, { status: 400 }) } + return { _tag: 'Ok', value: parsed.value as Record } +} + +function openCodeGoOutputLimit(value: unknown): number { + return typeof value === 'number' && Number.isInteger(value) && value > 0 + ? Math.min(value, OPENCODE_GO_MAX_OUTPUT_TOKENS) + : OPENCODE_GO_MAX_OUTPUT_TOKENS +} + /** Only npm package retrieval and the configured model can leave the sandbox. */ export async function forwardSandboxRequest( request: Request, - options: { provider: 'google' | 'anthropic', model: string, apiKey: string, consumeModelCall: () => Promise, fetch: typeof fetch }, + options: { provider: ModelProvider, model: string, apiKey: string, consumeModelCall: () => Promise, fetch: typeof fetch }, ): Promise { const fetchClient = options.fetch const url = new URL(request.url) @@ -12,13 +35,10 @@ export async function forwardSandboxRequest( const modelPath = `/v1beta/models/${options.model}:streamGenerateContent` if (options.provider === 'google' && url.hostname === 'generativelanguage.googleapis.com' && url.pathname === modelPath && request.method === 'POST') { - const body = await readBoundedBody(request, MAX_REQUEST_BYTES) - if (body === undefined) - return Response.json({ code: 'REQUEST_TOO_LARGE' }, { status: 413 }) - const parsed = parseJson(body) - if (parsed._tag === 'Err' || !parsed.value || typeof parsed.value !== 'object' || Array.isArray(parsed.value)) - return Response.json({ code: 'INVALID_MODEL_REQUEST' }, { status: 400 }) - const input = parsed.value as Record + const parsed = await readModelInput(request) + if (parsed._tag === 'Err') + return parsed.response + const input = parsed.value if (input.cachedContent) return Response.json({ code: 'MODEL_DENIED' }, { status: 403 }) if (Array.isArray(input.tools) && input.tools.some(tool => @@ -39,13 +59,10 @@ export async function forwardSandboxRequest( } if (options.provider === 'anthropic' && url.hostname === 'api.anthropic.com' && url.pathname === '/v1/messages' && request.method === 'POST') { - const body = await readBoundedBody(request, MAX_REQUEST_BYTES) - if (body === undefined) - return Response.json({ code: 'REQUEST_TOO_LARGE' }, { status: 413 }) - const parsed = parseJson(body) - if (parsed._tag === 'Err' || !parsed.value || typeof parsed.value !== 'object' || Array.isArray(parsed.value)) - return Response.json({ code: 'INVALID_MODEL_REQUEST' }, { status: 400 }) - const input = parsed.value as Record + const parsed = await readModelInput(request) + if (parsed._tag === 'Err') + return parsed.response + const input = parsed.value if (input.mcp_servers || (Array.isArray(input.tools) && input.tools.some(tool => !tool || typeof tool !== 'object' || Array.isArray(tool) || ('type' in tool && tool.type !== 'custom'), @@ -67,6 +84,35 @@ export async function forwardSandboxRequest( }) } + if (options.provider === 'opencode-go' && `${url.origin}${url.pathname}` === OPENCODE_GO_CHAT_URL && request.method === 'POST') { + const parsed = await readModelInput(request) + if (parsed._tag === 'Err') + return parsed.response + const input = parsed.value + if (OPENAI_COMPATIBLE_HOSTED_FIELDS.some(field => field in input) || (input.tools !== undefined && (!Array.isArray(input.tools) || input.tools.some(tool => + !tool || typeof tool !== 'object' || Array.isArray(tool) || tool.type !== 'function', + )))) { + return Response.json({ code: 'MODEL_TOOLS_DENIED' }, { status: 403 }) + } + if (!await options.consumeModelCall()) + return Response.json({ code: 'MODEL_CALL_LIMIT' }, { status: 403 }) + // Extra choices multiply output cost. A priority tier raises the price. + const standardInput = { ...input } + delete standardInput.n + delete standardInput.service_tier + return fetchClient(OPENCODE_GO_CHAT_URL, { + method: 'POST', + headers: { 'content-type': 'application/json', 'authorization': `Bearer ${options.apiKey}` }, + body: JSON.stringify({ + ...standardInput, + model: options.model, + max_tokens: openCodeGoOutputLimit(input.max_tokens), + ...('max_completion_tokens' in input ? { max_completion_tokens: openCodeGoOutputLimit(input.max_completion_tokens) } : {}), + }), + redirect: 'manual', + }) + } + const sourceHost = url.hostname === 'registry.npmjs.org' || url.hostname === 'codeload.github.com' if (sourceHost && request.method === 'GET') { return fetchClient(url, { diff --git a/workers/skill-harness/src/index.ts b/workers/skill-harness/src/index.ts index 8862fdc4..48224f1f 100644 --- a/workers/skill-harness/src/index.ts +++ b/workers/skill-harness/src/index.ts @@ -1,4 +1,5 @@ import type { ProofInput, ProofResult, ProofState } from './contracts' +import type { ModelProvider } from './gateway' import { Files, SandboxFileError } from '@cloudflare/sandbox' import { DurableObject, WorkerEntrypoint } from 'cloudflare:workers' import { JOB_TIMEOUT_MS, MAX_MODEL_CALLS, MAX_REQUEST_BYTES, MAX_RESULT_BYTES, parseJson, parseProofInput, parseProofResult, readBoundedBody } from './contracts' @@ -8,7 +9,9 @@ import { startOutsideLock } from './startup' export { GithubJobs } from './github-jobs' -function credentials(env: { PROVIDER: string, ANTHROPIC_API_KEY?: string, GOOGLE_GENERATIVE_AI_API_KEY?: string }): { provider: 'google' | 'anthropic', apiKey: string } | undefined { +function credentials(env: { PROVIDER: string, ANTHROPIC_API_KEY?: string, GOOGLE_GENERATIVE_AI_API_KEY?: string, OPENCODE_API_KEY?: string }): { provider: ModelProvider, apiKey: string } | undefined { + if (env.PROVIDER === 'opencode-go' && env.OPENCODE_API_KEY) + return { provider: 'opencode-go', apiKey: env.OPENCODE_API_KEY } if (env.PROVIDER === 'anthropic' && env.ANTHROPIC_API_KEY) return { provider: 'anthropic', apiKey: env.ANTHROPIC_API_KEY } if (env.PROVIDER === 'google' && env.GOOGLE_GENERATIVE_AI_API_KEY) @@ -70,6 +73,7 @@ export class SkillSandbox extends DurableObject { PROVIDER: this.env.PROVIDER, ANTHROPIC_API_KEY: 'sandbox-placeholder', GOOGLE_GENERATIVE_AI_API_KEY: 'sandbox-placeholder', + OPENCODE_API_KEY: 'sandbox-placeholder', NODE_EXTRA_CA_CERTS: '/etc/cloudflare/certs/cloudflare-containers-ca.crt', SSL_CERT_FILE: '/etc/cloudflare/certs/cloudflare-containers-ca.crt', }, diff --git a/workers/skill-harness/test/gateway.test.ts b/workers/skill-harness/test/gateway.test.ts index bb491d00..5b35eb6f 100644 --- a/workers/skill-harness/test/gateway.test.ts +++ b/workers/skill-harness/test/gateway.test.ts @@ -19,7 +19,100 @@ function modelRequest(body: unknown, model = 'gemini-3.8-flash') { }) } +function openCodeGo() { + return { ...fixture(), provider: 'opencode-go' as const, model: 'glm-5.3' } +} + +function openCodeGoRequest(body: unknown, url = 'https://opencode.ai/zen/go/v1/chat/completions?key=untrusted') { + return new Request(url, { + method: 'POST', + headers: { 'authorization': 'Bearer sandbox-placeholder', 'content-type': 'application/json', 'x-opencode-session': 'untrusted', 'cookie': 'private=cookie' }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }) +} + describe('sandbox credential gateway', () => { + it('routes OpenCode Go with Worker credentials and enforced model limits', async () => { + const options = openCodeGo() + const response = await forwardSandboxRequest(openCodeGoRequest({ + model: 'expensive-model', + max_tokens: 32000, + max_completion_tokens: 131072, + n: 4, + service_tier: 'priority', + stream: true, + messages: [], + tools: [{ type: 'function', function: { name: 'read', parameters: {} } }], + }), options) + expect(response.status).toBe(200) + const [url, forwarded] = options.fetch.mock.calls[0]! + expect(String(url)).toBe('https://opencode.ai/zen/go/v1/chat/completions') + expect(forwarded?.redirect).toBe('manual') + expect([...new Headers(forwarded?.headers).entries()]).toEqual([['authorization', 'Bearer worker-only-secret'], ['content-type', 'application/json']]) + const body = JSON.parse(String(forwarded?.body)) + expect(body).toMatchObject({ model: 'glm-5.3', max_tokens: 8192, max_completion_tokens: 8192, stream: true, tools: [{ type: 'function' }] }) + expect(body).not.toHaveProperty('n') + expect(body).not.toHaveProperty('service_tier') + expect(options.consumeModelCall).toHaveBeenCalledOnce() + }) + + it.each([ + [{ max_tokens: 1000 }, { max_tokens: 1000 }], + [{}, { max_tokens: 8192 }], + [{ max_tokens: 'unbounded' }, { max_tokens: 8192 }], + [{ max_completion_tokens: 500 }, { max_tokens: 8192, max_completion_tokens: 500 }], + ])('bounds OpenCode Go output for %j', async (limits, expected) => { + const options = openCodeGo() + await forwardSandboxRequest(openCodeGoRequest({ messages: [], ...limits }), options) + const [, forwarded] = options.fetch.mock.calls[0]! + expect(JSON.parse(String(forwarded?.body))).toMatchObject(expected) + }) + + it.each([ + { tools: [{ type: 'web_search', web_search: { enable: true } }] }, + { tools: [{ type: 'retrieval', retrieval: {} }] }, + { tools: [{ function: { name: 'read' } }] }, + { tools: 'read' }, + { web_search_options: {} }, + { plugins: [{ id: 'web' }] }, + { search_parameters: { mode: 'on' } }, + { mcp_servers: [{ url: 'https://external.example' }] }, + ])('blocks OpenCode Go hosted execution %j', async (body) => { + const options = openCodeGo() + const response = await forwardSandboxRequest(openCodeGoRequest({ messages: [], ...body }), options) + expect(response.status).toBe(403) + expect(options.fetch).not.toHaveBeenCalled() + expect(options.consumeModelCall).not.toHaveBeenCalled() + }) + + it.each(['https://opencode.ai/zen/go/v1/responses', 'https://opencode.ai/zen/v1/chat/completions', 'https://opencode.ai/zen/go/v1/models'])('denies OpenCode route %s', async (url) => { + const options = openCodeGo() + const response = await forwardSandboxRequest(openCodeGoRequest({ messages: [] }, url), options) + expect(response.status).toBe(403) + expect(options.fetch).not.toHaveBeenCalled() + }) + + it('denies OpenCode Go while another provider is configured', async () => { + const options = { ...fixture(), provider: 'anthropic' as const } + const response = await forwardSandboxRequest(openCodeGoRequest({ messages: [] }), options) + expect(response.status).toBe(403) + expect(options.fetch).not.toHaveBeenCalled() + }) + + it('stops OpenCode Go calls after the job allowance', async () => { + const options = { ...openCodeGo(), consumeModelCall: vi.fn(async () => false) } + const response = await forwardSandboxRequest(openCodeGoRequest({ messages: [] }), options) + expect(response.status).toBe(403) + expect(options.fetch).not.toHaveBeenCalled() + }) + + it.each(['{', '[]'])('rejects invalid OpenCode Go JSON %s', async (body) => { + const options = openCodeGo() + const response = await forwardSandboxRequest(openCodeGoRequest(body), options) + expect(response.status).toBe(400) + expect(options.fetch).not.toHaveBeenCalled() + }) + it('routes Anthropic with Worker credentials and enforced model limits', async () => { const options = { ...fixture(), provider: 'anthropic' as const, model: 'claude-sonnet-4-6' } const response = await forwardSandboxRequest(new Request('https://api.anthropic.com/v1/messages', { diff --git a/workers/skill-harness/worker-configuration.d.ts b/workers/skill-harness/worker-configuration.d.ts index 7f3c9795..cd98dc29 100644 --- a/workers/skill-harness/worker-configuration.d.ts +++ b/workers/skill-harness/worker-configuration.d.ts @@ -1,12 +1,13 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types --env-file=.dev.vars.example --include-runtime=false --env-interface=HarnessEnv` (hash: 388b9853d3dcec863631966608d9909d) +// Generated by Wrangler by running `wrangler types --env-file=.dev.vars.example --include-runtime=false --env-interface=HarnessEnv` (hash: ed403e612d246090ec412d137bdb3cc9) interface __BaseEnv_HarnessEnv { - PROVIDER: "anthropic"; - MODEL: "claude-sonnet-4-6"; + PROVIDER: "opencode-go"; + MODEL: "glm-5.3"; SKILLD_SITE_URL: "https://skilld.dev"; GOOGLE_GENERATIVE_AI_API_KEY: string; PROOF_TOKEN: string; ANTHROPIC_API_KEY: string; + OPENCODE_API_KEY: string; GITHUB_APP_ID: string; GITHUB_APP_PRIVATE_KEY_PKCS8: string; GITHUB_APP_WEBHOOK_SECRET: string; @@ -26,5 +27,5 @@ type StringifyValues> = { [Binding in keyof EnvType]: EnvType[Binding] extends string ? EnvType[Binding] : string; }; declare namespace NodeJS { - interface ProcessEnv extends StringifyValues> {} + interface ProcessEnv extends StringifyValues> {} } diff --git a/workers/skill-harness/wrangler.jsonc b/workers/skill-harness/wrangler.jsonc index cab7a0e2..9dc5ee3f 100644 --- a/workers/skill-harness/wrangler.jsonc +++ b/workers/skill-harness/wrangler.jsonc @@ -6,7 +6,7 @@ "compatibility_flags": ["nodejs_compat"], "workers_dev": true, "observability": { "enabled": true }, - "vars": { "PROVIDER": "anthropic", "MODEL": "claude-sonnet-4-6", "SKILLD_SITE_URL": "https://skilld.dev" }, + "vars": { "PROVIDER": "opencode-go", "MODEL": "glm-5.3", "SKILLD_SITE_URL": "https://skilld.dev" }, "containers": [{ "class_name": "SkillSandbox", "scheduling_policy": "durable_object",