Skip to content

Commit e6f2347

Browse files
committed
feat: add byte estimation for request messages and enhance compaction logic
1 parent 5727a46 commit e6f2347

3 files changed

Lines changed: 232 additions & 23 deletions

File tree

‎packages/core/src/common/compaction.ts‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
import type { ChatCompletionMessageParam } from "openai/resources/chat/completions";
22
import type { SessionMessage } from "../session";
33

4+
export const MAX_REQUEST_BODY_BYTES = 48 * 1024 * 1024;
5+
46
export type CompactBudget = {
57
output: number;
68
input: number;
79
threshold: number;
10+
byteThreshold: number;
811
retain: number;
912
};
1013

@@ -29,12 +32,19 @@ export function compactBudget(
2932
const input = window - output;
3033
const threshold = Math.min(configuredThreshold, input - output);
3134
if (threshold <= 0) throw new Error("The context window leaves no room for conversation input.");
35+
// Reserve room for tools and request options without measuring the entire wire payload.
36+
const byteThreshold = MAX_REQUEST_BODY_BYTES - window * 2;
37+
if (byteThreshold <= 0) throw new Error("The context window leaves no room in the request size budget.");
3238
const retain = retention.retainTokens ?? Math.floor(input * (retention.retainRatio ?? 0.16));
3339
if (retain >= threshold)
3440
throw new Error(
3541
`The retention budget (${retain}) must be less than the compaction threshold (${threshold}). Adjust retainRatio or retainTokens.`
3642
);
37-
return { output, input, threshold, retain };
43+
return { output, input, threshold, byteThreshold, retain };
44+
}
45+
46+
export function estimateRequestBytes(messages: readonly ChatCompletionMessageParam[]): number {
47+
return Buffer.byteLength(JSON.stringify(messages), "utf8");
3848
}
3949

4050
/** An estimate, not a tokenizer. Never charge binary image encodings as text. */

‎packages/core/src/session.ts‎

Lines changed: 44 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ import {
8282
import {
8383
compactBudget,
8484
compactPrefixes,
85+
estimateRequestBytes,
8586
estimateRequestTokens,
8687
imageOffloadCount,
8788
isContextOverflow,
@@ -2004,7 +2005,7 @@ ${agentInstructions}
20042005
anchor.hashes.length <= hashes.length &&
20052006
anchor.hashes.every((hash, index) => hash === hashes[index]);
20062007
const tokens = anchored ? anchor.tokens + estimate - anchor.estimate : estimate;
2007-
return { route, hashes, estimate, tokens };
2008+
return { route, hashes, estimate, tokens, bytes: estimateRequestBytes(messages) };
20082009
}
20092010

20102011
/** Shared file-reference recovery; it never repeats tool execution. */
@@ -2066,23 +2067,34 @@ ${agentInstructions}
20662067
const budget = this.getCompactionBudget(model);
20672068
let context = await this.prepareContext(this.listSessionMessages(sessionId), model, thinkingEnabled);
20682069
let measurement = this.measureRequest(sessionId, context.messages, model);
2069-
for (let attempt = 0; measurement.tokens >= budget.threshold && attempt < 2; attempt++) {
2070-
if (attempt === 0) {
2071-
const notice = this.buildAssistantMessage(sessionId, "The conversation is getting long, compacting...", null);
2072-
notice.meta = { asThinking: true };
2073-
this.onAssistantMessage(notice, false);
2074-
}
2075-
const changed = await this.compactSessionOnce(sessionId, signal, budget.retain);
2076-
if (!changed) break;
2077-
context = await this.prepareContext(this.listSessionMessages(sessionId), model, thinkingEnabled);
2078-
measurement = this.measureRequest(sessionId, context.messages, model);
2079-
}
2080-
if (measurement.tokens >= budget.threshold) {
2081-
throw new Error("The conversation is too large to compact safely. Shorten the input or start a new session.");
2082-
}
2070+
let compactAttempts = 0;
2071+
let recovering = false;
20832072
let overflowRecovered = false;
20842073
for (;;) {
20852074
signal.throwIfAborted();
2075+
const needsCompaction = () =>
2076+
measurement.bytes > budget.byteThreshold || (!recovering && measurement.tokens >= budget.threshold);
2077+
while (needsCompaction() && compactAttempts < 2) {
2078+
const attempt = compactAttempts++;
2079+
if (attempt === 0) {
2080+
const notice = this.buildAssistantMessage(sessionId, "The conversation is getting long, compacting...", null);
2081+
notice.meta = { asThinking: true };
2082+
this.onAssistantMessage(notice, false);
2083+
}
2084+
const oversized = measurement.bytes > budget.byteThreshold;
2085+
const retain = attempt > 0 && oversized ? 0 : budget.retain;
2086+
const changed = await this.compactSessionOnce(sessionId, signal, retain);
2087+
if (!changed) {
2088+
// A token-based retention budget can leave all image-heavy history untouched.
2089+
if (attempt === 0 && oversized && retain > 0) continue;
2090+
break;
2091+
}
2092+
context = await this.prepareContext(this.listSessionMessages(sessionId), model, thinkingEnabled);
2093+
measurement = this.measureRequest(sessionId, context.messages, model);
2094+
}
2095+
if (needsCompaction()) {
2096+
throw new Error("The conversation is too large to compact safely. Shorten the input or start a new session.");
2097+
}
20862098
try {
20872099
const response = await this.sendPreparedCompletion(
20882100
context.sources,
@@ -2121,6 +2133,7 @@ ${agentInstructions}
21212133
measurement = this.measureRequest(sessionId, context.messages, model);
21222134
if (measurement.tokens > budget.input)
21232135
throw new Error("The conversation still exceeds the model context window.");
2136+
recovering = true;
21242137
}
21252138
}
21262139
}
@@ -2175,11 +2188,15 @@ ${agentInstructions}
21752188
model,
21762189
thinkingEnabled
21772190
);
2191+
const summaryMessages: ChatCompletionMessageParam[] = [
2192+
...candidate.messages,
2193+
{ role: "user", content: getCompactPrompt() },
2194+
];
21782195
const cost = estimateRequestTokens(
2179-
[...candidate.messages, { role: "user", content: getCompactPrompt() }],
2196+
summaryMessages,
21802197
getTools(this.getPromptToolOptions(), this.mcpToolDefinitions)
21812198
);
2182-
if (cost <= budget.input) {
2199+
if (cost <= budget.input && estimateRequestBytes(summaryMessages) <= budget.byteThreshold) {
21832200
selected = prefix;
21842201
replay = candidate;
21852202
break;
@@ -2274,7 +2291,16 @@ ${agentInstructions}
22742291
message.id === firstSelectedId ? [summary] : selectedIds.has(message.id) ? [] : [message]
22752292
);
22762293
const next = await this.prepareContext(replacement, model, thinkingEnabled);
2277-
if (estimateRequestTokens(next.messages) >= estimateRequestTokens(context.messages)) {
2294+
const tokens = estimateRequestTokens(
2295+
next.messages,
2296+
getTools(this.getPromptToolOptions(), this.mcpToolDefinitions)
2297+
);
2298+
const previousBytes = estimateRequestBytes(context.messages);
2299+
const reduced =
2300+
previousBytes > budget.byteThreshold
2301+
? estimateRequestBytes(next.messages) < previousBytes && tokens <= budget.input
2302+
: estimateRequestTokens(next.messages) < estimateRequestTokens(context.messages);
2303+
if (!reduced) {
22782304
throw new Error("Compaction did not reduce the conversation size. The original conversation was kept.");
22792305
}
22802306
signal.throwIfAborted();
@@ -2288,10 +2314,6 @@ ${agentInstructions}
22882314
});
22892315
this.saveSessionMessages(sessionId, committed);
22902316
this.requestUsageAnchors.delete(sessionId);
2291-
const tokens = estimateRequestTokens(
2292-
next.messages,
2293-
getTools(this.getPromptToolOptions(), this.mcpToolDefinitions)
2294-
);
22952317
this.updateSessionEntry(sessionId, (entry) => ({ ...entry, activeTokens: tokens, updateTime: now }));
22962318
return true;
22972319
}

‎packages/core/src/tests/compaction.test.ts‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ import { getCompactPrompt, frameCompactSummary } from "../prompt";
1111
import {
1212
compactBudget,
1313
compactPrefixes as selectPrefixes,
14+
estimateRequestBytes,
1415
estimateRequestTokens,
1516
imageOffloadCount,
1617
isContextOverflow,
18+
MAX_REQUEST_BODY_BYTES,
1719
} from "../common/compaction";
1820

1921
const originalHome = process.env.HOME;
@@ -775,3 +777,178 @@ test("automatic compaction honors the configured tail budget", async () => {
775777
}
776778
assert.deepEqual(retained, [2, 4]);
777779
});
780+
781+
function imageMessage(id: string, bytes: number): SessionMessage {
782+
return {
783+
...message(id, "user", id),
784+
contentParams: [{ type: "image_url", image_url: { url: `data:image/png;base64,${"A".repeat(bytes)}` } }],
785+
};
786+
}
787+
788+
function setByteBudget(f: ReturnType<typeof fixture>, bytes: number, retainTokens = 0) {
789+
// Exercise the real formula with small payloads instead of allocating 48 MiB per test.
790+
f.settings.contextWindow = (MAX_REQUEST_BODY_BYTES - bytes) / 2;
791+
f.settings.retainTokens = retainTokens;
792+
}
793+
794+
test("message byte estimates include UTF-8, JSON escaping and image data", () => {
795+
const messages = [{ role: "user" as const, content: '中文\n"' }];
796+
assert.equal(estimateRequestBytes(messages), new TextEncoder().encode(JSON.stringify(messages)).length);
797+
assert.ok(estimateRequestBytes(messages) > JSON.stringify(messages).length);
798+
const image = new OpenAIMessageConverter().buildMessages([imageMessage("image", 10000)], true, "test-vision");
799+
assert.ok(estimateRequestBytes(image) > 10000);
800+
assert.equal(compactBudget(1024 * 1024).byteThreshold, 46 * 1024 * 1024);
801+
for (const window of [24 * 1024 * 1024, 25 * 1024 * 1024]) {
802+
assert.throws(() => compactBudget(window), /request size budget/);
803+
}
804+
});
805+
806+
for (const delta of [-1, 0, 1]) {
807+
test(`automatic compaction checks the byte threshold with a ${delta}-byte difference`, async () => {
808+
const history = [imageMessage("old", 20000), message("tail", "user", "continue ".repeat(1000))];
809+
const f = fixture(history);
810+
const context = await f.internal.prepareContext(f.read(), f.settings.model, true);
811+
const bytes = estimateRequestBytes(context.messages);
812+
setByteBudget(f, bytes - delta);
813+
assert.ok(f.internal.measureRequest("compact-test", context.messages, f.settings.model).tokens < 190000);
814+
await f.run();
815+
assert.equal(f.requests.length, delta > 0 ? 2 : 1);
816+
assert.equal(f.requests[0].tool_choice === "none", delta > 0);
817+
assert.ok(f.requests.every((request) => estimateRequestBytes(request.messages) <= bytes - delta));
818+
assert.equal(f.read().find((item) => item.id === "tail")?.compacted, false);
819+
});
820+
}
821+
822+
test("oversized summary candidates shrink to the longest prefix within the byte budget", async () => {
823+
const f = fixture([
824+
...["first", "second", "third", "fourth"].map((id) => imageMessage(id, 11000)),
825+
message("tail", "user", "continue"),
826+
]);
827+
setByteBudget(f, 30000);
828+
await f.run();
829+
assert.equal(f.requests.length, 2);
830+
assert.equal(f.requests[0].tool_choice, "none");
831+
assert.equal(f.requests[0].messages.at(-1).content, getCompactPrompt());
832+
assert.deepEqual(
833+
f
834+
.read()
835+
.filter((item) => item.compacted)
836+
.map((item) => item.id),
837+
["first", "second"]
838+
);
839+
assert.ok(f.requests.every((request) => estimateRequestBytes(request.messages) <= 30000));
840+
});
841+
842+
for (const retainTokens of [12000, 100000]) {
843+
test(`byte compaction falls back to no extra retention after a ${retainTokens}-token retention pass`, async () => {
844+
const f = fixture([
845+
message("old", "user", "x".repeat(7000)),
846+
...["first", "second", "third"].map((id) => imageMessage(id, 15000)),
847+
message("tail", "user", "continue"),
848+
]);
849+
setByteBudget(f, 40000, retainTokens);
850+
await f.run();
851+
assert.equal(f.requests.length, retainTokens === 12000 ? 3 : 2);
852+
assert.ok(f.requests.slice(0, -1).every((request) => request.tool_choice === "none"));
853+
assert.notEqual(f.requests.at(-1).tool_choice, "none");
854+
assert.ok(f.requests.every((request) => estimateRequestBytes(request.messages) <= 40000));
855+
assert.equal(f.read().find((item) => item.id === "tail")?.compacted, false);
856+
});
857+
}
858+
859+
test("byte compaction accepts a smaller payload even when its summary uses more tokens", async () => {
860+
const f = fixture(
861+
[
862+
...["first", "second", "third"].map((id) => ({
863+
...message(id, "user", id),
864+
contentParams: [
865+
{ type: "image_url", image_url: { url: `data:image/png;base64,${"A".repeat(11000)}`, detail: "low" } },
866+
],
867+
})),
868+
message("tail", "user", "continue"),
869+
],
870+
() => answer("x".repeat(10000))
871+
);
872+
setByteBudget(f, 30000);
873+
// Compacting two images saves bytes, although the text summary costs more tokens.
874+
const before = await f.internal.prepareContext(f.read(), f.settings.model, true);
875+
assert.equal(await f.compact(), true);
876+
assert.equal(f.read().filter((item) => item.compacted).length, 2);
877+
const after = await f.internal.prepareContext(f.read(), f.settings.model, true);
878+
assert.ok(estimateRequestTokens(after.messages) > estimateRequestTokens(before.messages));
879+
assert.ok(estimateRequestBytes(after.messages) < estimateRequestBytes(before.messages));
880+
});
881+
882+
test("unshrinkable byte-heavy input terminates without sending an oversized request", async () => {
883+
for (const kind of ["fixed", "latest", "prefix", "many"] as const) {
884+
const huge = imageMessage("huge", 31000);
885+
const history =
886+
kind === "fixed"
887+
? [{ ...huge, role: "system" as const }, message("old"), message("tail")]
888+
: kind === "latest"
889+
? [message("old"), huge]
890+
: kind === "prefix"
891+
? [huge, message("tail")]
892+
: [...Array.from({ length: 10 }, (_, i) => imageMessage(`image-${i}`, 11000)), message("tail")];
893+
const f = fixture(history);
894+
setByteBudget(f, 30000);
895+
await assert.rejects(f.run(), /too large|did not reduce/);
896+
assert.ok(f.requests.length <= 2);
897+
assert.ok(
898+
f.requests.every((request) => request.tool_choice === "none" && estimateRequestBytes(request.messages) <= 30000)
899+
);
900+
assert.equal(f.read().at(-1)?.compacted, false);
901+
assert.ok(f.read().every((item) => !item.meta?.omittedImages));
902+
if (kind === "fixed" || kind === "prefix") assert.deepEqual(f.read(), history);
903+
}
904+
});
905+
906+
test("new tool results trigger byte compaction before the next assistant step", async () => {
907+
const call = {
908+
...message("call", "assistant", ""),
909+
messageParams: { tool_calls: [{ id: "read", type: "function", function: { name: "Read", arguments: "{}" } }] },
910+
};
911+
const result = { ...message("result", "tool", "x".repeat(15000)), messageParams: { tool_call_id: "read" } };
912+
const f = fixture([imageMessage("old", 20000), call, result]);
913+
setByteBudget(f, 30000);
914+
await f.run();
915+
assert.equal(f.requests.length, 2);
916+
assert.equal(f.requests[0].tool_choice, "none");
917+
assert.equal(f.requests[1].messages.filter((item: any) => item.role === "tool").length, 1);
918+
assert.equal(f.requests[1].messages.find((item: any) => item.role === "assistant").tool_calls.length, 1);
919+
assert.ok(f.requests.every((request) => estimateRequestBytes(request.messages) <= 30000));
920+
});
921+
922+
test("context-overflow recovery rechecks bytes when the summary grows the payload", async () => {
923+
const f = fixture([message("old", "user", "x".repeat(10000)), imageMessage("tail", 18000)], (_, call) => {
924+
if (call === 1) throw { status: 400, code: "context_length_exceeded" };
925+
return answer("中".repeat(4500));
926+
});
927+
setByteBudget(f, 30000);
928+
await assert.rejects(f.run(), /too large|did not reduce/);
929+
assert.equal(f.requests[0].tool_choice, undefined);
930+
assert.ok(f.read().some((item) => item.meta?.isSummary));
931+
assert.ok(f.requests.slice(1).every((request) => request.tool_choice === "none"));
932+
assert.ok(f.requests.every((request) => estimateRequestBytes(request.messages) <= 30000));
933+
});
934+
935+
test("Files API conservatively checks bytes before uploads and sends references after compaction", async () => {
936+
const f = fixture([
937+
...["first", "second", "third"].map((id) => imageMessage(id, 11000)),
938+
message("tail", "user", "continue"),
939+
]);
940+
setByteBudget(f, 30000);
941+
const originalClient = f.internal.createOpenAIClient;
942+
f.internal.createOpenAIClient = () => ({ ...originalClient(), apiKey: "test-key" });
943+
Object.assign(f.settings, { filesApiEnabled: true });
944+
f.internal.deepSeekFiles = {
945+
ensureUploaded: async () => ({ fileId: "file-image", imageHash: "a".repeat(64), bytes: 8250 }),
946+
};
947+
await f.run();
948+
assert.equal(f.requests.length, 2);
949+
assert.equal(f.requests[0].tool_choice, "none");
950+
for (const request of f.requests) {
951+
assert.match(JSON.stringify(request.messages), /file-image/);
952+
assert.doesNotMatch(JSON.stringify(request.messages), /base64/);
953+
}
954+
});

0 commit comments

Comments
 (0)