Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,12 @@ describe('session environment prompt', () => {
assert.doesNotMatch(prompt, /Git branch: .*\nmalicious/);
});

it('is injected into the main system prompt path before tool-specific context', async () => {
it('is injected as a current-turn tail instead of durable system prefix', async () => {
const source = await readFile(join(process.cwd(), 'src/main/main.ts'), 'utf8');

assert.match(source, /buildSessionEnvironmentPromptFragment\(\{/);
assert.match(source, /turnTailPrompt:\s*\(\{ cwd \}\) => buildTurnTailPrompt\(cwd\)/);
assert.match(source, /async function buildTurnTailPrompt\(cwd\?: string\)/);
assert.match(source, /projectGit:\s*await resolveProjectGitInfo\(cwd\)/);
assert.match(source, /personalization\.text,\n\s*environment,\n\s*deepResearch/);
assert.doesNotMatch(source, /personalization\.text,\n\s*environment,\n\s*deepResearch/);
});
});
16 changes: 9 additions & 7 deletions apps/desktop/src/main/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -736,6 +736,7 @@ backends.register('ai-sdk', async (ctx) => {
tools: builtinTools,
providerOptions: buildProviderOptions(connection, model),
systemPrompt: ({ cwd }) => buildSystemPrompt(ctx.header, cwd),
turnTailPrompt: ({ cwd }) => buildTurnTailPrompt(cwd),
recordLlmCall: (event) => recordLlmCall({ repo: telemetryRepo, lookupPricing }, event),
recordToolInvocation: (event) =>
recordToolInvocation(
Expand Down Expand Up @@ -3287,12 +3288,6 @@ async function handleQuickChatStart(rawInput: unknown): Promise<QuickChatResult>
async function buildSystemPrompt(header: Pick<SessionHeader, 'labels'>, cwd?: string): Promise<string | undefined> {
const settings = await settingsStore.get();
const personalization = buildPersonalizationPromptFragment(settings.personalization);
const environment = cwd
? buildSessionEnvironmentPromptFragment({
cwd,
projectGit: await resolveProjectGitInfo(cwd),
})
: undefined;
const skills = await buildSkillsPromptFragment(workspaceRoot);
const workspaceInstructions = settings.workspaceInstructions.enabled && cwd
? await buildWorkspaceInstructionsPromptFragment(cwd)
Expand All @@ -3312,7 +3307,6 @@ async function buildSystemPrompt(header: Pick<SessionHeader, 'labels'>, cwd?: st
const memoryFragment = await buildLocalMemoryPromptFragment();
const fragments = [
personalization.text,
environment,
deepResearch,
botPlatformHint,
skills,
Expand All @@ -3322,6 +3316,14 @@ async function buildSystemPrompt(header: Pick<SessionHeader, 'labels'>, cwd?: st
return fragments.length > 0 ? fragments.join('\n\n') : undefined;
}

async function buildTurnTailPrompt(cwd?: string): Promise<string | undefined> {
if (!cwd) return undefined;
return buildSessionEnvironmentPromptFragment({
cwd,
projectGit: await resolveProjectGitInfo(cwd),
});
}

async function buildLocalMemoryPromptFragment(): Promise<string | undefined> {
try {
const state = await localMemory.getState();
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/

import type { PermissionRequest, PermissionResponse, ToolCategory } from './permission.js';
import type { PrefixChangeReason } from './usage-stats/types.js';

export const TOOL_OUTPUT_STREAMS = ['stdout', 'stderr'] as const;
export const TOOL_OUTPUT_DELTA_MAX_CHARS = 8192;
Expand Down Expand Up @@ -332,6 +333,8 @@ export interface TokenUsageEvent extends BaseEvent {
cacheCreation?: number;
costUsd?: number;
contextRemaining?: number;
prefixHash?: string;
prefixChangeReason?: PrefixChangeReason;
}

export interface ErrorEvent extends BaseEvent {
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/runtime-event.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import type { AttachmentRef } from './events.js';
import type { PermissionRequest, PermissionResponse } from './permission.js';
import type { PrefixChangeReason } from './usage-stats/types.js';

// ============================================================================
// Role / Author / Status
Expand Down Expand Up @@ -177,6 +178,8 @@ export interface RuntimeEventTokenUsage {
cacheCreation?: number;
costUsd?: number;
contextRemaining?: number;
prefixHash?: string;
prefixChangeReason?: PrefixChangeReason;
}

/**
Expand Down
3 changes: 3 additions & 0 deletions packages/core/src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import type { AttachmentRef, ToolResultContent } from './events.js';
import type { PermissionMode } from './permission.js';
import type { PrefixChangeReason } from './usage-stats/types.js';

export const SESSION_STATUSES = [
'active',
Expand Down Expand Up @@ -240,6 +241,8 @@ export interface TokenUsageMessage {
/** Backward-compatible alias for cacheWriteInput. */
cacheCreation?: number;
costUsd?: number;
prefixHash?: string;
prefixChangeReason?: PrefixChangeReason;
}

export interface TurnStateMessage {
Expand Down
14 changes: 14 additions & 0 deletions packages/core/src/usage-stats/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ export interface UsageLogRow {
errorClass?: string;
sessionId?: string;
turnId?: string;
prefixHash?: string;
prefixChangeReason?: PrefixChangeReason;
}

export interface PricingConfig {
Expand Down Expand Up @@ -113,8 +115,20 @@ export interface LlmCallRecord {
status: 'success' | 'error' | 'aborted';
errorClass?: string;
startedAt: number;
prefixHash?: string;
prefixChangeReason?: PrefixChangeReason;
}

export type PrefixChangeReason =
| 'first_turn'
| 'system_prompt_changed'
| 'tool_schema_changed'
| 'provider_options_changed'
| 'model_or_provider_changed'
| 'history_projection_changed'
| 'stable'
| 'unknown';

export interface ToolInvocationRecord {
sessionId?: string;
turnId?: string;
Expand Down
173 changes: 173 additions & 0 deletions packages/runtime/src/__tests__/ai-sdk-backend.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import type { SessionEvent } from '@maka/core/events';
import type { RuntimeEvent } from '@maka/core/runtime-event';
import type { ToolResultMessage } from '@maka/core/session';
import type { LlmCallRecord } from '@maka/core/usage-stats/types';
import { z } from 'zod';
import {
AiSdkBackend,
INVALID_TOOL_NAME,
Expand All @@ -19,6 +20,10 @@ import {
type RunTraceEvent,
} from '../ai-sdk-backend.js';
import { PermissionEngine } from '../permission-engine.js';
import {
canonicalizeToolSet,
computeRequestShapeDiagnostic,
} from '../request-shape.js';

describe('AiSdkBackend model history', () => {
test('prefers RuntimeEvent prior messages and appends current user once', async () => {
Expand Down Expand Up @@ -644,6 +649,157 @@ describe('AiSdkBackend usage telemetry', () => {
});
});

describe('AiSdkBackend request-shape diagnostics', () => {
test('identical request shape keeps the same hash and reports stable after first turn', () => {
const tools = canonicalizeToolSet([
testTool('Read', z.object({ path: z.string() })),
testTool('Bash', z.object({ command: z.string() })),
], testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() })));
const first = computeRequestShapeDiagnostic({
connection: connection(),
modelId: 'mock-model-id',
systemPrompt: 'durable system',
providerOptions: { temperature: 0, nested: { b: 2, a: 1 } },
providerTools: tools.providerTools,
activeTools: tools.activeTools,
priorMessages: [{ role: 'user', content: 'hello' }],
}, undefined);
const second = computeRequestShapeDiagnostic({
connection: connection(),
modelId: 'mock-model-id',
systemPrompt: 'durable system',
providerOptions: { nested: { a: 1, b: 2 }, temperature: 0 },
providerTools: tools.providerTools,
activeTools: tools.activeTools,
priorMessages: [{ role: 'user', content: 'hello' }],
}, first);

assert.equal(first.prefixChangeReason, 'first_turn');
assert.equal(second.prefixChangeReason, 'stable');
assert.equal(second.prefixHash, first.prefixHash);
});

test('classifies targeted request-shape changes', () => {
const tools = canonicalizeToolSet([
testTool('Read', z.object({ path: z.string() })),
], testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() })));
const baseInput = {
connection: connection(),
modelId: 'mock-model-id',
systemPrompt: 'durable system',
providerOptions: { temperature: 0 },
providerTools: tools.providerTools,
activeTools: tools.activeTools,
priorMessages: [{ role: 'user' as const, content: 'hello' }],
};
const base = computeRequestShapeDiagnostic(baseInput, undefined);

assert.equal(computeRequestShapeDiagnostic({
...baseInput,
systemPrompt: 'changed system',
}, base).prefixChangeReason, 'system_prompt_changed');
assert.equal(computeRequestShapeDiagnostic({
...baseInput,
providerTools: canonicalizeToolSet([
testTool('Read', z.object({ path: z.string(), offset: z.number().optional() })),
], testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() }))).providerTools,
}, base).prefixChangeReason, 'tool_schema_changed');
assert.equal(computeRequestShapeDiagnostic({
...baseInput,
providerOptions: { temperature: 1 },
}, base).prefixChangeReason, 'provider_options_changed');
assert.equal(computeRequestShapeDiagnostic({
...baseInput,
modelId: 'other-model',
}, base).prefixChangeReason, 'model_or_provider_changed');
assert.equal(computeRequestShapeDiagnostic({
...baseInput,
priorMessages: [{ role: 'assistant' as const, content: 'hello' }],
}, base).prefixChangeReason, 'history_projection_changed');
});

test('tool canonicalization is independent of registration order and places invalid last', () => {
const invalid = testTool(INVALID_TOOL_NAME, z.object({ tool: z.string().optional() }));
const first = canonicalizeToolSet([
testTool('Write', z.object({ path: z.string(), content: z.string() })),
testTool('Read', z.object({ path: z.string() })),
], invalid);
const second = canonicalizeToolSet([
testTool('Read', z.object({ path: z.string() })),
testTool('Write', z.object({ content: z.string(), path: z.string() })),
], invalid);

assert.deepEqual(first.activeTools, ['Read', 'Write']);
assert.deepEqual(first.providerTools.map((tool) => tool.name), ['Read', 'Write', INVALID_TOOL_NAME]);
assert.deepEqual(second.providerTools.map((tool) => tool.name), ['Read', 'Write', INVALID_TOOL_NAME]);
assert.equal(
computeRequestShapeDiagnostic({
connection: connection(),
modelId: 'mock-model-id',
providerTools: first.providerTools,
activeTools: first.activeTools,
priorMessages: [],
}, undefined).componentHashes.toolSchemaHash,
computeRequestShapeDiagnostic({
connection: connection(),
modelId: 'mock-model-id',
providerTools: second.providerTools,
activeTools: second.activeTools,
priorMessages: [],
}, undefined).componentHashes.toolSchemaHash,
);
});

test('volatile turn-tail facts do not churn the durable prefix hash', async () => {
const events: SessionEvent[] = [];
const llmRecords: LlmCallRecord[] = [];
const models: MockLanguageModelV3[] = [];
let date = '2026-05-29';
const backend = new AiSdkBackend({
sessionId: 'session-1',
header: header(),
appendMessage: async () => {},
connection: connection(),
apiKey: 'sk-test',
modelId: 'mock-model-id',
permissionEngine: new PermissionEngine({ newId: () => 'permission-id', now: () => 1 }),
modelFactory: () => {
const model = completionModel();
models.push(model);
return model;
},
tools: [],
newId: idGenerator(),
now: monotonicClock(),
systemPrompt: 'durable system prompt',
turnTailPrompt: () => `Maka session environment:\n<env>\n Today's date: ${date}\n</env>`,
recordLlmCall: (record) => {
llmRecords.push(record);
},
});

for await (const event of backend.send({ turnId: 'turn-1', text: 'hi', context: [] })) {
events.push(event);
}
date = '2026-05-30';
for await (const event of backend.send({ turnId: 'turn-2', text: 'hi', context: [] })) {
events.push(event);
}

const usageEvents = events.filter((event): event is Extract<SessionEvent, { type: 'token_usage' }> =>
event.type === 'token_usage'
);
assert.equal(usageEvents[0]?.prefixChangeReason, 'first_turn');
assert.equal(usageEvents[1]?.prefixChangeReason, 'stable');
assert.equal(usageEvents[1]?.prefixHash, usageEvents[0]?.prefixHash);
assert.equal(llmRecords[1]?.prefixChangeReason, 'stable');
assert.match(JSON.stringify(compactPrompt(models[0]!)), /2026-05-29/);
assert.match(JSON.stringify(compactPrompt(models[1]!)), /2026-05-30/);
assert.equal(JSON.stringify(modelCallSettings(models[0]!)).includes('2026-05-29'), false);
assert.equal(JSON.stringify(modelCallSettings(models[1]!)).includes('2026-05-30'), false);
});
});

describe('AiSdkBackend RunTrace', () => {
test('records turn, model, usage, and completion trace events without changing SessionEvents', async () => {
const trace: RunTraceEvent[] = [];
Expand Down Expand Up @@ -1687,6 +1843,23 @@ function compactPrompt(model: MockLanguageModelV3): unknown {
}));
}

function modelCallSettings(model: MockLanguageModelV3): unknown {
const call = model.doStreamCalls[0] as unknown as Record<string, unknown> | undefined;
if (!call) return {};
const { prompt: _prompt, ...rest } = call;
return rest;
}

function testTool(name: string, parameters: unknown): MakaTool {
return {
name,
description: `${name} description`,
parameters,
permissionRequired: false,
impl: async () => ({ ok: true }),
};
}

async function drain(iterable: AsyncIterable<unknown>): Promise<void> {
for await (const _ of iterable) {
// consume
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ describe('RunTrace extraction contract', () => {
assert.match(backend, /private currentRunTrace: RunTrace \| null = null;/);
assert.match(backend, /trace\.turnStarted\(\)/);
assert.match(backend, /trace\.modelResolved\(\)/);
assert.match(backend, /trace\.modelStreamStarted\(activeTools\)/);
assert.match(backend, /trace\.usageRecorded\(tokenUsage\)/);
assert.match(backend, /trace\.modelStreamStarted\(activeTools, \{/);
assert.match(backend, /trace\.usageRecorded\(\{\n\s+\.\.\.tokenUsage,/);
assert.match(backend, /this\.currentRunTrace\?\.abortRequested\(_reason\)/);
assert.match(barrel, /RunTraceEvent/);
assert.match(barrel, /RunTraceRecorder/);
Expand Down
Loading