Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions skills/chatgpt-review/scripts/chatgpt-review.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ export async function run(argv, dependencies = {}) {
}

const prepared = await prepare(options);
cleanup = prepared.cleanup ?? cleanup;
const prepareCleanup = prepared.cleanup ?? (async () => {});
cleanup = prepareCleanup;
if (options.session) {
session = await store.load(options.session);
if (session.mode !== options.mode || session.targetIdentity !== prepared.targetIdentity) throw new CliError('Session does not match this mode and target');
Expand All @@ -43,6 +44,23 @@ export async function run(argv, dependencies = {}) {
}
passNumber = (session.passCount ?? 0) + 1;
if (options.mode === 'pr' && passNumber > 3) throw new CliError('PR review sessions permit at most three total passes');

// Name each pass's upload distinctly (plan-590-pass4.md, not plan-590.md every time) — the
// plan/diff file's own path must never move (it is the plan-review-loop's session identity),
// so we upload a same-content, differently-named COPY. Reusing one literal filename across
// many passes made ChatGPT's own upload UI collision-rename it (plan-590(9).md) after enough
// retries, which is confusing and unrelated to the real pass count.
let uploadPath = prepared.uploadPath;
if (uploadPath) {
const ext = path.extname(uploadPath);
const base = path.basename(uploadPath, ext);
const content = await fs.readFile(uploadPath, 'utf8');
const renamed = await writePrivateTempFile(`${base}-pass${passNumber}${ext}`, content);
uploadPath = renamed.filename;
const uploadCleanup = renamed.cleanup;
cleanup = async () => { await uploadCleanup(); await prepareCleanup(); };
}

const context = options.questionFile ? await fs.readFile(path.resolve(options.questionFile), 'utf8') : '';
const prompt = buildPrompt({
mode: options.mode,
Expand All @@ -51,19 +69,19 @@ export async function run(argv, dependencies = {}) {
publish: options.requestedPublication,
pass: passNumber,
previousSha: session.reportedReviewedSha,
uploadName: prepared.uploadPath ? path.basename(prepared.uploadPath) : null,
uploadName: uploadPath ? path.basename(uploadPath) : null,
});
const review = await driver.review({
session: options.session ? session : null,
prompt,
uploadPath: prepared.uploadPath,
uploadPath,
timeoutMs: options.timeoutMs,
target: prepared.target,
publish: options.requestedPublication,
diagnosticsDir: options.diagnosticsDir ? path.resolve(options.diagnosticsDir) : null,
});
const metadata = extractReportedMetadata(review.responseText);
session = await store.write({ ...session, conversationUrl: review.conversationUrl, passCount: passNumber, ...metadata });
session = await store.write({ ...session, conversationUrl: review.conversationUrl, passCount: passNumber, lastResponseFingerprint: review.responseFingerprint ?? null, ...metadata });
return resultDocument({
status: 'completed', response_text: review.responseText, session: session.handle,
conversation_url: review.conversationUrl, elapsed_seconds: elapsed(started), pass_number: passNumber,
Expand Down
42 changes: 26 additions & 16 deletions skills/chatgpt-review/scripts/lib/browser.mjs
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
import fs from 'node:fs/promises';
import path from 'node:path';
import crypto from 'node:crypto';

// ChatGPT virtualizes/prunes older turns out of the DOM in long conversations, so a raw
// element COUNT of assistant messages does not grow monotonically — it can plateau or even
// drop as old turns are unmounted. Tracking "is there a new response" by content fingerprint
// of the current LAST assistant message (instead of by count) survives that pruning.
export function fingerprintText(text) {
return text ? crypto.createHash('sha256').update(text).digest('hex') : null;
}

export const SELECTORS = Object.freeze({
composer: ['[data-testid="prompt-textarea"]', '#prompt-textarea', 'textarea[placeholder*="Message"]', '[contenteditable="true"][role="textbox"]'],
Expand Down Expand Up @@ -73,21 +82,22 @@ export class ChatGptBrowser {
const { page, reopened } = await this.pageFor(session);
try {
await this.assertReady(page);
const existingCount = await this.assistantCount(page);
const generationActive = await anyVisible(page, SELECTORS.stop);
const recordedPasses = session?.passCount ?? 0;
if (session && (generationActive || existingCount > recordedPasses)) {
this.stderr.write(`Recovering an uncollected ChatGPT response (recorded passes: ${recordedPasses})...\n`);
const responseText = await this.waitForCompletion(page, { before: recordedPasses, timeoutMs, target, publish });
return { responseText, conversationUrl: page.url(), reopened, predefinedModelAndEffort: true, recovered: true };
const currentTail = await this.latestAssistantText(page);
const recordedFingerprint = session?.lastResponseFingerprint ?? null;
const hasUncollected = generationActive || (Boolean(currentTail) && fingerprintText(currentTail) !== recordedFingerprint);
if (session && hasUncollected) {
this.stderr.write('Recovering an uncollected ChatGPT response...\n');
const responseText = await this.waitForCompletion(page, { before: null, timeoutMs, target, publish });
return { responseText, conversationUrl: page.url(), reopened, predefinedModelAndEffort: true, recovered: true, responseFingerprint: fingerprintText(responseText) };
}
if (uploadPath) await this.upload(page, uploadPath);
const before = await this.assistantCount(page);
const before = currentTail;
await this.fillAndSend(page, prompt);
await this.waitForPermanentConversationUrl(page);
this.stderr.write(`Waiting for ChatGPT response (assistant messages before submit: ${before})...\n`);
this.stderr.write('Waiting for ChatGPT response...\n');
const responseText = await this.waitForCompletion(page, { before, timeoutMs, target, publish });
return { responseText, conversationUrl: page.url(), reopened, predefinedModelAndEffort: true, recovered: false };
return { responseText, conversationUrl: page.url(), reopened, predefinedModelAndEffort: true, recovered: false, responseFingerprint: fingerprintText(responseText) };
} catch (error) {
error.conversationUrl = page.url();
if (diagnosticsDir) await this.captureDiagnostics(page, diagnosticsDir, error).catch(() => {});
Expand Down Expand Up @@ -126,10 +136,6 @@ export class ChatGptBrowser {
else await composer.press('Enter');
}

async assistantCount(page) {
return page.locator(SELECTORS.assistant[0]).count();
}

async waitForPermanentConversationUrl(page) {
const deadline = this.now() + 15_000;
while (this.now() < deadline) {
Expand Down Expand Up @@ -181,12 +187,16 @@ export class ChatGptBrowser {
await this.handlePermission(page, target, publish);
const continuation = await firstVisible(page, SELECTORS.continue);
if (continuation) { await continuation.click(); stableSince = null; }
const count = await this.assistantCount(page);
const text = count > before ? await this.latestAssistantText(page) : '';
const currentText = await this.latestAssistantText(page);
// before === null means "recovering an uncollected response" — accept whatever is
// already there. Otherwise before is the pre-submission baseline text (possibly '');
// only a DIFFERENT tail counts as the new response. Content-based, not count-based,
// so DOM pruning of older turns in a long conversation cannot spuriously suppress it.
const text = (before === null || currentText !== before) ? currentText : '';
const generating = await anyVisible(page, SELECTORS.stop);
if (text && text === lastText) stableSince ??= this.now();
else { lastText = text; stableSince = text ? this.now() : null; }
if (count > before && text && !generating && stableSince !== null && this.now() - stableSince >= this.stableMs) return text;
if (text && !generating && stableSince !== null && this.now() - stableSince >= this.stableMs) return text;
await this.sleep(this.pollMs);
}
throw new ReviewError('timed_out', 'Timed out before ChatGPT produced a stable completed response', lastText);
Expand Down
2 changes: 1 addition & 1 deletion skills/chatgpt-review/scripts/lib/state.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,6 @@ export class SessionStore {
}

function sanitize(record) {
const allowed = ['handle', 'mode', 'targetIdentity', 'canonicalUrl', 'conversationUrl', 'passCount', 'createdAt', 'updatedAt', 'reportedReviewedSha', 'reportedGithubCommentUrl'];
const allowed = ['handle', 'mode', 'targetIdentity', 'canonicalUrl', 'conversationUrl', 'passCount', 'lastResponseFingerprint', 'createdAt', 'updatedAt', 'reportedReviewedSha', 'reportedGithubCommentUrl'];
return Object.fromEntries(allowed.filter((key) => record[key] !== undefined).map((key) => [key, record[key]]));
}
54 changes: 46 additions & 8 deletions skills/chatgpt-review/tests/browser.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,44 @@ test('session retry recovers an uncollected response without sending a duplicate
assert.equal(composer.value, undefined);
});

test('fresh submission detects a new response even when DOM pruning keeps the assistant-message count flat', async () => {
let sent = false;
const composer = new Element();
const send = new Element({ onClick: () => { sent = true; } });
// Simulates ChatGPT virtualizing old turns out of the DOM: the assistant locator always
// returns exactly one element (a fixed-size window), but its content is the STALE prior
// answer until submit, then the NEW one — never two elements at once, so a count-based
// before/after check could never observe growth.
const page = new Page('https://chatgpt.com/', {
[SELECTORS.composer[0]]: [composer], [SELECTORS.send[0]]: [send],
[SELECTORS.assistant[0]]: () => [new Element({ text: sent ? 'brand new answer' : 'stale old answer' })],
});
const driver = driverWith(page);
const result = await driver.review({ prompt: 'review', timeoutMs: 20, target: null, publish: false });
assert.equal(result.responseText, 'brand new answer');
});

test('recovery via stored fingerprint detects an uncollected response under DOM pruning, without a generation indicator', async () => {
let sent = false;
const composer = new Element();
const page = new Page('https://chatgpt.com/c/recover-pruned', {
[SELECTORS.composer[0]]: [composer],
[SELECTORS.send[0]]: [new Element({ onClick: () => { sent = true; } })],
[SELECTORS.assistant[0]]: [new Element({ text: 'new uncollected answer' })],
});
const driver = driverWith(page);
const result = await driver.review({
// passCount/fingerprint reflect an earlier, DIFFERENT response never seen live on this
// page — simulating a prior invocation that crashed after ChatGPT answered but before it
// recorded anything. Absolute message count plays no part in this decision.
session: { conversationUrl: page.url(), passCount: 3, lastResponseFingerprint: 'stale-fingerprint-from-a-different-answer' },
prompt: 'must not send', timeoutMs: 20, target: null, publish: false,
});
assert.equal(result.responseText, 'new uncollected answer');
assert.equal(result.recovered, true);
assert.equal(sent, false);
});

test('submission waits for ChatGPT to replace its temporary conversation URL', async () => {
const page = readyPage();
page.currentUrl = 'https://chatgpt.com/c/WEB:temporary';
Expand All @@ -125,7 +163,7 @@ test('continue generating is clicked harmlessly', async () => {
const button = new Element({ onClick: (self) => { self.visible = false; } });
const page = readyPage({ [SELECTORS.continue[0]]: [button], [SELECTORS.assistant[0]]: [new Element({ text: 'done' })] });
const driver = driverWith(page);
assert.equal(await driver.waitForCompletion(page, { before: 0, timeoutMs: 20, publish: false }), 'done');
assert.equal(await driver.waitForCompletion(page, { before: '', timeoutMs: 20, publish: false }), 'done');
assert.equal(button.visible, false);
});

Expand All @@ -138,7 +176,7 @@ test('message stream failures use Retry without completing or creating a new pro
[SELECTORS.streamRetry[0]]: () => failed ? [retry] : [],
[SELECTORS.assistant[0]]: () => [new Element({ text: failed ? 'Error in message stream\nRetry' : 'complete answer' })],
});
assert.equal(await driverWith(page).waitForCompletion(page, { before: 0, timeoutMs: 20, publish: false }), 'complete answer');
assert.equal(await driverWith(page).waitForCompletion(page, { before: '', timeoutMs: 20, publish: false }), 'complete answer');
assert.equal(retries, 1);
});

Expand All @@ -151,7 +189,7 @@ test('persistent message stream failure is typed after two retries', async () =>
[SELECTORS.assistant[0]]: [new Element({ text: 'Error in message stream\nRetry' })],
});
await assert.rejects(
() => driverWith(page).waitForCompletion(page, { before: 0, timeoutMs: 20, publish: false }),
() => driverWith(page).waitForCompletion(page, { before: '', timeoutMs: 20, publish: false }),
(error) => error.status === 'ui_incompatible' && /after two automatic retries/.test(error.message),
);
assert.equal(retries, 2);
Expand All @@ -163,19 +201,19 @@ test('login failure, UI drift, rate limit, and timeout are typed', async () => {
const drift = new Page();
await assert.rejects(() => driverWith(drift).assertReady(drift), (error) => error.status === 'ui_incompatible');
const rate = readyPage({ [SELECTORS.rateLimit[0]]: [new Element()] });
await assert.rejects(() => driverWith(rate).waitForCompletion(rate, { before: 0, timeoutMs: 2 }), (error) => error.status === 'rate_limited');
await assert.rejects(() => driverWith(rate).waitForCompletion(rate, { before: '', timeoutMs: 2 }), (error) => error.status === 'rate_limited');
const uiError = readyPage({ [SELECTORS.error[0]]: [new Element({ text: 'Something went wrong' })] });
await assert.rejects(() => driverWith(uiError).waitForCompletion(uiError, { before: 0, timeoutMs: 2 }), (error) => error.status === 'ui_incompatible');
await assert.rejects(() => driverWith(uiError).waitForCompletion(uiError, { before: '', timeoutMs: 2 }), (error) => error.status === 'ui_incompatible');
const timeout = readyPage();
await assert.rejects(() => driverWith(timeout).waitForCompletion(timeout, { before: 0, timeoutMs: 2 }), (error) => error.status === 'timed_out');
await assert.rejects(() => driverWith(timeout).waitForCompletion(timeout, { before: '', timeoutMs: 2 }), (error) => error.status === 'timed_out');
});

test('empty and status live-region alerts do not abort an active review', async () => {
const page = readyPage({
[SELECTORS.alert[0]]: [new Element({ text: '' }), new Element({ text: 'ChatGPT is working' })],
[SELECTORS.assistant[0]]: [new Element({ text: 'complete answer' })],
});
assert.equal(await driverWith(page).waitForCompletion(page, { before: 0, timeoutMs: 20, publish: false }), 'complete answer');
assert.equal(await driverWith(page).waitForCompletion(page, { before: '', timeoutMs: 20, publish: false }), 'complete answer');
});

test('live-region alerts are fatal only when their text identifies a real failure', async () => {
Expand All @@ -184,7 +222,7 @@ test('live-region alerts are fatal only when their text identifies a real failur
assert.deepEqual(classifyAlertText('Too many requests; try again later'), { status: 'rate_limited', message: 'Too many requests; try again later' });
assert.deepEqual(classifyAlertText('Something went wrong'), { status: 'ui_incompatible', message: 'Something went wrong' });
const page = readyPage({ [SELECTORS.alert[0]]: [new Element({ text: 'There was an error generating a response' })] });
await assert.rejects(() => driverWith(page).waitForCompletion(page, { before: 0, timeoutMs: 2 }), (error) => error.status === 'ui_incompatible');
await assert.rejects(() => driverWith(page).waitForCompletion(page, { before: '', timeoutMs: 2 }), (error) => error.status === 'ui_incompatible');
});

test('only a scoped comment permission is automatically approvable', () => {
Expand Down
19 changes: 16 additions & 3 deletions skills/chatgpt-review/tests/core.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ test('run retains a session and enforces three PR passes', async () => {
assert.match(result.error, /at most three/);
});

test('plan mode uploads exactly the supplied file and never authorizes publication', async (t) => {
test('plan mode uploads a pass-numbered copy (never the literal session-identity path) and never authorizes publication', async (t) => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'chatgpt-review-plan-'));
t.after(() => fs.rm(dir, { recursive: true, force: true }));
const planFile = path.join(dir, 'complete-plan.md');
Expand All @@ -178,10 +178,23 @@ test('plan mode uploads exactly the supplied file and never authorizes publicati
async create(data) { return { handle: '00000000-0000-4000-8000-000000000002', passCount: 0, ...data }; },
async write(value) { return value; },
};
const driver = { async review(input) { observed = input; return { responseText: 'review', conversationUrl: 'https://chatgpt.com/c/plan' }; } };
let uploadContentAtCallTime;
const driver = {
async review(input) {
observed = input;
// Must read here: the temp copy is cleaned up in run()'s `finally` before it returns.
uploadContentAtCallTime = await fs.readFile(input.uploadPath, 'utf8');
return { responseText: 'review', conversationUrl: 'https://chatgpt.com/c/plan' };
},
};
const result = await run(['plan', planFile], { store, driver });
assert.equal(result.status, 'completed');
assert.equal(observed.uploadPath, planFile);
// The plan file's own path is the review-session identity and must never be the literal
// upload target — re-uploading one unchanging filename every pass is what caused ChatGPT's
// own UI to collision-rename it (plan-590(9).md) after enough retries.
assert.notEqual(observed.uploadPath, planFile);
assert.match(path.basename(observed.uploadPath), /^complete-plan-pass1\.md$/);
assert.equal(uploadContentAtCallTime, '# Complete plan\n');
assert.equal(observed.publish, false);
assert.match(observed.prompt, /Do not write anything to GitHub/);
});