diff --git a/bin/explorbot-cli.ts b/bin/explorbot-cli.ts index 4bb4007..b1fef45 100755 --- a/bin/explorbot-cli.ts +++ b/bin/explorbot-cli.ts @@ -40,6 +40,7 @@ interface CLIOptions { headless?: boolean; incognito?: boolean; session?: string | boolean; + spec?: string; } function buildExplorBotOptions(from: string | undefined, options: CLIOptions): ExplorBotOptions { @@ -52,6 +53,7 @@ function buildExplorBotOptions(from: string | undefined, options: CLIOptions): E headless: options.headless, incognito: options.incognito, session: options.session, + applicationSpec: options.spec, } as ExplorBotOptions; } @@ -64,6 +66,7 @@ function addCommonOptions(cmd: Command): Command { .option('-s, --show', 'Show browser window') .option('--headless', 'Run browser in headless mode') .option('--incognito', 'Run without recording experiences') + .option('--spec ', 'Use a Docbot application spec directory or index.md') .option('--session [file]', 'Save/restore browser session from file'); } diff --git a/boat/doc-collector/src/cli.ts b/boat/doc-collector/src/cli.ts index 3c57677..481cdb0 100644 --- a/boat/doc-collector/src/cli.ts +++ b/boat/doc-collector/src/cli.ts @@ -55,6 +55,7 @@ export function createDocsCommands(name = 'docs'): Command { console.log(`Skipped ${result.skipped.length} page(s)`); console.log(`Spec index: ${result.indexPath}`); console.log(`Pages dir: ${path.join(result.outputDir, 'pages')}`); + console.log(`Use in Explorbot: npx explorbot start ${startPath} --spec "${result.outputDir}"`); await bot.stop(); process.exit(0); diff --git a/boat/doc-collector/src/docs-renderer.ts b/boat/doc-collector/src/docs-renderer.ts index 54dd1eb..2368611 100644 --- a/boat/doc-collector/src/docs-renderer.ts +++ b/boat/doc-collector/src/docs-renderer.ts @@ -1,9 +1,11 @@ import path from 'node:path'; +import matter from 'gray-matter'; +import { APPLICATION_SPEC_FORMAT, APPLICATION_SPEC_VERSION } from '../../../src/application-spec-contract.ts'; import { type WebPageState } from '../../../src/state-manager.ts'; +import { normalizeInlineText } from '../../../src/utils/strings.ts'; import type { PageDocumentation, StateTransition } from './ai/documentarian.ts'; import type { DocumentationScreenshot } from './screenshots.ts'; -import { buildStateGraph, renderMermaidFromGraph, renderStateMapFromGraph, type DocumentedPage, type SkippedPage } from './state-diagram.ts'; -import { normalizeInlineText } from '../../../src/utils/strings.ts'; +import { type DocumentedPage, type SkippedPage, buildStateGraph, renderMermaidFromGraph, renderStateMapFromGraph } from './state-diagram.ts'; function renderPageDocumentation(state: WebPageState, documentation: PageDocumentation, screenshots: DocumentationScreenshot[] = []): string { const lines: string[] = []; @@ -101,7 +103,11 @@ function renderPageDocumentation(state: WebPageState, documentation: PageDocumen lines.push(''); } - return `${lines.join('\n').trimEnd()}\n`; + return matter.stringify(`${lines.join('\n').trimEnd()}\n`, { + url: state.url, + format: APPLICATION_SPEC_FORMAT, + version: APPLICATION_SPEC_VERSION, + }); } function renderSpecIndex(outputDir: string, startPath: string, pages: DocumentedPage[], skipped: SkippedPage[], maxPages: number): string { diff --git a/docs/doc-collection/basics.md b/docs/doc-collection/basics.md index 37e47ad..b982d2b 100644 --- a/docs/doc-collection/basics.md +++ b/docs/doc-collection/basics.md @@ -80,6 +80,20 @@ output/docs/ `state-diagram.mmd` is the same state-transition map as a standalone Mermaid file (no markdown fences), so other agents can embed or post-process it without re-rendering `index.md`. +## Use the spec in Explorbot + +Pass the generated directory or its `index.md` to any Explorbot web command: + +```bash +npx explorbot start / --spec output/docs +npx explorbot plan /admin/users --spec output/docs +npx explorbot explore / --spec output/docs +``` + +You can also set `dirs.spec: 'output/docs'` in `explorbot.config.js` to use it by default. Explorbot selects documentation for the current URL instead of loading the whole site spec into every prompt. Proven capabilities and observed transitions are supporting context; possible capabilities remain explicitly unverified until the live UI confirms them. + +The format is not tied to Docbot. See [Application Specs](../workflow/application-spec.md) to create or generate a compatible bundle with another tool. + Each page file follows the same shape: ```markdown diff --git a/docs/index.json b/docs/index.json index c008165..cd9ef8e 100644 --- a/docs/index.json +++ b/docs/index.json @@ -52,6 +52,7 @@ "description": "These pages apply to web and API testing alike", "pages": [ { "title": "Knowledge", "file": "workflow/knowledge.md", "description": "Teach Explorbot about your app" }, + { "title": "Application Specs", "file": "workflow/application-spec.md", "description": "Reuse versioned application documentation" }, { "title": "Test plans", "file": "workflow/test-plans.md", "description": "The plan file format and how plans are reused" }, { "title": "Planning styles", "file": "workflow/planning-styles.md", "description": "Normal, curious, psycho, and your own" }, { "title": "Reporting", "file": "workflow/reporting.md", "description": "Local reports and Testomat.io" }, diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index d453a6b..0d605df 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -335,6 +335,7 @@ Change the paths: ```javascript dirs: { + spec: './test/spec', knowledge: './test/knowledge', experience: './test/experience', output: './test/output', @@ -504,6 +505,7 @@ export default { // Directory paths dirs: { + spec: 'spec', // Application specification bundle knowledge: 'knowledge', // Domain knowledge files experience: 'experience', // Learned patterns output: 'output', // Test results and logs @@ -526,4 +528,5 @@ export default { - [Researcher agent](../web-testing/researcher.md) — Researcher configuration and usage - [Planner agent](../web-testing/planner.md) — planning styles and customization - [Knowledge files](../workflow/knowledge.md) — domain knowledge format +- [Application specs](../workflow/application-spec.md) — reusable application documentation format - [Observability](../contributing/observability.md) — Langfuse integration diff --git a/docs/workflow/application-spec.md b/docs/workflow/application-spec.md new file mode 100644 index 0000000..4a23a48 --- /dev/null +++ b/docs/workflow/application-spec.md @@ -0,0 +1,73 @@ +# Application Specs + +An application spec is a versioned Markdown bundle that gives Explorbot previously collected information about an application. It can be produced by Docbot, another documentation tool, or by hand. + +Live HTML, ARIA, and screenshots remain the source of truth. Explorbot uses matching spec pages as supporting context and does not load the whole bundle into every prompt. + +## Configure + +Set the bundle directory in `explorbot.config.js`: + +```javascript +export default { + dirs: { + spec: 'spec', + }, +}; +``` + +Paths are resolved from the project directory. Use `--spec ` on a web command to override the configured bundle for one run. Both the bundle directory and its `index.md` path are accepted. + +## Bundle structure + +```text +spec/ +|-- index.md +`-- pages/ + |-- home.md + `-- users.md +``` + +`index.md` is required and serves as a human-readable entry point. Its contents are not injected into agents. Page files may be nested anywhere below `pages/` and must use the contract below. + +## Page contract + +Every page is a Markdown file with YAML front matter: + +```markdown +--- +format: explorbot-application-spec +version: 1 +url: /users +--- + +# Users + +## Purpose + +Lists the application's users. + +## User Can + +- user can search users by name + Proof: A search field is visible above the user list. + +## User Might + +- user might export the user list + Signal: An unlabeled download control is present. +``` + +The front matter fields are mandatory: + +- `format` must be `explorbot-application-spec`. +- `version` must be `1`. +- `url` is the URL pattern used to select the page for the current browser state. It supports the same patterns as [knowledge files](./knowledge.md#url-patterns). + +The Markdown body is supplied to agents as written, so headings beyond those shown above are allowed. Use `User Can` only for observed capabilities and transitions. Put inferred or unverified capabilities under `User Might`; Explorbot will require confirmation from the live UI before relying on them. + +Screenshots and other relative links may be included for readers, but Explorbot currently consumes the Markdown text only. + +## Validation + +Explorbot rejects a bundle when `index.md` or `pages/` is missing, when it contains no page files, or when a page has an unsupported format, version, or missing URL. diff --git a/src/ai/navigator.ts b/src/ai/navigator.ts index 0599b4a..826db33 100644 --- a/src/ai/navigator.ts +++ b/src/ai/navigator.ts @@ -13,7 +13,7 @@ import { HooksRunner } from '../utils/hooks-runner.ts'; import { createDebug, pluralize, tag } from '../utils/logger.js'; import { loop, pause } from '../utils/loop.js'; import { RulesLoader } from '../utils/rules-loader.ts'; -import { extractStatePath } from '../utils/url-matcher.js'; +import { extractStatePath, matchesNavigationUrl } from '../utils/url-matcher.js'; import type { Agent, AgentDeps } from './agent.js'; import type { Conversation } from './conversation.js'; import type { Provider } from './provider.js'; @@ -134,7 +134,7 @@ class Navigator implements Agent { return false; } const currentUrl = this.getComparableCurrentUrl(stateManager, expectedUrl); - return normalizeUrl(currentUrl) === normalizeUrl(expectedUrl); + return matchesNavigationUrl(expectedUrl, currentUrl); } async visit(url: string): Promise { @@ -196,7 +196,7 @@ class Navigator implements Agent { const action = opts?.action ?? this.explorer.action(); const expectedUrl = opts?.expectedUrl; - const knowledge = this.knowledgeTracker.renderRelevantKnowledge(actionResult); + const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult); let experience = ''; if (!actionResult.isInsideIframe) { @@ -373,14 +373,14 @@ class Navigator implements Agent { if (expectedUrl) { if (page) { try { - await page.waitForURL((url: URL) => normalizeUrl(url.pathname) === normalizeUrl(expectedUrl), { timeout: 5000 }); + await page.waitForURL((url: URL) => matchesNavigationUrl(expectedUrl, `${url.pathname}${url.search}${url.hash}`), { timeout: 5000 }); } catch { // URL did not transition to expectedUrl within timeout } } const freshState = await this.explorer.capture(); const currentUrl = /^https?:\/\//i.test(expectedUrl) ? freshState.fullUrl || freshState.url || '' : freshState.url || ''; - const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && normalizeUrl(currentUrl) === normalizeUrl(expectedUrl); + const urlMatches = this.isSameExpectedOrigin(expectedUrl, action.stateManager) && matchesNavigationUrl(expectedUrl, currentUrl); const stateChanged = freshState.getStateHash() !== actionResult.getStateHash(); resolved = urlMatches && stateChanged; @@ -625,7 +625,7 @@ class Navigator implements Agent { return { verified: cachedVerification, successfulCodes: [], assertionSteps: [], totalAttempted: 0 }; } - const knowledge = this.knowledgeTracker.renderRelevantKnowledge(actionResult); + const knowledge = this.knowledgeTracker.renderRelevantContext(actionResult); let experience = ''; if (!actionResult.isInsideIframe) { diff --git a/src/ai/planner.ts b/src/ai/planner.ts index 879dfec..5fff89d 100644 --- a/src/ai/planner.ts +++ b/src/ai/planner.ts @@ -53,6 +53,7 @@ export class Planner extends PlannerBase implements Agent { provider: Provider; stateManager: StateManager; private experienceTracker: ExperienceTracker; + private knowledgeTracker: AgentDeps['knowledgeTracker']; MIN_TASKS = 3; MAX_TASKS = 12; @@ -70,6 +71,7 @@ export class Planner extends PlannerBase implements Agent { this.researcher = researcher; this.stateManager = deps.stateManager; this.experienceTracker = deps.stateManager.getExperienceTracker(); + this.knowledgeTracker = deps.knowledgeTracker; } setFisherman(fisherman: Fisherman): void { @@ -411,6 +413,11 @@ export class Planner extends PlannerBase implements Agent { `); + const applicationContext = this.knowledgeTracker.renderApplicationSpec(state); + if (applicationContext) { + conversation.addUserText(applicationContext); + } + conversation.addUserText(dedent` ${this.buildApproach(style)} diff --git a/src/ai/researcher.ts b/src/ai/researcher.ts index a2a834a..2dc5e44 100644 --- a/src/ai/researcher.ts +++ b/src/ai/researcher.ts @@ -438,7 +438,7 @@ export class Researcher extends ResearcherBase implements Agent { if (!this.actionResult) throw new Error('actionResult is not set'); const html = await this.actionResult.combinedHtml(); - const knowledge = this.knowledgeTracker.renderRelevantKnowledge(this.actionResult); + const knowledge = this.knowledgeTracker.renderRelevantContext(this.actionResult); const ariaSnapshot = this.actionResult.getCompactARIA(); diff --git a/src/ai/task-agent.ts b/src/ai/task-agent.ts index 9c635bc..3cc91e1 100644 --- a/src/ai/task-agent.ts +++ b/src/ai/task-agent.ts @@ -73,7 +73,7 @@ export abstract class TaskAgent { } protected getKnowledge(actionResult: ActionResult): string { - return this.getKnowledgeTracker().renderRelevantKnowledge(actionResult); + return this.getKnowledgeTracker().renderRelevantContext(actionResult); } protected getExperience(actionResult: ActionResult): string { diff --git a/src/ai/tester.ts b/src/ai/tester.ts index 8fdf4e0..7375842 100644 --- a/src/ai/tester.ts +++ b/src/ai/tester.ts @@ -62,6 +62,7 @@ export class Tester extends TaskAgent implements Agent { private seenUiMapUrls = new Set(); private lastAnalyzedStateHash: string | null = null; private stalledIterations = 0; + private hasSuccessfulAssertion = false; private readonly MAX_STALLED_ITERATIONS = 3; constructor(deps: AgentDeps, researcher: Researcher, navigator: Navigator, agentTools?: any) { @@ -110,6 +111,7 @@ export class Tester extends TaskAgent implements Agent { this.seenUiMapUrls.clear(); this.lastAnalyzedStateHash = null; this.stalledIterations = 0; + this.hasSuccessfulAssertion = false; this.stateManager.clearHistory(); this.resetFailureCount(); this.pilot?.reset(); @@ -312,9 +314,17 @@ export class Tester extends TaskAgent implements Agent { const allToolNames = result?.toolExecutions?.map((execution: any) => execution.toolName) || []; const successfulToolNames = result?.toolExecutions?.filter((execution: any) => execution.wasSuccessful)?.map((execution: any) => execution.toolName) || []; const actionPerformed = !!allToolNames.find((toolName: string) => this.ACTION_TOOLS.includes(toolName)); + const successfulActionPerformed = !!successfulToolNames.find((toolName: string) => this.ACTION_TOOLS.includes(toolName)); assertionPerformed = !!successfulToolNames.find((toolName: string) => this.ASSERTION_TOOLS.includes(toolName)); const wasSuccessful = result?.toolExecutions?.every((execution: any) => execution.wasSuccessful); + if (successfulActionPerformed) { + this.hasSuccessfulAssertion = false; + } + if (assertionPerformed) { + this.hasSuccessfulAssertion = true; + } + this.trackToolExecutions(result?.toolExecutions || []); if (this.consecutiveEmptyResults >= 5) { @@ -464,6 +474,11 @@ export class Tester extends TaskAgent implements Agent { this.stalledIterations++; if (this.stalledIterations < this.MAX_STALLED_ITERATIONS) return false; + if (this.hasSuccessfulAssertion) { + task.addNote('No further browser progress after successful verification; requesting final review'); + return true; + } + task.addNote('No browser progress after repeated attempts on unchanged page', TestResult.FAILED); task.finish(TestResult.FAILED); return true; diff --git a/src/application-spec-contract.ts b/src/application-spec-contract.ts new file mode 100644 index 0000000..1bccf40 --- /dev/null +++ b/src/application-spec-contract.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; + +export const APPLICATION_SPEC_FORMAT = 'explorbot-application-spec'; +export const APPLICATION_SPEC_VERSION = 1; + +export const APPLICATION_SPEC_PAGE_SCHEMA = z.object({ + format: z.literal(APPLICATION_SPEC_FORMAT), + version: z.literal(APPLICATION_SPEC_VERSION), + url: z.string().trim().min(1), +}); diff --git a/src/application-spec.ts b/src/application-spec.ts new file mode 100644 index 0000000..2caa4c2 --- /dev/null +++ b/src/application-spec.ts @@ -0,0 +1,87 @@ +import { existsSync, statSync } from 'node:fs'; +import path from 'node:path'; +import dedent from 'dedent'; +import { ActionResult } from './action-result.ts'; +import { APPLICATION_SPEC_PAGE_SCHEMA } from './application-spec-contract.ts'; +import { ConfigParser } from './config.ts'; +import { tag } from './utils/logger.ts'; +import { loadMarkdownFiles } from './utils/markdown-files.ts'; + +export class ApplicationSpec { + private pages: ApplicationSpecPage[] = []; + readonly sourcePath: string; + + constructor(sourcePath: string) { + this.sourcePath = this.resolveSourcePath(sourcePath); + this.load(); + } + + renderFor(state: ActionResult): string { + const relevant = this.pages.filter((page) => state.isMatchedBy({ url: page.url })); + if (relevant.length === 0) return ''; + + tag('operation').log(`Found application specification for ${state.url}`); + return dedent` + + This is previously collected application documentation. Treat User Can and observed state transitions as supporting context, not as a replacement for the current page state. Treat User Might as unverified possibilities that must be confirmed before use. + + ${relevant.map((page) => page.content).join('\n\n')} + + `; + } + + get pageCount(): number { + return this.pages.length; + } + + private load(): void { + if (!existsSync(this.sourcePath)) { + throw new Error(`Application spec not found: ${this.sourcePath}`); + } + + const isDirectory = statSync(this.sourcePath).isDirectory(); + if (!isDirectory && path.basename(this.sourcePath).toLowerCase() !== 'index.md') { + throw new Error(`Application spec file must be index.md: ${this.sourcePath}`); + } + + const bundlePath = isDirectory ? this.sourcePath : path.dirname(this.sourcePath); + const indexPath = path.join(bundlePath, 'index.md'); + if (!existsSync(indexPath)) { + throw new Error(`Application spec index not found: ${indexPath}`); + } + + const pagesPath = path.join(bundlePath, 'pages'); + if (!existsSync(pagesPath)) { + throw new Error(`Application spec pages directory not found: ${pagesPath}`); + } + + for (const file of loadMarkdownFiles(pagesPath, { recursive: true })) { + const parsed = APPLICATION_SPEC_PAGE_SCHEMA.safeParse(file.data); + if (!parsed.success && parsed.error.issues.some((issue) => issue.path[0] === 'format')) { + throw new Error(`Invalid application spec format in ${file.filePath}`); + } + if (!parsed.success && parsed.error.issues.some((issue) => issue.path[0] === 'version')) { + throw new Error(`Unsupported application spec version in ${file.filePath}: ${String(file.data.version)}`); + } + if (!parsed.success) { + throw new Error(`Application spec page URL is missing in ${file.filePath}`); + } + this.pages.push({ url: parsed.data.url, content: file.content.trim() }); + } + + if (this.pages.length === 0) { + throw new Error(`Application spec contains no documented pages: ${pagesPath}`); + } + } + + private resolveSourcePath(sourcePath: string): string { + if (path.isAbsolute(sourcePath)) return path.resolve(sourcePath); + const configParser = ConfigParser.getInstance(); + return path.resolve(configParser.resolveProjectDir(sourcePath)); + } +} + +interface ApplicationSpecPage { + url: string; + content: string; +} diff --git a/src/config.ts b/src/config.ts index 30dc4d1..e0e3156 100644 --- a/src/config.ts +++ b/src/config.ts @@ -227,6 +227,7 @@ interface ExplorbotConfig { knowledge: string; experience: string; output: string; + spec?: string; }; experience?: { maxReadLines?: number; diff --git a/src/explorbot.ts b/src/explorbot.ts index 29991ab..342feac 100644 --- a/src/explorbot.ts +++ b/src/explorbot.ts @@ -46,6 +46,7 @@ export interface ExplorBotOptions { headless?: boolean; incognito?: boolean; session?: string | boolean; + applicationSpec?: string; } export type UserResolveFunction = (error?: Error, showWelcome?: boolean) => Promise; @@ -149,7 +150,7 @@ export class ExplorBot { } knowledgeTracker(): KnowledgeTracker { - return (this._knowledgeTracker ||= new KnowledgeTracker()); + return (this._knowledgeTracker ||= new KnowledgeTracker(this.options.applicationSpec)); } experienceTracker(): ExperienceTracker { diff --git a/src/knowledge-tracker.ts b/src/knowledge-tracker.ts index 13eeca4..37cb6a2 100644 --- a/src/knowledge-tracker.ts +++ b/src/knowledge-tracker.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import dedent from 'dedent'; import matter from 'gray-matter'; import { ActionResult } from './action-result.js'; +import { ApplicationSpec } from './application-spec.ts'; import { ConfigParser } from './config.js'; import { getCliName } from './utils/cli-name.ts'; import { createDebug, pluralize, tag } from './utils/logger.js'; @@ -24,8 +25,9 @@ export class KnowledgeTracker { private knowledgeDir: string; private knowledgeFiles: Knowledge[] = []; private isLoaded = false; + private applicationSpec?: ApplicationSpec; - constructor() { + constructor(applicationSpecPath?: string) { const configParser = ConfigParser.getInstance(); const config = configParser.getConfig(); this.knowledgeDir = configParser.resolveProjectDir(config.dirs?.knowledge || 'knowledge'); @@ -33,6 +35,12 @@ export class KnowledgeTracker { if (!existsSync(this.knowledgeDir)) { mkdirSync(this.knowledgeDir, { recursive: true }); } + + const specPath = applicationSpecPath || config.dirs?.spec; + if (specPath) { + this.applicationSpec = new ApplicationSpec(specPath); + tag('info').log(`Loaded application spec with ${this.applicationSpec.pageCount} documented pages`); + } } private loadKnowledgeFiles(): void { @@ -79,6 +87,14 @@ export class KnowledgeTracker { `; } + renderRelevantContext(state: ActionResult): string { + return [this.renderRelevantKnowledge(state), this.renderApplicationSpec(state)].filter(Boolean).join('\n\n'); + } + + renderApplicationSpec(state: ActionResult): string { + return this.applicationSpec?.renderFor(state) || ''; + } + addKnowledge(urlPattern: string, description: string): { filename: string; filePath: string; isNewFile: boolean } { const configParser = ConfigParser.getInstance(); const configPath = configParser.getConfigPath(); diff --git a/src/utils/url-matcher.ts b/src/utils/url-matcher.ts index 7198a3c..ca33146 100644 --- a/src/utils/url-matcher.ts +++ b/src/utils/url-matcher.ts @@ -93,3 +93,13 @@ export function extractStatePath(url: string): string { return url; } } + +export function matchesNavigationUrl(expected: string, current: string): boolean { + const expectedPath = extractStatePath(expected); + let currentPath = extractStatePath(current); + if (!expectedPath.includes('#')) { + currentPath = currentPath.split('#')[0]; + } + const normalize = (value: string) => value.replace(/^\/+|\/+$/g, '').toLowerCase(); + return normalize(expectedPath) === normalize(currentPath); +} diff --git a/tests/integration/planner.test.ts b/tests/integration/planner.test.ts index ece52b5..3ff1af5 100644 --- a/tests/integration/planner.test.ts +++ b/tests/integration/planner.test.ts @@ -65,7 +65,7 @@ const fakeState = { function createMockDeps(state = fakeState) { const mockExperienceTracker = { getSuccessfulExperience: () => [] }; - const mockKnowledgeTracker = { getRelevantKnowledge: () => [] }; + const mockKnowledgeTracker = { getRelevantKnowledge: () => [], renderApplicationSpec: () => '' }; const mockStateManager = { getCurrentState: () => state, getVisitCount: () => 0, diff --git a/tests/integration/researcher-browser.test.ts b/tests/integration/researcher-browser.test.ts index 947ff72..fe4fde4 100644 --- a/tests/integration/researcher-browser.test.ts +++ b/tests/integration/researcher-browser.test.ts @@ -105,7 +105,7 @@ describe('Researcher with real browser + aimock', () => { explorer, config: ConfigParser.getInstance().getConfig(), stateManager: mockStateManager, - knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', renderRelevantContext: () => '' }, requestStore: {}, playwrightRecorder: {}, }; diff --git a/tests/integration/researcher-sections.test.ts b/tests/integration/researcher-sections.test.ts index c693a55..ff73c0b 100644 --- a/tests/integration/researcher-sections.test.ts +++ b/tests/integration/researcher-sections.test.ts @@ -55,7 +55,7 @@ function createMockDeps(configOverrides: Record = {}, locatorCo explorer, config, stateManager, - knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '' }, + knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', renderRelevantContext: () => '' }, requestStore: {}, playwrightRecorder: {}, }; diff --git a/tests/integration/researcher.test.ts b/tests/integration/researcher.test.ts index 1f1b79a..7735653 100644 --- a/tests/integration/researcher.test.ts +++ b/tests/integration/researcher.test.ts @@ -56,6 +56,7 @@ function createMockDeps(state = fakeState) { const mockKnowledgeTracker = { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', + renderRelevantContext: () => '', }; const mockStateManager = { getCurrentState: () => state, diff --git a/tests/unit/application-spec.test.ts b/tests/unit/application-spec.test.ts new file mode 100644 index 0000000..3138bc5 --- /dev/null +++ b/tests/unit/application-spec.test.ts @@ -0,0 +1,93 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import matter from 'gray-matter'; +import { ActionResult } from '../../src/action-result.ts'; +import { APPLICATION_SPEC_FORMAT, APPLICATION_SPEC_VERSION } from '../../src/application-spec-contract.ts'; +import { ApplicationSpec } from '../../src/application-spec.ts'; +import { ConfigParser } from '../../src/config.ts'; + +const projectDir = '/tmp/explorbot-application-spec-test'; + +describe('ApplicationSpec', () => { + beforeEach(() => { + if (existsSync(projectDir)) rmSync(projectDir, { recursive: true, force: true }); + mkdirSync(`${projectDir}/output/docs/pages`, { recursive: true }); + writeFileSync(`${projectDir}/output/docs/index.md`, '# Website Spec', 'utf8'); + writeFileSync(`${projectDir}/config.ts`, '', 'utf8'); + + const configParser = ConfigParser.getInstance(); + (configParser as any).config = { + playwright: { browser: 'chromium', url: 'https://example.test' }, + ai: { model: 'test' }, + dirs: { knowledge: 'knowledge' }, + }; + (configParser as any).configPath = `${projectDir}/config.ts`; + }); + + afterEach(() => { + if (existsSync(projectDir)) rmSync(projectDir, { recursive: true, force: true }); + }); + + it('loads a Docbot directory and renders only the matching page', () => { + writePage('users.md', '/users', 'User list'); + writePage('settings.md', '/settings', 'Workspace settings'); + + const spec = new ApplicationSpec('output/docs'); + const rendered = spec.renderFor(new ActionResult({ url: '/users' })); + + expect(spec.pageCount).toBe(2); + expect(rendered).toContain(''); + expect(rendered).toContain('User list'); + expect(rendered).not.toContain('Workspace settings'); + expect(rendered).toContain('User Might as unverified'); + }); + + it('accepts the generated index.md path', () => { + writePage('users.md', '/users', 'User list'); + const spec = new ApplicationSpec('output/docs/index.md'); + + expect(spec.renderFor(new ActionResult({ url: '/users' }))).toContain('User list'); + }); + + it('returns no context for an undocumented URL', () => { + writePage('users.md', '/users', 'User list'); + + const spec = new ApplicationSpec('output/docs'); + + expect(spec.renderFor(new ActionResult({ url: '/billing' }))).toBe(''); + }); + + it('reports a friendly error for an invalid bundle', () => { + expect(() => new ApplicationSpec('missing/docs')).toThrow('Application spec not found'); + }); + + it('rejects an unsupported page format', () => { + writeFileSync(`${projectDir}/output/docs/pages/invalid.md`, matter.stringify('# /invalid', { url: '/invalid', format: 'unknown', version: APPLICATION_SPEC_VERSION }), 'utf8'); + + expect(() => new ApplicationSpec('output/docs')).toThrow('Invalid application spec format'); + }); + + it('rejects an unsupported page version', () => { + writeFileSync(`${projectDir}/output/docs/pages/invalid.md`, matter.stringify('# /invalid', { url: '/invalid', format: APPLICATION_SPEC_FORMAT, version: APPLICATION_SPEC_VERSION + 1 }), 'utf8'); + + expect(() => new ApplicationSpec('output/docs')).toThrow('Unsupported application spec version'); + }); + + it('rejects a page without a URL', () => { + writeFileSync(`${projectDir}/output/docs/pages/invalid.md`, matter.stringify('# Invalid', { format: APPLICATION_SPEC_FORMAT, version: APPLICATION_SPEC_VERSION }), 'utf8'); + + expect(() => new ApplicationSpec('output/docs')).toThrow('Application spec page URL is missing'); + }); +}); + +function writePage(filename: string, url: string, purpose: string): void { + writeFileSync( + `${projectDir}/output/docs/pages/${filename}`, + matter.stringify(`# ${url}\n\n## Purpose\n\n${purpose}\n\n## User Can\n\n- user can view the page -> page-level\n`, { + url, + format: APPLICATION_SPEC_FORMAT, + version: APPLICATION_SPEC_VERSION, + }), + 'utf8' + ); +} diff --git a/tests/unit/knowledge-tracker.test.ts b/tests/unit/knowledge-tracker.test.ts index 68fdba1..5ad7701 100644 --- a/tests/unit/knowledge-tracker.test.ts +++ b/tests/unit/knowledge-tracker.test.ts @@ -3,17 +3,22 @@ import { existsSync, rmSync } from 'node:fs'; import { mkdirSync, writeFileSync } from 'node:fs'; import matter from 'gray-matter'; import { ActionResult } from '../../src/action-result.js'; +import { APPLICATION_SPEC_FORMAT, APPLICATION_SPEC_VERSION } from '../../src/application-spec-contract.ts'; import { ConfigParser } from '../../src/config'; import { KnowledgeTracker } from '../../src/knowledge-tracker'; import { clearRegisteredSecrets, redactSecrets } from '../../src/utils/secrets'; const knowledgeDir = '/tmp/explorbot-test-knowledge'; +const applicationSpecDir = '/tmp/explorbot-test-application-spec'; describe('KnowledgeTracker', () => { beforeEach(() => { if (existsSync(knowledgeDir)) { rmSync(knowledgeDir, { recursive: true, force: true }); } + if (existsSync(applicationSpecDir)) { + rmSync(applicationSpecDir, { recursive: true, force: true }); + } mkdirSync(knowledgeDir, { recursive: true }); const configParser = ConfigParser.getInstance(); @@ -29,6 +34,9 @@ describe('KnowledgeTracker', () => { if (existsSync(knowledgeDir)) { rmSync(knowledgeDir, { recursive: true, force: true }); } + if (existsSync(applicationSpecDir)) { + rmSync(applicationSpecDir, { recursive: true, force: true }); + } }); function writeKnowledgeFile(filename: string, url: string, content: string) { @@ -54,6 +62,49 @@ describe('KnowledgeTracker', () => { expect(rendered).toContain('Use admin credentials'); expect(rendered).toContain(''); }); + + it('combines matching knowledge with a configured application spec', () => { + writeKnowledgeFile('login.md', '/login', 'Use admin credentials'); + mkdirSync(`${applicationSpecDir}/pages`, { recursive: true }); + writeFileSync(`${applicationSpecDir}/index.md`, '# Website Spec', 'utf8'); + writeFileSync( + `${applicationSpecDir}/pages/login.md`, + matter.stringify('# /login\n\n## Purpose\n\nSign in to the application.', { + url: '/login', + format: APPLICATION_SPEC_FORMAT, + version: APPLICATION_SPEC_VERSION, + }), + 'utf8' + ); + const tracker = new KnowledgeTracker(applicationSpecDir); + const state = new ActionResult({ url: '/login', html: '' }); + + const rendered = tracker.renderRelevantContext(state); + + expect(rendered).toContain(''); + expect(rendered).toContain(''); + expect(rendered).toContain('Sign in to the application.'); + }); + + it('loads an application spec from the configured directory', () => { + mkdirSync(`${applicationSpecDir}/pages`, { recursive: true }); + writeFileSync(`${applicationSpecDir}/index.md`, '# Website Spec', 'utf8'); + writeFileSync( + `${applicationSpecDir}/pages/login.md`, + matter.stringify('# /login\n\n## Purpose\n\nSign in to the application.', { + url: '/login', + format: APPLICATION_SPEC_FORMAT, + version: APPLICATION_SPEC_VERSION, + }), + 'utf8' + ); + (ConfigParser.getInstance() as any).config.dirs.spec = 'explorbot-test-application-spec'; + + const tracker = new KnowledgeTracker(); + const rendered = tracker.renderApplicationSpec(new ActionResult({ url: '/login' })); + + expect(rendered).toContain('Sign in to the application.'); + }); }); describe('interpolateVars', () => { diff --git a/tests/unit/tester-error-page.test.ts b/tests/unit/tester-error-page.test.ts index b758f00..54b064a 100644 --- a/tests/unit/tester-error-page.test.ts +++ b/tests/unit/tester-error-page.test.ts @@ -65,6 +65,7 @@ describe('Tester error page handling', () => { knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', + renderRelevantContext: () => '', }, requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, playwrightRecorder: {}, @@ -125,6 +126,7 @@ describe('Tester error page handling', () => { knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', + renderRelevantContext: () => '', }, requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, playwrightRecorder: {}, diff --git a/tests/unit/tester-focus-scope.test.ts b/tests/unit/tester-focus-scope.test.ts index 02ec3d7..6adfeb4 100644 --- a/tests/unit/tester-focus-scope.test.ts +++ b/tests/unit/tester-focus-scope.test.ts @@ -28,6 +28,7 @@ function buildTester(): Tester { knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', + renderRelevantContext: () => '', }, requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, playwrightRecorder: {}, @@ -61,6 +62,7 @@ function buildTesterWithExperience(): Tester { knowledgeTracker: { getRelevantKnowledge: () => [], renderRelevantKnowledge: () => '', + renderRelevantContext: () => '', }, requestStore: { clear: () => {}, onFailedRequest: () => () => {}, getFailedRequests: () => [] }, playwrightRecorder: {}, @@ -144,3 +146,19 @@ describe('Tester experience context', () => { expect(scenarioBlock).not.toContain('```'); }); }); + +describe('Tester stalled execution', () => { + it('hands a verified scenario to final review without marking it failed', () => { + const tester = buildTester(); + const task = new Test('filter items', 'normal', 'filtered items appear', '/page'); + const state = buildState('- main:', '/page'); + (tester as any).stateManager.getCurrentState = () => state; + (tester as any).hasSuccessfulAssertion = true; + + expect((tester as any).shouldStopForStalledExecution(task, state, [])).toBe(false); + expect((tester as any).shouldStopForStalledExecution(task, state, [])).toBe(false); + expect((tester as any).shouldStopForStalledExecution(task, state, [])).toBe(true); + expect(task.hasFinished).toBe(false); + expect(task.getPrintableNotes()).toContain('No further browser progress after successful verification; requesting final review'); + }); +}); diff --git a/tests/unit/url-matcher.test.ts b/tests/unit/url-matcher.test.ts index 71128fb..ad47713 100644 --- a/tests/unit/url-matcher.test.ts +++ b/tests/unit/url-matcher.test.ts @@ -1,7 +1,7 @@ import { beforeEach, describe, expect, it } from 'bun:test'; import { ConfigParser } from '../../src/config'; import { normalizeUrl } from '../../src/state-manager'; -import { extractStatePath, generalizeSegment, generalizeUrl, hasDynamicUrlSegment, isDynamicSegment, matchesUrl } from '../../src/utils/url-matcher'; +import { extractStatePath, generalizeSegment, generalizeUrl, hasDynamicUrlSegment, isDynamicSegment, matchesNavigationUrl, matchesUrl } from '../../src/utils/url-matcher'; describe('url-matcher', () => { beforeEach(() => { @@ -184,6 +184,20 @@ describe('url-matcher', () => { }); }); + describe('matchesNavigationUrl', () => { + it('accepts a fragment added by a client-side router when none was requested', () => { + expect(matchesNavigationUrl('/todomvc/', '/todomvc/#/')).toBe(true); + }); + + it('requires an explicitly requested fragment', () => { + expect(matchesNavigationUrl('/todomvc/#/active', '/todomvc/#/completed')).toBe(false); + }); + + it('compares absolute and relative URLs by path', () => { + expect(matchesNavigationUrl('/users', 'https://example.test/users#details')).toBe(true); + }); + }); + describe('normalizeUrl', () => { it('treats repeated leading slashes as a relative path, not a protocol-relative URL', () => { expect(normalizeUrl('///series/page/57/')).toBe('series/page/57');