From b858beaf8995b6bda03fc1cf8f168074f18d9a3e Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Fri, 31 Oct 2025 18:46:17 +0000 Subject: [PATCH 1/3] create some checks --- src/checks.ts | 35 ++++++++++++++ src/checks/client.ts | 47 ++++++++++++++++++ src/checks/index.ts | 7 +++ src/checks/server.ts | 36 ++++++++++++++ src/scenarios/client/initialize.ts | 7 ++- src/scenarios/index.ts | 15 +++++- src/scenarios/server/server_initialize.ts | 58 +++++++++++++++++++++++ src/types.ts | 6 +++ 8 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 src/checks/client.ts create mode 100644 src/checks/index.ts create mode 100644 src/checks/server.ts create mode 100644 src/scenarios/server/server_initialize.ts diff --git a/src/checks.ts b/src/checks.ts index 11a1feee..17068988 100644 --- a/src/checks.ts +++ b/src/checks.ts @@ -45,3 +45,38 @@ export function createServerInfoCheck(serverInfo: { name: string; version: strin } }; } + +export function createServerInitializationCheck(initializeResponse: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { + const result = initializeResponse?.result; + const protocolVersion = result?.protocolVersion; + const serverInfo = result?.serverInfo; + const capabilities = result?.capabilities; + + const errors: string[] = []; + if (!initializeResponse?.jsonrpc) errors.push('Missing jsonrpc field'); + if (!initializeResponse?.id) errors.push('Missing id field'); + if (!result) errors.push('Missing result field'); + if (!protocolVersion) errors.push('Missing protocolVersion in result'); + if (protocolVersion !== expectedSpecVersion) errors.push(`Protocol version mismatch: expected ${expectedSpecVersion}, got ${protocolVersion}`); + if (!serverInfo) errors.push('Missing serverInfo in result'); + if (!serverInfo?.name) errors.push('Missing server name in serverInfo'); + if (!serverInfo?.version) errors.push('Missing server version in serverInfo'); + if (capabilities === undefined) errors.push('Missing capabilities in result'); + + const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; + + return { + id: 'mcp-server-initialization', + name: 'MCPServerInitialization', + description: 'Validates that MCP server properly responds to initialize request', + status, + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + expectedSpecVersion, + ...initializeResponse + }, + errorMessage: errors.length > 0 ? errors.join('; ') : undefined, + logs: errors.length > 0 ? errors : undefined + }; +} diff --git a/src/checks/client.ts b/src/checks/client.ts new file mode 100644 index 00000000..0cc0ca6e --- /dev/null +++ b/src/checks/client.ts @@ -0,0 +1,47 @@ +import { ConformanceCheck, CheckStatus } from '../types.js'; + +export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck { + return { + id: 'server-info', + name: 'ServerInfo', + description: 'Test server info returned to client', + status: 'INFO', + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + serverName: serverInfo.name, + serverVersion: serverInfo.version + } + }; +} + +export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { + const protocolVersionSent = initializeRequest?.protocolVersion; + const versionMatch = protocolVersionSent === expectedSpecVersion; + + const errors: string[] = []; + if (!protocolVersionSent) errors.push('Protocol version not provided'); + if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`); + if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing'); + if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing'); + + const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; + + return { + id: 'mcp-client-initialization', + name: 'MCPClientInitialization', + description: 'Validates that MCP client properly initializes with server', + status, + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + protocolVersionSent, + expectedSpecVersion, + versionMatch, + clientName: initializeRequest?.clientInfo?.name, + clientVersion: initializeRequest?.clientInfo?.version + }, + errorMessage: errors.length > 0 ? errors.join('; ') : undefined, + logs: errors.length > 0 ? errors : undefined + }; +} \ No newline at end of file diff --git a/src/checks/index.ts b/src/checks/index.ts new file mode 100644 index 00000000..0ce84aad --- /dev/null +++ b/src/checks/index.ts @@ -0,0 +1,7 @@ +// Namespaced exports for client and server checks +import * as client from './client.js'; +import * as server from './server.js'; + + +export const clientChecks = client; +export const serverChecks = server; diff --git a/src/checks/server.ts b/src/checks/server.ts new file mode 100644 index 00000000..69d823a0 --- /dev/null +++ b/src/checks/server.ts @@ -0,0 +1,36 @@ +import { ConformanceCheck, CheckStatus } from '../types.js'; + +export function createServerInitializationCheck(initializeResponse: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { + const result = initializeResponse?.result; + const protocolVersion = result?.protocolVersion; + const serverInfo = result?.serverInfo; + const capabilities = result?.capabilities; + + const errors: string[] = []; + if (!initializeResponse?.jsonrpc) errors.push('Missing jsonrpc field'); + if (!initializeResponse?.id) errors.push('Missing id field'); + if (!result) errors.push('Missing result field'); + if (!protocolVersion) errors.push('Missing protocolVersion in result'); + if (protocolVersion !== expectedSpecVersion) errors.push(`Protocol version mismatch: expected ${expectedSpecVersion}, got ${protocolVersion}`); + if (!serverInfo) errors.push('Missing serverInfo in result'); + if (!serverInfo?.name) errors.push('Missing server name in serverInfo'); + if (!serverInfo?.version) errors.push('Missing server version in serverInfo'); + if (capabilities === undefined) errors.push('Missing capabilities in result'); + + const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; + + return { + id: 'mcp-server-initialization', + name: 'MCPServerInitialization', + description: 'Validates that MCP server properly responds to initialize request', + status, + timestamp: new Date().toISOString(), + specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], + details: { + expectedSpecVersion, + ...initializeResponse + }, + errorMessage: errors.length > 0 ? errors.join('; ') : undefined, + logs: errors.length > 0 ? errors : undefined + }; +} \ No newline at end of file diff --git a/src/scenarios/client/initialize.ts b/src/scenarios/client/initialize.ts index ff046f5a..80e9d075 100644 --- a/src/scenarios/client/initialize.ts +++ b/src/scenarios/client/initialize.ts @@ -1,6 +1,6 @@ import http from 'http'; import { Scenario, ScenarioUrls, ConformanceCheck } from '../../types.js'; -import { createClientInitializationCheck, createServerInfoCheck } from '../../checks.js'; +import { clientChecks } from '../../checks/index.js'; export class InitializeScenario implements Scenario { name = 'initialize'; @@ -96,7 +96,7 @@ export class InitializeScenario implements Scenario { private handleInitialize(request: any, res: http.ServerResponse): void { const initializeRequest = request.params; - const check = createClientInitializationCheck(initializeRequest); + const check = clientChecks.createClientInitializationCheck(initializeRequest); this.checks.push(check); const serverInfo = { @@ -104,8 +104,7 @@ export class InitializeScenario implements Scenario { version: '1.0.0' }; - const serverInfoCheck = createServerInfoCheck(serverInfo); - this.checks.push(serverInfoCheck); + this.checks.push(clientChecks.createServerInfoCheck(serverInfo)); const response = { jsonrpc: '2.0', diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 2682a6ea..72384bc8 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -1,12 +1,17 @@ -import { Scenario } from '../types.js'; +import { Scenario, ClientScenario } from '../types.js'; import { InitializeScenario } from './client/initialize.js'; import { ToolsCallScenario } from './client/tools_call.js'; +import { ServerInitializeClientScenario } from './server/server_initialize.js'; export const scenarios = new Map([ ['initialize', new InitializeScenario()], ['tools-call', new ToolsCallScenario()] ]); +export const clientScenarios = new Map([ + ['server-initialize', new ServerInitializeClientScenario()] +]); + export function registerScenario(name: string, scenario: Scenario): void { scenarios.set(name, scenario); } @@ -15,6 +20,14 @@ export function getScenario(name: string): Scenario | undefined { return scenarios.get(name); } +export function getClientScenario(name: string): ClientScenario | undefined { + return clientScenarios.get(name); +} + export function listScenarios(): string[] { return Array.from(scenarios.keys()); } + +export function listClientScenarios(): string[] { + return Array.from(clientScenarios.keys()); +} diff --git a/src/scenarios/server/server_initialize.ts b/src/scenarios/server/server_initialize.ts new file mode 100644 index 00000000..c79cc558 --- /dev/null +++ b/src/scenarios/server/server_initialize.ts @@ -0,0 +1,58 @@ +import { ClientScenario, ConformanceCheck } from '../../types.js'; +import { serverChecks } from '../../checks/index.js'; + +export class ServerInitializeClientScenario implements ClientScenario { + name = 'server-initialize'; + description = 'Acts as MCP client to test external server initialization'; + + async run(serverUrl: string): Promise { + const checks: ConformanceCheck[] = []; + + try { + const response = await fetch(serverUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: { + protocolVersion: '2025-06-18', + capabilities: {}, + clientInfo: { + name: 'conformance-test-client', + version: '1.0.0' + } + } + }) + }); + + if (!response.ok) { + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + const result = await response.json(); + const check = serverChecks.createServerInitializationCheck(result); + checks.push(check); + } catch (error) { + checks.push({ + id: 'server-initialize-request', + name: 'ServerInitializeRequest', + description: 'Tests server response to initialize request', + status: 'FAILURE', + timestamp: new Date().toISOString(), + errorMessage: `Failed to send initialize request: ${error instanceof Error ? error.message : String(error)}`, + specReferences: [ + { + id: 'MCP-Initialize', + url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization' + } + ] + }); + } + + return checks; + } +} \ No newline at end of file diff --git a/src/types.ts b/src/types.ts index 1bd77334..e49d08f4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -30,3 +30,9 @@ export interface Scenario { stop(): Promise; getChecks(): ConformanceCheck[]; } + +export interface ClientScenario { + name: string; + description: string; + run(serverUrl: string): Promise; +} From 46017abc610a0c1c151d618accdf29f4bddbcc0b Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Fri, 31 Oct 2025 18:55:47 +0000 Subject: [PATCH 2/3] basic initialize working --- src/checks/server.ts | 2 +- src/scenarios/index.ts | 2 +- src/scenarios/server/server_initialize.ts | 28 +++++- src/server-runner.ts | 101 ++++++++++++++++++++++ 4 files changed, 129 insertions(+), 4 deletions(-) create mode 100644 src/server-runner.ts diff --git a/src/checks/server.ts b/src/checks/server.ts index 69d823a0..cf4f8dab 100644 --- a/src/checks/server.ts +++ b/src/checks/server.ts @@ -28,7 +28,7 @@ export function createServerInitializationCheck(initializeResponse: any, expecte specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], details: { expectedSpecVersion, - ...initializeResponse + response: initializeResponse }, errorMessage: errors.length > 0 ? errors.join('; ') : undefined, logs: errors.length > 0 ? errors : undefined diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 72384bc8..0744c24f 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -9,7 +9,7 @@ export const scenarios = new Map([ ]); export const clientScenarios = new Map([ - ['server-initialize', new ServerInitializeClientScenario()] + ['initialize', new ServerInitializeClientScenario()] ]); export function registerScenario(name: string, scenario: Scenario): void { diff --git a/src/scenarios/server/server_initialize.ts b/src/scenarios/server/server_initialize.ts index c79cc558..53749972 100644 --- a/src/scenarios/server/server_initialize.ts +++ b/src/scenarios/server/server_initialize.ts @@ -13,6 +13,7 @@ export class ServerInitializeClientScenario implements ClientScenario { method: 'POST', headers: { 'Content-Type': 'application/json', + 'Accept': 'application/json, text/event-stream', }, body: JSON.stringify({ jsonrpc: '2.0', @@ -30,10 +31,29 @@ export class ServerInitializeClientScenario implements ClientScenario { }); if (!response.ok) { - throw new Error(`HTTP ${response.status}: ${response.statusText}`); + const responseBody = await response.text(); + throw new Error(`HTTP ${response.status}: ${response.statusText}. Response body: ${responseBody}`); } - const result = await response.json(); + const responseText = await response.text(); + + // Handle SSE format + let result; + if (responseText.startsWith('event:') || responseText.includes('\ndata:')) { + // Parse SSE format - extract JSON from data: lines + const lines = responseText.split('\n'); + const dataLines = lines.filter(line => line.startsWith('data: ')); + if (dataLines.length > 0) { + const jsonData = dataLines[0].substring(6); // Remove 'data: ' prefix + result = JSON.parse(jsonData); + } else { + throw new Error(`SSE response without data line: ${responseText}`); + } + } else { + // Regular JSON response + result = JSON.parse(responseText); + } + const check = serverChecks.createServerInitializationCheck(result); checks.push(check); } catch (error) { @@ -44,6 +64,10 @@ export class ServerInitializeClientScenario implements ClientScenario { status: 'FAILURE', timestamp: new Date().toISOString(), errorMessage: `Failed to send initialize request: ${error instanceof Error ? error.message : String(error)}`, + details: { + error: error instanceof Error ? error.message : String(error), + serverUrl + }, specReferences: [ { id: 'MCP-Initialize', diff --git a/src/server-runner.ts b/src/server-runner.ts new file mode 100644 index 00000000..8228c180 --- /dev/null +++ b/src/server-runner.ts @@ -0,0 +1,101 @@ +import { promises as fs } from 'fs'; +import path from 'path'; +import { ConformanceCheck } from './types.js'; +import { getClientScenario } from './scenarios/index.js'; + +async function ensureResultsDir(): Promise { + const resultsDir = path.join(process.cwd(), 'results'); + await fs.mkdir(resultsDir, { recursive: true }); + return resultsDir; +} + +function createResultDir(scenario: string): string { + const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); + return path.join('results', `server-${scenario}-${timestamp}`); +} + +export async function runServerConformanceTest( + serverUrl: string, + scenarioName: string +): Promise<{ + checks: ConformanceCheck[]; + resultDir: string; +}> { + await ensureResultsDir(); + const resultDir = createResultDir(scenarioName); + await fs.mkdir(resultDir, { recursive: true }); + + const scenario = getClientScenario(scenarioName); + if (!scenario) { + throw new Error(`Unknown client scenario: ${scenarioName}`); + } + + console.log(`Running client scenario '${scenarioName}' against server: ${serverUrl}`); + + const checks = await scenario.run(serverUrl); + + await fs.writeFile(path.join(resultDir, 'checks.json'), JSON.stringify(checks, null, 2)); + + console.log(`Results saved to ${resultDir}`); + + return { + checks, + resultDir + }; +} + +async function main(): Promise { + const args = process.argv.slice(2); + let serverUrl: string | null = null; + let scenario: string | null = null; + + for (let i = 0; i < args.length; i++) { + if (args[i] === '--server-url' && i + 1 < args.length) { + serverUrl = args[i + 1]; + i++; + } else if (args[i] === '--scenario' && i + 1 < args.length) { + scenario = args[i + 1]; + i++; + } + } + + if (!serverUrl || !scenario) { + console.error('Usage: server-runner --server-url --scenario '); + console.error('Example: server-runner --server-url http://localhost:3000 --scenario initialize'); + process.exit(1); + } + + try { + const result = await runServerConformanceTest(serverUrl, scenario); + + const denominator = result.checks.filter(c => c.status === 'SUCCESS' || c.status == 'FAILURE').length; + const passed = result.checks.filter(c => c.status === 'SUCCESS').length; + const failed = result.checks.filter(c => c.status === 'FAILURE').length; + + console.log(`Checks:\n${JSON.stringify(result.checks, null, 2)}`); + + console.log(`\nTest Results:`); + console.log(`Passed: ${passed}/${denominator}, ${failed} failed`); + + if (failed > 0) { + console.log('\nFailed Checks:'); + result.checks + .filter(c => c.status === 'FAILURE') + .forEach(c => { + console.log(` - ${c.name}: ${c.description}`); + if (c.errorMessage) { + console.log(` Error: ${c.errorMessage}`); + } + }); + } + + process.exit(failed > 0 ? 1 : 0); + } catch (error) { + console.error('Server test runner error:', error); + process.exit(1); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} \ No newline at end of file From e2ceb0466137a773c1e499e4ee0fd470e328d056 Mon Sep 17 00:00:00 2001 From: Paul Carleton Date: Fri, 31 Oct 2025 19:22:53 +0000 Subject: [PATCH 3/3] rename --- src/checks.ts | 82 --------------------------------- src/{ => checks}/checks.test.ts | 2 +- 2 files changed, 1 insertion(+), 83 deletions(-) delete mode 100644 src/checks.ts rename src/{ => checks}/checks.test.ts (97%) diff --git a/src/checks.ts b/src/checks.ts deleted file mode 100644 index 17068988..00000000 --- a/src/checks.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { ConformanceCheck, CheckStatus } from './types.js'; - -export function createClientInitializationCheck(initializeRequest: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { - const protocolVersionSent = initializeRequest?.protocolVersion; - const versionMatch = protocolVersionSent === expectedSpecVersion; - - const errors: string[] = []; - if (!protocolVersionSent) errors.push('Protocol version not provided'); - if (!versionMatch) errors.push(`Version mismatch: expected ${expectedSpecVersion}, got ${protocolVersionSent}`); - if (!initializeRequest?.clientInfo?.name) errors.push('Client name missing'); - if (!initializeRequest?.clientInfo?.version) errors.push('Client version missing'); - - const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; - - return { - id: 'mcp-client-initialization', - name: 'MCPClientInitialization', - description: 'Validates that MCP client properly initializes with server', - status, - timestamp: new Date().toISOString(), - specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], - details: { - protocolVersionSent, - expectedSpecVersion, - versionMatch, - clientName: initializeRequest?.clientInfo?.name, - clientVersion: initializeRequest?.clientInfo?.version - }, - errorMessage: errors.length > 0 ? errors.join('; ') : undefined, - logs: errors.length > 0 ? errors : undefined - }; -} - -export function createServerInfoCheck(serverInfo: { name: string; version: string }): ConformanceCheck { - return { - id: 'server-info', - name: 'ServerInfo', - description: 'Test server info returned to client', - status: 'INFO', - timestamp: new Date().toISOString(), - specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], - details: { - serverName: serverInfo.name, - serverVersion: serverInfo.version - } - }; -} - -export function createServerInitializationCheck(initializeResponse: any, expectedSpecVersion: string = '2025-06-18'): ConformanceCheck { - const result = initializeResponse?.result; - const protocolVersion = result?.protocolVersion; - const serverInfo = result?.serverInfo; - const capabilities = result?.capabilities; - - const errors: string[] = []; - if (!initializeResponse?.jsonrpc) errors.push('Missing jsonrpc field'); - if (!initializeResponse?.id) errors.push('Missing id field'); - if (!result) errors.push('Missing result field'); - if (!protocolVersion) errors.push('Missing protocolVersion in result'); - if (protocolVersion !== expectedSpecVersion) errors.push(`Protocol version mismatch: expected ${expectedSpecVersion}, got ${protocolVersion}`); - if (!serverInfo) errors.push('Missing serverInfo in result'); - if (!serverInfo?.name) errors.push('Missing server name in serverInfo'); - if (!serverInfo?.version) errors.push('Missing server version in serverInfo'); - if (capabilities === undefined) errors.push('Missing capabilities in result'); - - const status: CheckStatus = errors.length === 0 ? 'SUCCESS' : 'FAILURE'; - - return { - id: 'mcp-server-initialization', - name: 'MCPServerInitialization', - description: 'Validates that MCP server properly responds to initialize request', - status, - timestamp: new Date().toISOString(), - specReferences: [{ id: 'MCP-Lifecycle', url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle' }], - details: { - expectedSpecVersion, - ...initializeResponse - }, - errorMessage: errors.length > 0 ? errors.join('; ') : undefined, - logs: errors.length > 0 ? errors : undefined - }; -} diff --git a/src/checks.test.ts b/src/checks/checks.test.ts similarity index 97% rename from src/checks.test.ts rename to src/checks/checks.test.ts index b7c19eb8..d8e2f887 100644 --- a/src/checks.test.ts +++ b/src/checks/checks.test.ts @@ -1,4 +1,4 @@ -import { createClientInitializationCheck } from './checks'; +import { createClientInitializationCheck } from './client'; describe('createClientInitializationCheck', () => { it('should return SUCCESS for a valid initialize request', () => {