diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index ad0a34afc3b1..22da8a1c0092 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -903,8 +903,26 @@ export default abstract class Server< 'http.target': req.url, }, }, - async (span) => - this.handleRequestImpl(req, res, parsedUrl).finally(() => { + async (span) => { + if (getRequestMeta(req, 'middlewareInvoke')) { + // When middleware lets the request continue, the router server + // calls `handleRequest` twice: once to run middleware (this pass) + // and once to render. Publish this span's W3C trace context on + // the request, so the middleware span (including the Edge + // sandbox, which extracts it from the request headers) is a + // child of this span instead of starting a new trace. The + // middleware forwards its own span's context to the render pass + // via request-header overrides, which replace this value; it + // only stays when there is no middleware span to forward (e.g. + // OpenTelemetry isn't registered in the Edge runtime), keeping + // the render in this trace. + const traceContext = tracer.getTraceContextHeaders() + if (traceContext) { + Object.assign(req.headers, traceContext) + } + } + + return this.handleRequestImpl(req, res, parsedUrl).finally(() => { if (!span) return const isRSCRequest = getRequestMeta(req, 'isRSCRequest') ?? false @@ -963,6 +981,7 @@ export default abstract class Server< span.updateName(isRSCRequest ? `RSC ${method}` : `${method}`) } }) + } ) }) diff --git a/packages/next/src/server/lib/trace/tracer.ts b/packages/next/src/server/lib/trace/tracer.ts index 65a57c035eb3..31748027f38e 100644 --- a/packages/next/src/server/lib/trace/tracer.ts +++ b/packages/next/src/server/lib/trace/tracer.ts @@ -178,6 +178,14 @@ interface NextTracer { */ getTracePropagationData(): ClientTraceDataEntry[] + /** + * Returns the W3C trace context (`traceparent` / `tracestate`) of the + * currently active context, as produced by the configured propagator. + * Returns `undefined` when OpenTelemetry is not enabled, no span context is + * active, or the propagator doesn't produce a `traceparent`. + */ + getTraceContextHeaders(): TraceContextHeaders | undefined + /** * Executes a function with the given span set as the active span in the context. * This allows child spans created within the function to automatically parent to this span. @@ -211,6 +219,11 @@ export interface ClientTraceDataEntry { value: string } +export interface TraceContextHeaders { + traceparent: string + tracestate?: string +} + const clientTraceDataSetter: TextMapSetter = { set(carrier, key, value) { carrier.push({ @@ -260,6 +273,25 @@ class NextTracerImpl implements NextTracer { return entries } + public getTraceContextHeaders(): TraceContextHeaders | undefined { + const activeContext = context.active() + if ( + !trace.getSpanContext(activeContext) || + !this.isOpenTelemetryEnabled() + ) { + return undefined + } + + const carrier: Record = {} + propagation.inject(activeContext, carrier) + + const { traceparent, tracestate } = carrier + if (!traceparent) { + return undefined + } + return tracestate ? { traceparent, tracestate } : { traceparent } + } + public getActiveScopeSpan(): Span | undefined { const localSpanRecorder = getLocalSpanRecorder() const activeLocalSpan = localSpanRecorder?.getActiveLocalSpan() @@ -277,19 +309,6 @@ class NextTracerImpl implements NextTracer { return activeLocalSpan } - /** - * Run `fn` with the active span cleared, so spans created inside become new - * roots (or parent to incoming propagated context). Used for in-process - * Node.js middleware so its span is a sibling of the request span, matching - * edge middleware which runs in a detached sandbox. - */ - public runWithDetachedContext(fn: () => T): T { - if (!NEXT_OTEL_PERFORMANCE_PREFIX && !this.isOpenTelemetryEnabled()) { - return fn() - } - return context.with(ROOT_CONTEXT, fn) - } - public withPropagatedContext( carrier: C, fn: () => T, diff --git a/packages/next/src/server/next-server.ts b/packages/next/src/server/next-server.ts index e5faaa3673e6..628a4c51c204 100644 --- a/packages/next/src/server/next-server.ts +++ b/packages/next/src/server/next-server.ts @@ -1866,24 +1866,21 @@ export default class NextNodeServer extends BaseServer< try { // Node.js middleware runs in-process, inside the active - // `handleRequest` span. Detach that span so the middleware span - // becomes a sibling root (or parents to an incoming traceparent), - // matching edge middleware which runs in a detached sandbox. - result = await getTracer().runWithDetachedContext(() => - adapterFn({ - handler: - middlewareModule.proxy || - middlewareModule.middleware || - middlewareModule, - request: { - ...requestData, - body: hasRequestBody - ? requestData.body.cloneBodyStream() - : undefined, - }, - page: 'middleware', - }) - ) + // `handleRequest` span, so the middleware span (and the trace context + // it forwards to the render) stays in the request's trace. + result = await adapterFn({ + handler: + middlewareModule.proxy || + middlewareModule.middleware || + middlewareModule, + request: { + ...requestData, + body: hasRequestBody + ? requestData.body.cloneBodyStream() + : undefined, + }, + page: 'middleware', + }) } finally { if (hasRequestBody) { await requestData.body.finalize() diff --git a/packages/next/src/server/web/adapter.ts b/packages/next/src/server/web/adapter.ts index 8113b2f2ebc7..a86c5974b326 100644 --- a/packages/next/src/server/web/adapter.ts +++ b/packages/next/src/server/web/adapter.ts @@ -30,10 +30,11 @@ import { workAsyncStorage } from '../app-render/work-async-storage.external' import { InvariantError } from '../../shared/lib/invariant-error' import type { ResolvedCacheLifeProfiles } from '../config-shared' import { NEXT_ROUTER_PREFETCH_HEADER } from '../../client/components/app-router-headers' -import { getTracer } from '../lib/trace/tracer' +import { getTracer, type TraceContextHeaders } from '../lib/trace/tracer' import type { TextMapGetter } from 'next/dist/compiled/@opentelemetry/api' import { MiddlewareSpan } from '../lib/trace/constants' import { CloseController } from './web-on-close' +import { forwardTraceContextToRequest } from './forward-trace-context' import { getEdgePreviewProps } from './get-edge-preview-props' import { getBuiltinRequestContext } from '../after/builtin-request-context' import { getImplicitTags } from '../lib/implicit-tags' @@ -266,6 +267,11 @@ export async function adapter( }) let response let cookiesFromResponse + // The middleware span's trace context to forward to the downstream request + // (see `forwardTraceContextToRequest`), and the request headers as the + // middleware received them. + let forwardedTraceContext: TraceContextHeaders | undefined + let forwardedRequestHeaders: Headers | undefined response = await propagator(request, () => { // we only care to make async storage available for middleware @@ -293,6 +299,16 @@ export async function adapter( }, }, async () => { + // Forward the middleware span's own context, so the render is always + // nested inside the middleware span, whatever the middleware span is + // parented to (an incoming `traceparent`, a host span, next-server's + // `handleRequest` span, or nothing). + forwardedTraceContext = getTracer().getTraceContextHeaders() + if (forwardedTraceContext) { + // Snapshot before the handler can mutate `request.headers`. + forwardedRequestHeaders = new Headers(request.headers) + } + try { const onUpdateCookies = (cookies: Array) => { cookiesFromResponse = cookies @@ -520,6 +536,18 @@ export async function adapter( const finalResponse = response ? response : NextResponse.next() + if (forwardedRequestHeaders) { + // Nest the downstream request (render) inside the middleware span. Every + // host applies these request-header overrides to the render, including + // next-server's router server, so this works whether or not the host + // invokes the middleware separately from the render. + forwardTraceContextToRequest( + finalResponse.headers, + forwardedRequestHeaders, + forwardedTraceContext + ) + } + // Flight headers are not overridable / removable so they are applied at the end. const middlewareOverrideHeaders = finalResponse.headers.get( 'x-middleware-override-headers' diff --git a/packages/next/src/server/web/forward-trace-context.test.ts b/packages/next/src/server/web/forward-trace-context.test.ts new file mode 100644 index 000000000000..c5e970dde5fa --- /dev/null +++ b/packages/next/src/server/web/forward-trace-context.test.ts @@ -0,0 +1,163 @@ +import { forwardTraceContextToRequest } from './forward-trace-context' +import { NextResponse } from './spec-extension/response' + +const TRACEPARENT = '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' +const TRACESTATE = 'vendor=value' + +function requestOverrides(headers: Headers) { + const list = headers.get('x-middleware-override-headers') + if (list === null) return undefined + return Object.fromEntries( + list + .split(',') + .filter(Boolean) + .map((key) => [key, headers.get(`x-middleware-request-${key}`)]) + ) +} + +describe('forwardTraceContextToRequest', () => { + const requestHeaders = () => + new Headers({ accept: 'text/html', cookie: 'a=b', 'x-custom': 'custom' }) + + it('lists every request header plus the trace context without user overrides', () => { + const response = NextResponse.next() + forwardTraceContextToRequest(response.headers, requestHeaders(), { + traceparent: TRACEPARENT, + tracestate: TRACESTATE, + }) + + expect(requestOverrides(response.headers)).toEqual({ + accept: 'text/html', + cookie: 'a=b', + 'x-custom': 'custom', + traceparent: TRACEPARENT, + tracestate: TRACESTATE, + }) + }) + + it('replaces a traceparent the middleware received when there are no user overrides', () => { + const headers = requestHeaders() + headers.set( + 'traceparent', + '00-11111111111111111111111111111111-2222222222222222-01' + ) + const response = NextResponse.next() + forwardTraceContextToRequest(response.headers, headers, { + traceparent: TRACEPARENT, + }) + + const overrides = requestOverrides(response.headers)! + expect(overrides.traceparent).toBe(TRACEPARENT) + expect( + response.headers + .get('x-middleware-override-headers')! + .split(',') + .filter((key) => key === 'traceparent') + ).toHaveLength(1) + }) + + it('works for rewrites', () => { + const response = NextResponse.rewrite(new URL('https://example.com/other')) + forwardTraceContextToRequest(response.headers, requestHeaders(), { + traceparent: TRACEPARENT, + }) + + expect(requestOverrides(response.headers)?.traceparent).toBe(TRACEPARENT) + }) + + it('only appends the trace context to existing user overrides', () => { + const userHeaders = new Headers({ 'x-user': 'user' }) + const response = NextResponse.next({ request: { headers: userHeaders } }) + forwardTraceContextToRequest(response.headers, requestHeaders(), { + traceparent: TRACEPARENT, + tracestate: TRACESTATE, + }) + + expect(requestOverrides(response.headers)).toEqual({ + 'x-user': 'user', + traceparent: TRACEPARENT, + tracestate: TRACESTATE, + }) + }) + + it('keeps a traceparent explicitly set by the user', () => { + const userTraceparent = + '00-33333333333333333333333333333333-4444444444444444-01' + const response = NextResponse.next({ + request: { headers: new Headers({ traceparent: userTraceparent }) }, + }) + forwardTraceContextToRequest(response.headers, requestHeaders(), { + traceparent: TRACEPARENT, + tracestate: TRACESTATE, + }) + + expect(requestOverrides(response.headers)).toEqual({ + traceparent: userTraceparent, + tracestate: TRACESTATE, + }) + }) + + it('replaces trace context headers copied unchanged into user overrides', () => { + const receivedTraceparent = + '00-11111111111111111111111111111111-2222222222222222-01' + const received = requestHeaders() + received.set('traceparent', receivedTraceparent) + received.set('tracestate', 'received=1') + // The common `new Headers(request.headers)` pattern copies the received + // trace context into the overrides without choosing it. + const userHeaders = new Headers(received) + userHeaders.set('x-user', 'user') + const response = NextResponse.next({ request: { headers: userHeaders } }) + forwardTraceContextToRequest(response.headers, received, { + traceparent: TRACEPARENT, + tracestate: TRACESTATE, + }) + + const overrides = requestOverrides(response.headers)! + expect(overrides.traceparent).toBe(TRACEPARENT) + expect(overrides.tracestate).toBe(TRACESTATE) + expect(overrides['x-user']).toBe('user') + expect( + response.headers + .get('x-middleware-override-headers')! + .split(',') + .filter((key) => key === 'traceparent' || key === 'tracestate') + ).toEqual(['traceparent', 'tracestate']) + }) + + it('treats an empty user override list like no overrides', () => { + const response = NextResponse.next({ request: { headers: new Headers() } }) + expect(response.headers.get('x-middleware-override-headers')).toBe('') + forwardTraceContextToRequest(response.headers, requestHeaders(), { + traceparent: TRACEPARENT, + }) + + expect(requestOverrides(response.headers)).toEqual({ + accept: 'text/html', + cookie: 'a=b', + 'x-custom': 'custom', + traceparent: TRACEPARENT, + }) + }) + + it('does nothing without a trace context', () => { + const response = NextResponse.next() + forwardTraceContextToRequest(response.headers, requestHeaders(), undefined) + + expect(requestOverrides(response.headers)).toBeUndefined() + expect([...response.headers.keys()]).toEqual(['x-middleware-next']) + }) + + it.each([ + ['a redirect', () => NextResponse.redirect('https://example.com/')], + ['a direct response', () => new NextResponse('hello')], + ])('does nothing for %s', (_, createResponse) => { + const response = createResponse() + const before = [...response.headers.entries()] + forwardTraceContextToRequest(response.headers, requestHeaders(), { + traceparent: TRACEPARENT, + }) + + expect([...response.headers.entries()]).toEqual(before) + }) +}) diff --git a/packages/next/src/server/web/forward-trace-context.ts b/packages/next/src/server/web/forward-trace-context.ts new file mode 100644 index 000000000000..ab40071b588e --- /dev/null +++ b/packages/next/src/server/web/forward-trace-context.ts @@ -0,0 +1,90 @@ +import type { TraceContextHeaders } from '../lib/trace/tracer' + +const OVERRIDE_HEADERS = 'x-middleware-override-headers' +const REQUEST_HEADER_PREFIX = 'x-middleware-request-' + +/** + * Forwards the middleware's W3C trace context to the downstream request by + * adding `traceparent` / `tracestate` request-header overrides to a middleware + * response that continues the request (`NextResponse.next()` or a rewrite). + * + * Every host applies these overrides to the downstream request: hosts that + * invoke middleware separately from the render (deployments, adapters, direct + * entrypoints) as well as next-server's router server. Forwarding the + * middleware span's context therefore nests the render inside the middleware + * span instead of starting a new trace. + * + * Request-header overrides replace the whole downstream request header set, + * so: + * - Without user overrides (or with an empty list, which hosts ignore), every + * request header the middleware received is listed (unchanged) alongside + * the trace context. + * - With user overrides, they are kept and only the trace context headers are + * added. A trace context header the user explicitly set (to a value other + * than the one the middleware received) wins. One copied unchanged from the + * request (e.g. via `new Headers(request.headers)`) is replaced, as it is no + * explicit choice. + */ +export function forwardTraceContextToRequest( + responseHeaders: Headers, + requestHeaders: Headers, + traceContext: TraceContextHeaders | undefined +): void { + if (!traceContext) { + return + } + + // Redirects and direct responses have no downstream request. + if ( + !responseHeaders.has('x-middleware-next') && + !responseHeaders.has('x-middleware-rewrite') + ) { + return + } + + const traceHeaders: Array<[string, string]> = [ + ['traceparent', traceContext.traceparent], + ] + if (traceContext.tracestate) { + traceHeaders.push(['tracestate', traceContext.tracestate]) + } + + const userOverrides = responseHeaders.get(OVERRIDE_HEADERS) + // An empty override list is ignored by hosts (no headers are replaced), so + // treat it like no overrides rather than turning it into "only the trace + // context headers". + if (!userOverrides) { + const keys: string[] = [] + for (const [key, value] of requestHeaders) { + responseHeaders.set(REQUEST_HEADER_PREFIX + key, value) + keys.push(key) + } + for (const [key, value] of traceHeaders) { + responseHeaders.set(REQUEST_HEADER_PREFIX + key, value) + if (!keys.includes(key)) { + keys.push(key) + } + } + responseHeaders.set(OVERRIDE_HEADERS, keys.join(',')) + return + } + + const keys = userOverrides + .split(',') + .map((key) => key.trim()) + .filter(Boolean) + for (const [key, value] of traceHeaders) { + if (keys.includes(key)) { + if ( + responseHeaders.get(REQUEST_HEADER_PREFIX + key) !== + requestHeaders.get(key) + ) { + continue + } + } else { + keys.push(key) + } + responseHeaders.set(REQUEST_HEADER_PREFIX + key, value) + } + responseHeaders.set(OVERRIDE_HEADERS, keys.join(',')) +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/app/headers-echo/page.tsx b/test/e2e/app-dir/otel-middleware-single-trace/app/headers-echo/page.tsx new file mode 100644 index 000000000000..b067a3fa4749 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/app/headers-echo/page.tsx @@ -0,0 +1,22 @@ +import { trace } from '@opentelemetry/api' +import { headers } from 'next/headers' +import { Suspense } from 'react' + +async function RequestInfo() { + const requestHeaders = Object.fromEntries((await headers()).entries()) + const traceId = trace.getActiveSpan()?.spanContext().traceId ?? 'none' + return ( + <> +

{traceId}

+
{JSON.stringify(requestHeaders)}
+ + ) +} + +export default function Page() { + return ( + + + + ) +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/app/layout.tsx b/test/e2e/app-dir/otel-middleware-single-trace/app/layout.tsx new file mode 100644 index 000000000000..7c3f422f0039 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/app/layout.tsx @@ -0,0 +1,9 @@ +import type { ReactNode } from 'react' + +export default function Root({ children }: { children: ReactNode }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/app/page.tsx b/test/e2e/app-dir/otel-middleware-single-trace/app/page.tsx new file mode 100644 index 000000000000..c639109026f1 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/app/page.tsx @@ -0,0 +1,3 @@ +export default function Home() { + return

otel-middleware-single-trace

+} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/app/page/page.tsx b/test/e2e/app-dir/otel-middleware-single-trace/app/page/page.tsx new file mode 100644 index 000000000000..dde438077448 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/app/page/page.tsx @@ -0,0 +1,25 @@ +import { trace } from '@opentelemetry/api' +import { headers } from 'next/headers' +import { Suspense } from 'react' + +async function TraceInfo() { + // Reading request headers also makes this run at request time, so the + // reported trace belongs to the request. + const traceparent = (await headers()).get('traceparent') ?? 'none' + const traceId = trace.getActiveSpan()?.spanContext().traceId ?? 'none' + return ( + <> +

{traceId}

+ {/* The trace context the render was started from (its parent span). */} +

{traceparent}

+ + ) +} + +export default function Page() { + return ( + + + + ) +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/app/route-handler/route.ts b/test/e2e/app-dir/otel-middleware-single-trace/app/route-handler/route.ts new file mode 100644 index 000000000000..b57b52834bf6 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/app/route-handler/route.ts @@ -0,0 +1,11 @@ +import { trace } from '@opentelemetry/api' +import { connection } from 'next/server' + +export async function GET(request: Request) { + // Run at request time so the reported trace belongs to the request. + await connection() + const traceId = trace.getActiveSpan()?.spanContext().traceId ?? 'none' + // The trace context the render was started from (its parent span). + const traceparent = request.headers.get('traceparent') ?? 'none' + return Response.json({ traceId, traceparent }) +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/app/unmatched/page.tsx b/test/e2e/app-dir/otel-middleware-single-trace/app/unmatched/page.tsx new file mode 100644 index 000000000000..72b9bca42d00 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/app/unmatched/page.tsx @@ -0,0 +1,19 @@ +import { trace } from '@opentelemetry/api' +import { connection } from 'next/server' +import { Suspense } from 'react' + +async function TraceId() { + // Run at request time so the reported trace belongs to the request. + await connection() + const traceId = trace.getActiveSpan()?.spanContext().traceId ?? 'none' + return

{traceId}

+} + +// Not matched by the middleware: a baseline for requests without middleware. +export default function Page() { + return ( + + + + ) +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/instrumentation.ts b/test/e2e/app-dir/otel-middleware-single-trace/instrumentation.ts new file mode 100644 index 000000000000..69676ee671e6 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/instrumentation.ts @@ -0,0 +1,7 @@ +import { registerOTel } from '@vercel/otel' + +export function register() { + // No exporter is needed: the app itself reports the active trace IDs, so the + // suite also works when deployed (where no local collector is reachable). + registerOTel({ serviceName: 'otel-middleware-single-trace' }) +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/middleware-impl.ts b/test/e2e/app-dir/otel-middleware-single-trace/middleware-impl.ts new file mode 100644 index 000000000000..13cab10f82c7 --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/middleware-impl.ts @@ -0,0 +1,41 @@ +import { trace } from '@opentelemetry/api' +import { NextResponse, type NextRequest } from 'next/server' + +export const USER_TRACE_ID = '0af7651916cd43dd8448eb211c80319c' +export const USER_SPAN_ID = 'b7ad6b7169203331' + +// Shared by `proxy.ts` (Node.js runtime) and `middleware.ts` (Edge runtime). +export function handler(request: NextRequest) { + const spanContext = trace.getActiveSpan()?.spanContext() + const traceId = spanContext?.traceId ?? 'none' + const spanId = spanContext?.spanId ?? 'none' + const { pathname } = request.nextUrl + + let response: NextResponse + if (pathname === '/rewrite') { + response = NextResponse.rewrite(new URL('/page', request.url)) + } else if (pathname === '/redirect') { + response = NextResponse.redirect(new URL('/page', request.url)) + } else if (pathname === '/direct') { + response = new NextResponse('direct response from middleware') + } else if (pathname === '/user-overrides') { + const headers = new Headers(request.headers) + headers.set('x-user-added', 'added-by-middleware') + headers.delete('x-delete-me') + response = NextResponse.rewrite(new URL('/headers-echo', request.url), { + request: { headers }, + }) + } else if (pathname === '/user-traceparent') { + const headers = new Headers(request.headers) + headers.set('traceparent', `00-${USER_TRACE_ID}-${USER_SPAN_ID}-01`) + response = NextResponse.rewrite(new URL('/headers-echo', request.url), { + request: { headers }, + }) + } else { + response = NextResponse.next() + } + + response.headers.set('x-mw-trace-id', traceId) + response.headers.set('x-mw-span-id', spanId) + return response +} diff --git a/test/e2e/app-dir/otel-middleware-single-trace/next.config.ts b/test/e2e/app-dir/otel-middleware-single-trace/next.config.ts new file mode 100644 index 000000000000..e4f5738a310b --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/next.config.ts @@ -0,0 +1,5 @@ +import type { NextConfig } from 'next' + +const nextConfig: NextConfig = {} + +export default nextConfig diff --git a/test/e2e/app-dir/otel-middleware-single-trace/otel-middleware-single-trace.test.ts b/test/e2e/app-dir/otel-middleware-single-trace/otel-middleware-single-trace.test.ts new file mode 100644 index 000000000000..c8d32d4cf3dc --- /dev/null +++ b/test/e2e/app-dir/otel-middleware-single-trace/otel-middleware-single-trace.test.ts @@ -0,0 +1,198 @@ +import { isNextDeploy, nextTestSetup } from 'e2e-utils' +import cheerio from 'cheerio' + +const USER_TRACE_ID = '0af7651916cd43dd8448eb211c80319c' +const INCOMING_TRACE_ID = '4bf92f3577b34da6a3ce929d0e0e4736' +const INCOMING_TRACEPARENT = `00-${INCOMING_TRACE_ID}-00f067aa0ba902b7-01` +const TRACE_ID = /^(?!0{32}$)[0-9a-f]{32}$/ +const SPAN_ID = /^(?!0{16}$)[0-9a-f]{16}$/ + +const MATCHER = [ + '/page', + '/route-handler', + '/rewrite', + '/redirect', + '/direct', + '/user-overrides', + '/user-traceparent', +] + +// The middleware entry file is generated per variant: `config` must be a +// static literal in the entry file itself, and the shared implementation is +// imported relative to the app root, where the entry file is written. +function middlewareEntry(exportName: 'proxy' | 'middleware') { + return [ + `import type { NextRequest } from 'next/server'`, + `import { handler } from './middleware-impl'`, + ``, + `export function ${exportName}(request: NextRequest) {`, + ` return handler(request)`, + `}`, + ``, + `export const config = { matcher: ${JSON.stringify(MATCHER)} }`, + ``, + ].join('\n') +} + +// The app reports its active trace IDs itself (no local collector), so these +// assertions also hold when deployed. Every request that passes through +// middleware must stay in a single trace: the trace ID seen by the middleware +// has to be the one seen by the page / route handler that renders afterwards. +// The render must also be nested inside the middleware span: the trace context +// it was started from (the `traceparent` it received) is the middleware span. +describe.each([ + // `proxy.ts` always runs in the Node.js runtime. + { name: 'proxy (nodejs runtime)', file: 'proxy.ts', exportName: 'proxy' }, + // `middleware.ts` defaults to the edge runtime. + { + name: 'middleware (edge runtime)', + file: 'middleware.ts', + exportName: 'middleware', + }, +] as const)('otel-middleware-single-trace - $name', ({ file, exportName }) => { + const { next } = nextTestSetup({ + files: __dirname, + overrideFiles: { + [file]: middlewareEntry(exportName), + }, + dependencies: { + '@vercel/otel': '2.1.3', + '@opentelemetry/api': '1.9.1', + '@opentelemetry/api-logs': '0.223.0', + '@opentelemetry/instrumentation': '0.223.0', + '@opentelemetry/resources': '2.12.0', + '@opentelemetry/sdk-logs': '0.223.0', + '@opentelemetry/sdk-metrics': '2.12.0', + '@opentelemetry/sdk-trace-base': '2.12.0', + }, + }) + + function getMiddlewareTraceId(res: Response) { + const traceId = res.headers.get('x-mw-trace-id') + expect(traceId).toMatch(TRACE_ID) + return traceId + } + + // Asserts the render is in the middleware's trace and a direct child of the + // middleware span. + function expectNestedInMiddleware( + res: Response, + render: { traceId: string; traceparent: string } + ) { + const traceId = getMiddlewareTraceId(res) + const spanId = res.headers.get('x-mw-span-id') + expect(spanId).toMatch(SPAN_ID) + expect(render.traceId).toBe(traceId) + const [, parentTraceId, parentSpanId] = render.traceparent.split('-') + expect({ parentTraceId, parentSpanId }).toEqual({ + parentTraceId: traceId, + parentSpanId: spanId, + }) + } + + async function getPageTrace(res: Response) { + const $ = cheerio.load(await res.text()) + return { + traceId: $('#trace-id').text(), + traceparent: $('#traceparent').text(), + } + } + + function expectNoInternalHeaders(res: Response) { + for (const key of res.headers.keys()) { + expect(key).not.toMatch(/^x-middleware-(request-|override-headers)/) + } + } + + it('nests a page rendered after NextResponse.next() in the middleware span', async () => { + const res = await next.fetch('/page') + expect(res.status).toBe(200) + expectNestedInMiddleware(res, await getPageTrace(res)) + expectNoInternalHeaders(res) + }) + + it('nests a route handler executed after NextResponse.next() in the middleware span', async () => { + const res = await next.fetch('/route-handler') + expect(res.status).toBe(200) + expectNestedInMiddleware(res, await res.json()) + expectNoInternalHeaders(res) + }) + + it('nests a rewritten page in the middleware span', async () => { + const res = await next.fetch('/rewrite') + expect(res.status).toBe(200) + expectNestedInMiddleware(res, await getPageTrace(res)) + expectNoInternalHeaders(res) + }) + + it('traces a middleware redirect', async () => { + const res = await next.fetch('/redirect', { redirect: 'manual' }) + expect(res.status).toBe(307) + getMiddlewareTraceId(res) + expectNoInternalHeaders(res) + }) + + it('traces a direct middleware response', async () => { + const res = await next.fetch('/direct') + expect(res.status).toBe(200) + expect(await res.text()).toBe('direct response from middleware') + getMiddlewareTraceId(res) + expectNoInternalHeaders(res) + }) + + it('traces a request the middleware does not match', async () => { + const res = await next.fetch('/unmatched') + expect(res.status).toBe(200) + expect(res.headers.get('x-mw-trace-id')).toBeNull() + const $ = cheerio.load(await res.text()) + expect($('#trace-id').text()).toMatch(TRACE_ID) + expectNoInternalHeaders(res) + }) + + it.each(['/page', '/route-handler', '/rewrite'])( + 'nests %s in the middleware span with an incoming traceparent', + async (pathname) => { + const res = await next.fetch(pathname, { + headers: { traceparent: INCOMING_TRACEPARENT }, + }) + expect(res.status).toBe(200) + const render = + pathname === '/route-handler' + ? await res.json() + : await getPageTrace(res) + expectNestedInMiddleware(res, render) + // Self-hosted Next.js continues the caller's trace. When deployed, the + // platform decides whether a client-supplied traceparent is trusted, so + // only the single-trace and nesting guarantees are asserted there. + if (!isNextDeploy) { + expect(render.traceId).toBe(INCOMING_TRACE_ID) + } + expectNoInternalHeaders(res) + } + ) + + it('still applies request header overrides set by middleware', async () => { + const res = await next.fetch('/user-overrides', { + headers: { 'x-delete-me': 'please', 'x-keep-me': 'kept' }, + }) + expect(res.status).toBe(200) + const $ = cheerio.load(await res.text()) + const headers = JSON.parse($('#headers').text()) + expect(headers['x-user-added']).toBe('added-by-middleware') + expect(headers['x-keep-me']).toBe('kept') + expect(headers['x-delete-me']).toBeUndefined() + expectNestedInMiddleware(res, { + traceId: $('#trace-id').text(), + traceparent: headers['traceparent'], + }) + expectNoInternalHeaders(res) + }) + + it('respects a traceparent explicitly set by middleware', async () => { + const res = await next.fetch('/user-traceparent') + expect(res.status).toBe(200) + const $ = cheerio.load(await res.text()) + expect($('#trace-id').text()).toBe(USER_TRACE_ID) + expectNoInternalHeaders(res) + }) +}) diff --git a/test/e2e/opentelemetry/instrumentation/app/behind-middleware/route-handler/route.ts b/test/e2e/opentelemetry/instrumentation/app/behind-middleware/route-handler/route.ts new file mode 100644 index 000000000000..c428c0254b1f --- /dev/null +++ b/test/e2e/opentelemetry/instrumentation/app/behind-middleware/route-handler/route.ts @@ -0,0 +1,5 @@ +export async function GET() { + return new Response(JSON.stringify({ behind: 'middleware' })) +} + +export const dynamic = 'force-dynamic' diff --git a/test/e2e/opentelemetry/instrumentation/collector.ts b/test/e2e/opentelemetry/instrumentation/collector.ts index 8caa8adc4e22..c21c390a24fd 100644 --- a/test/e2e/opentelemetry/instrumentation/collector.ts +++ b/test/e2e/opentelemetry/instrumentation/collector.ts @@ -3,6 +3,12 @@ import { SavedSpan } from './constants' export interface Collector { getSpans: () => SavedSpan[] + /** + * Like `getSpans`, but also includes spans marked `next.bubble` (e.g. + * next-server's middleware-invoke `handleRequest` span), which are needed to + * check the full span hierarchy of a request. + */ + getAllSpans: () => SavedSpan[] reset: () => void shutdown: () => Promise } @@ -13,6 +19,7 @@ export async function connectCollector({ port: number }): Promise { const spans: SavedSpan[] = [] + const allSpans: SavedSpan[] = [] const server = new HttpServer(async (req, res) => { if (req.method !== 'POST') { @@ -33,6 +40,7 @@ export async function connectCollector({ }) const newSpans = JSON.parse(body.toString('utf-8')) as SavedSpan[] + allSpans.push(...newSpans) const filteredSpans = newSpans.filter((span) => { if (span.attributes?.['next.bubble'] === true) { return false @@ -61,8 +69,12 @@ export async function connectCollector({ getSpans() { return spans }, + getAllSpans() { + return allSpans + }, reset() { spans.length = 0 + allSpans.length = 0 }, shutdown() { return new Promise((resolve, reject) => diff --git a/test/e2e/opentelemetry/instrumentation/custom-entrypoint-server.ts b/test/e2e/opentelemetry/instrumentation/custom-entrypoint-server.ts index 33bdacfe01a1..e8fdb04db4b7 100644 --- a/test/e2e/opentelemetry/instrumentation/custom-entrypoint-server.ts +++ b/test/e2e/opentelemetry/instrumentation/custom-entrypoint-server.ts @@ -63,6 +63,10 @@ async function main() { 'app/app/[param]/rsc-fetch/error/page.js', ], [/^\/behind-middleware$/, 'app/behind-middleware/page.js'], + [ + /^\/behind-middleware\/route-handler$/, + 'app/behind-middleware/route-handler/route.js', + ], // --- [/^\/api\/pages\/param\/basic$/, 'pages/api/pages/[param]/basic.js'], [/^\/api\/pages\/param\/error$/, 'pages/api/pages/[param]/error.js'], diff --git a/test/e2e/opentelemetry/instrumentation/middleware-node.ts b/test/e2e/opentelemetry/instrumentation/middleware-node.ts index 7c90ccaa2edb..a5aa079e001f 100644 --- a/test/e2e/opentelemetry/instrumentation/middleware-node.ts +++ b/test/e2e/opentelemetry/instrumentation/middleware-node.ts @@ -10,5 +10,15 @@ export async function middleware( request: NextRequest, event?: NextFetchEvent ): Promise { + const { pathname } = request.nextUrl + if (pathname === '/behind-middleware/rewrite') { + // Keep the query string so tests can correlate the rewritten render. + const url = request.nextUrl.clone() + url.pathname = '/behind-middleware' + return NextResponse.rewrite(url) + } + if (pathname === '/behind-middleware/redirect') { + return NextResponse.redirect(new URL('/behind-middleware', request.url)) + } return NextResponse.next() } diff --git a/test/e2e/opentelemetry/instrumentation/middleware.ts b/test/e2e/opentelemetry/instrumentation/middleware.ts index 15e710b95a92..7361fab96f1f 100644 --- a/test/e2e/opentelemetry/instrumentation/middleware.ts +++ b/test/e2e/opentelemetry/instrumentation/middleware.ts @@ -9,5 +9,15 @@ export async function middleware( request: NextRequest, event?: NextFetchEvent ): Promise { + const { pathname } = request.nextUrl + if (pathname === '/behind-middleware/rewrite') { + // Keep the query string so tests can correlate the rewritten render. + const url = request.nextUrl.clone() + url.pathname = '/behind-middleware' + return NextResponse.rewrite(url) + } + if (pathname === '/behind-middleware/redirect') { + return NextResponse.redirect(new URL('/behind-middleware', request.url)) + } return NextResponse.next() } diff --git a/test/e2e/opentelemetry/instrumentation/opentelemetry.test.ts b/test/e2e/opentelemetry/instrumentation/opentelemetry.test.ts index 1c6f46c982ff..f02debd09cd6 100644 --- a/test/e2e/opentelemetry/instrumentation/opentelemetry.test.ts +++ b/test/e2e/opentelemetry/instrumentation/opentelemetry.test.ts @@ -47,7 +47,15 @@ function setupCollector(next: NextInstance, port: number) { } } -function setup({ useDirectEntrypointHandler, useNodeMiddleware }) { +function setup({ + useDirectEntrypointHandler, + useNodeMiddleware, + withParentSpan = false, +}: { + useDirectEntrypointHandler?: boolean + useNodeMiddleware?: boolean + withParentSpan?: boolean +}) { const testSetup = nextTestSetup({ files: __dirname, skipDeployment: true, @@ -66,8 +74,9 @@ function setup({ useDirectEntrypointHandler, useNodeMiddleware }) { startCommand: 'pnpm start-entrypoint', packageJson: { scripts: { - 'start-entrypoint': - 'pnpm tsx custom-entrypoint-server.ts --without-parent-span', + 'start-entrypoint': withParentSpan + ? 'pnpm tsx custom-entrypoint-server.ts' + : 'pnpm tsx custom-entrypoint-server.ts --without-parent-span', }, }, serverReadyPattern: /- Local:/, @@ -2236,95 +2245,266 @@ describe.each( name: 'direct entrypoints', useDirectEntrypointHandler: true, }, - ].filter(Boolean) -)('opentelemetry - middleware $name', ({ useDirectEntrypointHandler }) => { - describe.each(['edge', 'nodejs'])('%s runtime', (runtime) => { - const { - next: { next, skipped }, - getCollector, - } = setup({ - useDirectEntrypointHandler, - useNodeMiddleware: runtime === 'nodejs', - }) + isNextStart && { + name: 'direct entrypoints with parent span', + useDirectEntrypointHandler: true, + withParentSpan: true, + }, + ].filter(Boolean) as { + name: string + useDirectEntrypointHandler: boolean + withParentSpan?: boolean + }[] +)( + 'opentelemetry - middleware $name', + ({ useDirectEntrypointHandler, withParentSpan = false }) => { + describe.each(['edge', 'nodejs'])('%s runtime', (runtime) => { + const { + next: { next, skipped }, + getCollector, + } = setup({ + useDirectEntrypointHandler, + useNodeMiddleware: runtime === 'nodejs', + withParentSpan, + }) - if (skipped) { - return - } + if (skipped) { + return + } - if (useDirectEntrypointHandler && runtime === 'edge') { - it.skip('direct entrypoint handler is not implemented for edge runtime', () => {}) - return - } + if (useDirectEntrypointHandler && runtime === 'edge') { + it.skip('direct entrypoint handler is not implemented for edge runtime', () => {}) + return + } - for (const env of [ - { - name: 'root context', - fetchInit: undefined, - span: { - traceId: '[trace-id]', - rootParentId: undefined, - }, - }, - { - name: 'incoming context propagation', - fetchInit: { - headers: { - traceparent: `00-${EXTERNAL.traceId}-${EXTERNAL.spanId}-01`, - }, + const requests = [ + { name: 'page', pathname: '/behind-middleware', render: true }, + { + name: 'route handler', + pathname: '/behind-middleware/route-handler', + render: true, }, - span: { - traceId: EXTERNAL.traceId, - rootParentId: EXTERNAL.spanId, - }, - }, - ]) { - ;(process.env.__NEXT_CACHE_COMPONENTS ? describe.skip : describe)( - env.name, - () => { - it('should trace middleware', async () => { - await next.fetch('/behind-middleware', env.fetchInit) - let expected = [ + // The direct entrypoint harness only serves paths that map to an + // entrypoint, so it can't serve rewrites or redirects. + ...(useDirectEntrypointHandler + ? [] + : [ { - runtime: runtime, - traceId: env.span.traceId, - parentId: env.span.rootParentId, - name: 'middleware GET', - attributes: { - 'http.method': 'GET', - 'http.target': '/behind-middleware', - 'next.span_name': 'middleware GET', - 'next.span_type': 'Middleware.execute', - }, - status: { code: 0 }, - spans: [], + name: 'rewrite', + pathname: '/behind-middleware/rewrite', + render: true, }, { - runtime: 'nodejs', - traceId: env.span.traceId, - parentId: env.span.rootParentId, - name: 'GET /behind-middleware', - attributes: { - 'http.method': 'GET', - 'http.route': '/behind-middleware', - 'http.status_code': 200, - 'http.target': '/behind-middleware', - 'next.route': '/behind-middleware', - 'next.span_name': 'GET /behind-middleware', - 'next.span_type': 'BaseServer.handleRequest', - }, + name: 'redirect', + pathname: '/behind-middleware/redirect', + render: false, }, - ] - if (runtime === 'nodejs') { - // TODO unclear why this is reversed for Node.js runtime - expected.reverse() - } - await expectTrace(getCollector(), expected) - }) - } + ]), + ] + + const contexts = [ + { name: 'root context', traceparent: false }, + // The parent-span harness starts its own root span without extracting + // incoming headers, so incoming propagation is not applicable there. + ...(withParentSpan + ? [] + : [{ name: 'incoming context propagation', traceparent: true }]), + ] + + for (const ctx of contexts) { + ;(process.env.__NEXT_CACHE_COMPONENTS ? describe.skip : describe)( + ctx.name, + () => { + it.each(requests)( + 'should keep the $name request in a single trace', + async ({ pathname, render }) => { + const id = Math.random().toString(36).slice(2) + // Span timestamps are epoch microseconds. + const startedAt = Date.now() * 1000 + const res = await next.fetch(`${pathname}?id=${id}`, { + redirect: 'manual', + headers: ctx.traceparent + ? { + traceparent: `00-${EXTERNAL.traceId}-${EXTERNAL.spanId}-01`, + } + : {}, + }) + expect(res.status).toBe(render ? 200 : 307) + + await expectSingleTrace(getCollector(), { + id, + startedAt, + pathname, + // The parent-span harness exports through its own provider, + // which doesn't record the runtime. + middlewareRuntime: withParentSpan ? undefined : runtime, + render, + traceparent: ctx.traceparent, + shape: withParentSpan + ? 'host-root' + : useDirectEntrypointHandler + ? 'middleware-root' + : 'handle-request-root', + }) + } + ) + } + ) + } + }) + } +) + +// Root spans of one-time work that is not part of any request: server startup +// (instrumentation registration, route module prewarming) and the lazy, +// once-per-process route module preparation of direct entrypoints. +const STARTUP_SPAN_TYPES = new Set([ + 'Instrumentation.loadModule', + 'Instrumentation.register', + 'LoadComponents.loadRouteModule', + 'RouteModule.prepare', +]) + +/** + * Asserts that every span of the request tagged with `?id=` belongs to a + * single trace, using the real trace IDs (unlike `expectTrace`, which + * normalizes them away). The middleware span's `http.target` is the pathname + * without the query, so it is matched by `pathname` (the collector is reset + * before every test). + */ +async function expectSingleTrace( + collector: Collector, + { + id, + startedAt, + pathname, + middlewareRuntime, + render, + traceparent, + shape, + }: { + id: string + startedAt: number + pathname: string + middlewareRuntime: string | undefined + render: boolean + traceparent: boolean + // Where the middleware span sits. In every shape, the render is a direct + // child of the middleware span. + // handle-request-root: next-server's middleware-invoke `handleRequest` is + // the root and the middleware span's parent (self-hosted router server). + // middleware-root: the middleware span is the root (separately invoked + // middleware without a host span, e.g. direct entrypoints). + // host-root: a span started by the host wraps everything. + shape: 'handle-request-root' | 'middleware-root' | 'host-root' + } +) { + await retry(async () => { + // Include `next.bubble` spans: next-server's middleware-invoke + // `handleRequest` span is one of them and is the root of the trace. + const spans = collector.getAllSpans() + const byId = new Map(spans.map((span) => [span.id, span])) + const isAncestor = (ancestor: SavedSpan, span: SavedSpan) => { + let parentId = span.parentId + while (parentId) { + if (parentId === ancestor.id) return true + parentId = byId.get(parentId)?.parentId + } + return false + } + + const requestSpans = spans.filter((span) => + String(span.attributes?.['http.target'] ?? '').includes(`id=${id}`) + ) + const middlewareSpans = spans.filter( + (span) => + span.attributes?.['next.span_type'] === 'Middleware.execute' && + span.attributes?.['http.target'] === pathname + ) + expect(middlewareSpans).toHaveLength(1) + const [middlewareSpan] = middlewareSpans + expect(middlewareSpan.runtime).toBe(middlewareRuntime) + requestSpans.push(middlewareSpan) + + const handleRequestSpans = requestSpans.filter( + (span) => + span.attributes?.['next.span_type'] === 'BaseServer.handleRequest' + ) + const renderSpan = handleRequestSpans.find( + (span) => span.attributes?.['next.route'] + ) + if (render) { + expect(renderSpan).toBeDefined() + // The render is always nested inside the middleware span. + expect(renderSpan!.parentId).toBe(middlewareSpan.id) + } else { + expect(renderSpan).toBeUndefined() + } + + const traceIds = new Set(requestSpans.map((span) => span.traceId)) + expect([...traceIds]).toEqual([ + traceparent ? EXTERNAL.traceId : middlewareSpan.traceId, + ]) + // Every span started for this request (not only the correlated entry + // spans) must be in that trace, so detached child spans fail too. One-time + // work (`STARTUP_SPAN_TYPES`) runs in its own root traces and is not part + // of the request. + const rootOf = (span: SavedSpan) => { + let current = span + while (current.parentId && byId.has(current.parentId)) { + current = byId.get(current.parentId)! + } + return current + } + const spansSinceRequest = spans.filter( + (span) => + (span.timestamp ?? 0) >= startedAt && + !STARTUP_SPAN_TYPES.has( + String(rootOf(span).attributes?.['next.span_type']) + ) + ) + expect( + spansSinceRequest + .filter((span) => span.traceId !== middlewareSpan.traceId) + .map((span) => `${span.name}|${span.attributes?.['next.span_type']}`) + ).toEqual([]) + // Every span of that trace must be reachable from a single root (or from + // the external parent when the request carried a traceparent). + const traceSpans = spans.filter( + (span) => span.traceId === middlewareSpan.traceId + ) + const roots = traceSpans.filter( + (span) => + !span.parentId || + (traceparent && span.parentId === EXTERNAL.spanId) || + !byId.has(span.parentId) + ) + + if (shape === 'handle-request-root') { + expect(roots).toHaveLength(1) + const [root] = roots + expect(root.attributes?.['next.span_type']).toBe( + 'BaseServer.handleRequest' ) + // The middleware-invoke pass keeps its existing name. + expect(root.name).toBe('GET') + expect(root.attributes?.['next.bubble']).toBe(true) + expect(root.parentId).toBe(traceparent ? EXTERNAL.spanId : undefined) + expect(middlewareSpan.parentId).toBe(root.id) + } else if (shape === 'middleware-root') { + if (traceparent) { + expect(middlewareSpan.parentId).toBe(EXTERNAL.spanId) + } else { + expect(roots).toEqual([middlewareSpan]) + } + } else { + expect(roots).toHaveLength(1) + const [root] = roots + expect(root.attributes?.['next.span_type']).toBeUndefined() + expect(isAncestor(root, middlewareSpan)).toBe(true) } }) -}) +} describe.each( [