Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/cli/script/ensure-generated.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import { existsSync } from "node:fs";
const SCHEMA_OUTPUTS = [
"src/generated/api-schema.json",
"src/generated/idempotent-operations.json",
"src/generated/api-path-templates.json",
];
/** Produced by `generate:grammar`; imported by `src/lib/arg-parsing.ts`. */
const GRAMMAR_OUTPUTS = [
Expand Down
17 changes: 16 additions & 1 deletion packages/cli/script/generate-api-schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
* Generate the CLI discovery index from the installed SDK's operation catalog.
*
* Usage: tsx script/generate-api-schema.ts
* Outputs: src/generated/api-schema.json and src/generated/idempotent-operations.json
* Outputs: src/generated/api-schema.json, src/generated/idempotent-operations.json
* and src/generated/api-path-templates.json
*/

import { mkdir, writeFile } from "node:fs/promises";
Expand All @@ -15,6 +16,7 @@ import { projectApiOperations } from "./lib/api-schema.js";

const OUTPUT_PATH = "src/generated/api-schema.json";
const IDEMPOTENT_OUTPUT_PATH = "src/generated/idempotent-operations.json";
const PATH_TEMPLATES_OUTPUT_PATH = "src/generated/api-path-templates.json";
const endpoints = projectApiOperations(operations);

await mkdir(dirname(OUTPUT_PATH), { recursive: true });
Expand All @@ -41,3 +43,16 @@ await writeFile(
console.log(
`Generated ${IDEMPOTENT_OUTPUT_PATH} (${idempotentOperations.length} operations)`,
);

// Telemetry templates request paths against these, so it loads only the
// paths, not the catalog.
const pathTemplates = [...new Set(operations.map((op) => op.path))].sort();

await writeFile(
PATH_TEMPLATES_OUTPUT_PATH,
JSON.stringify(pathTemplates, null, 2),
);

console.log(
`Generated ${PATH_TEMPLATES_OUTPUT_PATH} (${pathTemplates.length} paths)`,
);
3 changes: 2 additions & 1 deletion packages/cli/src/commands/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import {
hasPreviousPage,
resolveCursor,
} from "../lib/db/pagination.js";
import { normalizeEndpoint as templateEndpoint } from "../lib/error-reporting.js";
import {
OutputError,
stringifyUnknown,
Expand Down Expand Up @@ -1463,7 +1464,7 @@ function recordApiErrorAttributes(
const span = Sentry.getActiveSpan();
if (span) {
span.setAttribute("api_error.status", status);
span.setAttribute("api_error.endpoint", endpoint);
span.setAttribute("api_error.endpoint", templateEndpoint(endpoint));
span.setAttribute("api_error.method", method);
}
}
Expand Down
109 changes: 85 additions & 24 deletions packages/cli/src/lib/error-reporting.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ import {
UnsatisfiedFlagError,
UnsatisfiedPositionalError,
} from "@stricli/core";
import { matchApiPathTemplate } from "@sentry/toolkit-core/api-path-template";
import apiPathTemplates from "../generated/api-path-templates.json";
import { redactCredentialText } from "./credential-redaction.js";
import {
ApiError,
Expand Down Expand Up @@ -212,41 +214,100 @@ function recordSilencedError(error: unknown, reason: SilenceReason): void {
// Grouping tags
// ---------------------------------------------------------------------------

/** Endpoint normalization patterns — compiled once at module scope. */
const ENDPOINT_PATTERNS: [RegExp, string][] = [
[/\/organizations\/[^/]+/, "/organizations/{org}"],
[/\/projects\/[^/]+\/[^/]+/, "/projects/{org}/{project}"],
[/\/issues\/[^/]+/, "/issues/{id}"],
[/\/events\/[^/]+/, "/events/{id}"],
[/\/groups\/[^/]+/, "/groups/{id}"],
[/\/releases\/[^/]+/, "/releases/{version}"],
[/\/teams\/[^/]+\/[^/]+/, "/teams/{org}/{team}"],
[/\/dashboards\/[^/]+/, "/dashboards/{id}"],
[/\/customers\/[^/]+/, "/customers/{org}"],
];
/** The API version prefix every OpenAPI path template starts with. */
const API_PREFIX = "/api/0";

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this also come from the @sentry/api package? We can make it happen

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I think it should. @sentry/api doesn't export the API prefix today, in 0.428.0 on this branch or 1.59.0 on main. Toolkit defines "/api/0" privately in three places: toolkit-core/src/api-client/core.ts, cli/src/commands/api.ts, and now error-reporting.ts. An export from @sentry/api, for example next to buildSentryApiUrl, would let all three share it.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well then, make it happen. Ideally this would come all the way from the source in getsentry/sentry so we don't drift.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I looked at what it would take to source this from the spec:

  • getsentry/sentry: the OpenAPI spec's servers entry is just https://{region}.sentry.io. /api/0 is spelled out at the start of every path in the spec, so the prefix isn't a separate field there today.
  • getsentry/sentry-api-schema (publishes @sentry/api): the prefix is private there too. It's a const API_PREFIX in lib/api-client/core.ts, plus hard-coded /api/0/ in api-path.ts and api-path-scope.ts.

To avoid drift, sentry-api-schema could derive the prefix from the spec's paths at build time and export it. toolkit's three copies would then import it. Moving /api/0 into servers in getsentry/sentry would also work, but it changes the spec for every client.

This is a change in other repos, so I'm checking the scope with @betegon before I start. I'll leave this thread open until then.


/** OpenAPI path templates without their trailing slash, with segment counts. */
const PATH_TEMPLATES = (apiPathTemplates as string[]).map((path) => {
const template = path.replace(/\/$/, "");
return { template, length: template.split("/").length };
});

/** Segments the templates spell out literally, such as `issues`. */
const LITERAL_SEGMENTS = new Set(
PATH_TEMPLATES.flatMap(({ template }) => template.split("/")).filter(
(segment) => !segment.startsWith("{"),
),
);

/** Scheme and host of an absolute URL. */
const URL_ORIGIN_RE = /^[a-z][a-z\d+.-]*:\/\/[^/?#]*/i;

/**
* Strip remaining bare numeric segments (e.g. /12345/) but preserve
* the API version prefix /0/ which is always the second segment.
* Template `apiPath` with the OpenAPI path template that matches the most of
* it, most specific first. Segments past that match keep their literal
* segments and replace every other segment with `{id}`, so a path outside
* the catalog leaks nothing.
*/
const BARE_NUMERIC_SEGMENT_RE = /(?<=\/api\/0\/.*)\/\d+(?=\/|$)/g;
function templateApiPath(apiPath: string): string {
const segments = apiPath.split("/");
let best = {
template: API_PREFIX,
length: API_PREFIX.split("/").length,
literalSegments: 0,
};
for (const { template, length } of PATH_TEMPLATES) {
if (length > segments.length || length < best.length) {
continue;
}
const match = matchApiPathTemplate(
template,
segments.slice(0, length).join("/"),
);
if (
match &&
(length > best.length || match.literalSegments > best.literalSegments)
) {
best = { template, length, literalSegments: match.literalSegments };
}
}
const rest = segments
.slice(best.length)
.map((segment) => (LITERAL_SEGMENTS.has(segment) ? segment : "{id}"));
return [best.template, ...rest].join("/");
}

/**
* Normalize an API endpoint path by parameterizing variable segments.
*
* Replaces org slugs, project slugs, issue IDs, event IDs, and other
* entity identifiers with placeholders so that server-side fingerprint
* rules can sub-group `ApiError` by endpoint shape rather than exact path.
* entity identifiers with the placeholders of the `@sentry/api` OpenAPI
* path templates, so that server-side fingerprint rules can sub-group
* `ApiError` by endpoint shape rather than exact path.
*
* Accepts a path relative to `/api/0/`, with or without a leading slash, or
* a full URL, whose origin and any install prefix before `/api/0/` are kept.
* Leaves full URLs outside `/api/0/` alone, since they are not API requests.
* Drops the query string and fragment.
*
* `"/api/0/projects/my-org/my-project/events/abc123/"` →
* `"/api/0/projects/{org}/{project}/events/{id}/"`
* `"/api/0/projects/{organization_id_or_slug}/{project_id_or_slug}/events/{event_id}/"`
*/
export function normalizeEndpoint(endpoint: string): string {
let result = endpoint;
for (const [pattern, replacement] of ENDPOINT_PATTERNS) {
result = result.replace(pattern, replacement);
}
return result.replace(BARE_NUMERIC_SEGMENT_RE, "/{id}");
const origin = URL_ORIGIN_RE.exec(endpoint)?.[0] ?? "";
const path = endpoint.slice(origin.length).split(/[?#]/, 1)[0] ?? "";
const segments = path.split("/").filter(Boolean);
const apiIndex = segments.findIndex(
(segment, index) => segment === "api" && segments[index + 1] === "0",
);
if (origin && apiIndex === -1) {
return origin + path;
}
// A path without /api/0 is relative to it. Keep a self-hosted install
// prefix before it.
const relative = apiIndex === -1;
const template = templateApiPath(
[API_PREFIX, ...segments.slice(relative ? 0 : apiIndex + 2)].join("/"),
);
const prefix = segments
.slice(0, Math.max(apiIndex, 0))
.map((segment) => `/${segment}`)
.join("");
let result = relative ? template.slice(API_PREFIX.length) : prefix + template;
if (path.endsWith("/") && !result.endsWith("/")) {
result += "/";
}
return origin || path.startsWith("/") ? origin + result : result.slice(1);
}

/**
Expand Down Expand Up @@ -396,7 +457,7 @@ function setCliErrorContext(scope: Sentry.Scope, error: unknown): void {
if (error instanceof ApiError) {
scope.setContext("api_error", {
status: error.status,
endpoint: error.endpoint,
endpoint: error.endpoint && normalizeEndpoint(error.endpoint),
detail: error.detail,
});
} else if (error instanceof ContextError) {
Expand Down
25 changes: 25 additions & 0 deletions packages/cli/src/lib/telemetry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/

import { createRequire } from "node:module";
import type { TransactionEvent } from "@sentry/core";
// oxlint-disable-next-line sentry-cli/no-namespace-import -- Sentry SDK recommends namespace import
import * as Sentry from "@sentry/node";

Expand All @@ -36,6 +37,7 @@ import { getEnv } from "./env.js";
import {
classifySilenced,
enrichEventWithGroupingTags,
normalizeEndpoint,
reportCliError,
} from "./error-reporting.js";
import { ApiError, isUserError } from "./errors.js";
Expand Down Expand Up @@ -396,6 +398,28 @@ export function recordApiErrorOnSpan(span: Span, error: ApiError): void {
}
}

/** `http.client` span attributes that hold the request URL or path. */
const HTTP_SPAN_URL_KEYS = ["url", "url.full", "http.url", "url.path"];

/** Template org and project slugs out of `http.client` span URLs. */
function normalizeHttpSpans(event: TransactionEvent): void {
for (const span of event.spans ?? []) {
if (span.op !== "http.client") {
continue;
}
// Descriptions are `${method} ${url}`.
span.description = span.description?.replace(/\S+$/, (url) =>
normalizeEndpoint(url),
);
for (const key of HTTP_SPAN_URL_KEYS) {
const value = span.data[key];
if (typeof value === "string") {
span.data[key] = normalizeEndpoint(value);
}
}
}
}

/**
* Integrations to exclude for CLI.
* These add overhead without benefit for short-lived CLI processes.
Expand Down Expand Up @@ -690,6 +714,7 @@ export function initSentry(
beforeSendTransaction: (event) => {
// Remove server_name which may contain hostname (PII)
event.server_name = undefined;
normalizeHttpSpans(event);
return event;
},

Expand Down
71 changes: 20 additions & 51 deletions packages/cli/test/lib/error-reporting.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,56 +158,25 @@ describe("extractMessagePrefix", () => {
// ---------------------------------------------------------------------------

describe("normalizeEndpoint", () => {
test("parameterizes org slug in organizations path", () => {
expect(normalizeEndpoint("/api/0/organizations/my-org/issues/")).toBe(
"/api/0/organizations/{org}/issues/",
);
});

test("parameterizes org and project in projects path", () => {
expect(
normalizeEndpoint("/api/0/projects/my-org/my-project/events/abc123/"),
).toBe("/api/0/projects/{org}/{project}/events/{id}/");
});

test("parameterizes issue, event, group, release IDs", () => {
expect(normalizeEndpoint("/api/0/issues/12345/")).toBe(
"/api/0/issues/{id}/",
);
expect(normalizeEndpoint("/api/0/groups/99/events/abc/")).toBe(
"/api/0/groups/{id}/events/{id}/",
);
expect(normalizeEndpoint("/api/0/releases/1.0.0/")).toBe(
"/api/0/releases/{version}/",
);
});

test("parameterizes teams path", () => {
expect(normalizeEndpoint("/api/0/teams/my-org/backend/")).toBe(
"/api/0/teams/{org}/{team}/",
);
});

test("parameterizes dashboards path", () => {
expect(normalizeEndpoint("/api/0/dashboards/42/")).toBe(
"/api/0/dashboards/{id}/",
);
});

test("parameterizes customers path", () => {
expect(normalizeEndpoint("/api/0/customers/my-org/")).toBe(
"/api/0/customers/{org}/",
);
});

test("parameterizes bare numeric segments", () => {
expect(normalizeEndpoint("/api/0/some/123/thing/456")).toBe(
"/api/0/some/{id}/thing/{id}",
);
});

test("leaves paths without variable segments unchanged", () => {
expect(normalizeEndpoint("/api/0/auth/")).toBe("/api/0/auth/");
test.each([
[
"/api/0/projects/my-org/my-project/events/abc123/",
"/api/0/projects/{organization_id_or_slug}/{project_id_or_slug}/events/{event_id}/",
],
[
"organizations/my-org/releases/?per_page=1",
"organizations/{organization_id_or_slug}/releases/",
],
[
"https://sentry.example.com/sentry/api/0/organizations/my-org/issues/",
"https://sentry.example.com/sentry/api/0/organizations/{organization_id_or_slug}/issues/",
],
[
"organizations/my-org/not-an-endpoint/my-thing/",
"organizations/{organization_id_or_slug}/{id}/{id}/",
],
])("templates %s", (endpoint, expected) => {
expect(normalizeEndpoint(endpoint)).toBe(expected);
});
});

Expand Down Expand Up @@ -714,7 +683,7 @@ describe("reportCliError integration", () => {
expect(tags["cli_error.api_status"]).toBe("400");
expect(tags["cli_error.kind"]).toBe("400");
expect(tags["cli_error.api_endpoint"]).toBe(
"/api/0/organizations/{org}/issues/",
"/api/0/organizations/{organization_id_or_slug}/issues/",
);
});

Expand Down
Loading