Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/sentry/api/endpoints/organization_explore_formulas.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,12 +251,12 @@ def validate(self, data: FormulaData) -> FormulaData:
parameters,
calculations,
[SimpleNamespace(**param) for param in data["references"]],
resolver.resolve_column,
resolver,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dummy column fails non-span validation

Medium Severity

Formula save validation substitutes span.duration for every COLUMN parameter, then parse_formula and parse_arithmetic check that result against the dataset allowlist. Logs and metrics definitions do not include span.duration, so saving those formulas with column parameters now fails even when the real columns would be valid.

Additional Locations (1)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 366845f. Configure here.

)
except InvalidSearchQuery as e:
raise ValidationError(str(e))
try:
parse_arithmetic(formula)
parse_arithmetic(formula, definitions=resolver.definitions)
except (ArithmeticParseError, ArithmeticValidationError) as e:
raise ValidationError(str(e))

Expand Down
48 changes: 40 additions & 8 deletions src/sentry/discover/arithmetic.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,20 @@

import re
from dataclasses import dataclass
from typing import Literal, Union, overload
from typing import Iterable, Literal, Union, overload

from parsimonious.exceptions import ParseError
from parsimonious.grammar import Grammar
from parsimonious.nodes import NodeVisitor

from sentry.exceptions import InvalidSearchQuery
from sentry.search.events.constants import TOTAL_COUNT_ALIAS, TOTAL_TRANSACTION_DURATION_ALIAS
from sentry.search.eap.columns import ColumnDefinitions
from sentry.search.events.constants import (
DURATION_UNITS,
SIZE_UNITS,
TOTAL_COUNT_ALIAS,
TOTAL_TRANSACTION_DURATION_ALIAS,
)

# prefix on fields so we know they're equations
EQUATION_PREFIX = "equation|"
Expand Down Expand Up @@ -148,7 +154,7 @@ class ArithmeticVisitor(NodeVisitor):
# Don't wrap in VisitationErrors
unwrapped_exceptions = (ArithmeticError,)

field_allowlist = {
field_allowlist: Iterable[str] = {
"transaction.duration",
"spans.http",
"spans.db",
Expand Down Expand Up @@ -176,7 +182,7 @@ class ArithmeticVisitor(NodeVisitor):
TOTAL_COUNT_ALIAS,
TOTAL_TRANSACTION_DURATION_ALIAS,
}
function_allowlist = {
function_allowlist: Iterable[str] = {
"count",
"count_if",
"count_unique",
Expand Down Expand Up @@ -222,11 +228,36 @@ class ArithmeticVisitor(NodeVisitor):
"trace_status_rate",
}

def __init__(self, max_operators: int | None):
def __init__(self, max_operators: int | None, definitions: ColumnDefinitions | None):
super().__init__()
self.operators: int = 0
self.terms: int = 0
self.max_operators = max_operators if max_operators else self.DEFAULT_MAX_OPERATORS
self.definitions = definitions
if definitions:
self.field_allowlist = [
name
for name, column in definitions.columns.items()
if column.search_type
in [
"number",
"integer",
"duration",
"rate",
"currency",
"percentage",
*SIZE_UNITS.keys(),
*DURATION_UNITS.keys(),
]
]
self.function_allowlist = [
name
for name, function in definitions.aggregates.items()
if function.valid_arithmetic
]
self.function_allowlist.extend(
[name for name, formula in definitions.formulas.items() if formula.valid_arithmetic]
)
self.fields: set[str] = set()
Comment thread
sentry-warden[bot] marked this conversation as resolved.
Resolved
self.functions: set[str] = set()

Expand Down Expand Up @@ -323,20 +354,21 @@ def parse_arithmetic(
equation: str,
max_operators: int | None = None,
*,
definitions: ColumnDefinitions | None = None,
validate_single_operator: Literal[True],
) -> tuple[Operation, list[str], list[str]]: ...


@overload
def parse_arithmetic(
equation: str,
max_operators: int | None = None,
equation: str, max_operators: int | None = None, definitions: ColumnDefinitions | None = None
) -> tuple[Operation | float | str, list[str], list[str]]: ...


def parse_arithmetic(
equation: str,
max_operators: int | None = None,
definitions: ColumnDefinitions | None = None,
validate_single_operator: bool = False,
) -> tuple[Operation | float | str, list[str], list[str]]:
"""Given a string equation try to parse it into a set of Operations"""
Expand All @@ -346,7 +378,7 @@ def parse_arithmetic(
raise ArithmeticParseError(
"Unable to parse your equation, make sure it is well formed arithmetic"
)
visitor = ArithmeticVisitor(max_operators)
visitor = ArithmeticVisitor(max_operators, definitions)
result = visitor.visit(tree)
# total count is the exception to the no mixing rule
if (
Expand Down
2 changes: 2 additions & 0 deletions src/sentry/search/eap/columns.py
Original file line number Diff line number Diff line change
Expand Up @@ -478,6 +478,8 @@ class FunctionDefinition:
processor: Callable[[Any], Any] | None = None
# if a function is private, assume it can't be used unless it's provided in `SearchResolverConfig.functions_acl`
private: bool = False
# Usable in arithmetic
valid_arithmetic: bool = False

@property
def required_arguments(
Expand Down
1 change: 1 addition & 0 deletions src/sentry/search/eap/common_aggregates.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,5 @@ def count_unique_aggregate_definition(
default_arg=default_arg,
)
],
valid_arithmetic=True,
)
4 changes: 4 additions & 0 deletions src/sentry/search/eap/occurrences/aggregates.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
attribute_types=COMMON_COUNTABLE_ATTRIBUTE_TYPES,
)
],
valid_arithmetic=True,
),
"count": AggregateDefinition(
internal_function=Function.FUNCTION_COUNT,
Expand All @@ -59,6 +60,7 @@
)
],
attribute_resolver=count_argument_resolver_optimized(OCCURRENCES_ALWAYS_PRESENT_ATTRIBUTES),
valid_arithmetic=True,
),
"count_if": ConditionalAggregateDefinition(
internal_function=Function.FUNCTION_COUNT,
Expand Down Expand Up @@ -99,6 +101,7 @@
), # Second value is only for between, so it must be a number
],
aggregate_resolver=resolve_key_eq_value_filter,
valid_arithmetic=True,
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
),
"min": AggregateDefinition(
internal_function=Function.FUNCTION_MIN,
Expand All @@ -111,6 +114,7 @@
},
)
],
valid_arithmetic=True,
),
"count_unique": count_unique_aggregate_definition(default_arg="group_id"),
"first_seen": AggregateDefinition(
Expand Down
11 changes: 11 additions & 0 deletions src/sentry/search/eap/ourlogs/aggregates.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
)
],
attribute_resolver=count_argument_resolver_optimized(LOGS_ALWAYS_PRESENT_ATTRIBUTES),
valid_arithmetic=True,
),
"count_unique": count_unique_aggregate_definition(),
"sum": AggregateDefinition(
Expand All @@ -50,6 +51,7 @@
},
)
],
valid_arithmetic=True,
),
"avg": AggregateDefinition(
internal_function=Function.FUNCTION_AVG,
Expand All @@ -67,6 +69,7 @@
},
)
],
valid_arithmetic=True,
),
"p50": AggregateDefinition(
internal_function=Function.FUNCTION_P50,
Expand All @@ -84,6 +87,7 @@
},
)
],
valid_arithmetic=True,
),
"p75": AggregateDefinition(
internal_function=Function.FUNCTION_P75,
Expand All @@ -101,6 +105,7 @@
},
)
],
valid_arithmetic=True,
),
"p90": AggregateDefinition(
internal_function=Function.FUNCTION_P90,
Expand All @@ -118,6 +123,7 @@
},
)
],
valid_arithmetic=True,
),
"p95": AggregateDefinition(
internal_function=Function.FUNCTION_P95,
Expand All @@ -135,6 +141,7 @@
},
)
],
valid_arithmetic=True,
),
"p99": AggregateDefinition(
internal_function=Function.FUNCTION_P99,
Expand All @@ -152,6 +159,7 @@
},
)
],
valid_arithmetic=True,
),
"max": AggregateDefinition(
internal_function=Function.FUNCTION_MAX,
Expand All @@ -169,6 +177,7 @@
},
)
],
valid_arithmetic=True,
),
"min": AggregateDefinition(
internal_function=Function.FUNCTION_MIN,
Expand All @@ -186,6 +195,7 @@
},
)
],
valid_arithmetic=True,
),
}

Expand All @@ -204,6 +214,7 @@ def if_combinator(definition: AggregateDefinition) -> AggregateDefinition:
ValueArgumentDefinition(argument_types={"query"}, validator=if_query_validator),
*definition.arguments,
],
valid_arithmetic=definition.valid_arithmetic,
)


Expand Down
2 changes: 2 additions & 0 deletions src/sentry/search/eap/preprod_size/aggregates.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
},
)
],
valid_arithmetic=True,
),
"min": AggregateDefinition(
internal_function=Function.FUNCTION_MIN,
Expand All @@ -29,5 +30,6 @@
},
)
],
valid_arithmetic=True,
),
}
1 change: 1 addition & 0 deletions src/sentry/search/eap/processing_errors/aggregates.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
attribute_resolver=count_argument_resolver_optimized(
PROCESSING_ERRORS_ALWAYS_PRESENT_ATTRIBUTES
),
valid_arithmetic=True,
),
"count_unique": count_unique_aggregate_definition(default_arg="event_id"),
}
4 changes: 4 additions & 0 deletions src/sentry/search/eap/replays/aggregates.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
)
],
attribute_resolver=count_argument_resolver_optimized(REPLAYS_ALWAYS_PRESENT_ATTRIBUTES),
valid_arithmetic=True,
),
"sum": AggregateDefinition(
internal_function=Function.FUNCTION_SUM,
Expand All @@ -45,6 +46,7 @@
},
)
],
valid_arithmetic=True,
),
"min": AggregateDefinition(
internal_function=Function.FUNCTION_MIN,
Expand All @@ -63,6 +65,7 @@
},
)
],
valid_arithmetic=True,
),
"max": AggregateDefinition(
internal_function=Function.FUNCTION_MAX,
Expand All @@ -81,5 +84,6 @@
},
)
],
valid_arithmetic=True,
),
}
6 changes: 4 additions & 2 deletions src/sentry/search/eap/resolver.py
Original file line number Diff line number Diff line change
Expand Up @@ -1492,7 +1492,9 @@ def resolve_equation(
"""Resolve an equation creating a ResolvedEquation object, we don't just return a Column.BinaryFormula since
it'll help callers with extra information, like the existence of aggregates and the search type
"""
operation, fields, functions = arithmetic.parse_arithmetic(equation)
operation, fields, functions = arithmetic.parse_arithmetic(
equation, definitions=self.definitions
)
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
# Handle the case where the equation is just a single term
if isinstance(operation, str):
# Resolve the column, and turn it into a RPC Column so it can be used in a BinaryFormula
Expand Down Expand Up @@ -1637,5 +1639,5 @@ def resolve_formula(

arguments = fields.parse_arguments(formula.name, columns)

equation = resolve_and_parse_formula(formula, arguments, self.resolve_column)
equation = resolve_and_parse_formula(formula, arguments, self)
return self.resolve_equation(equation, alias)
Loading
Loading