mirror of
https://github.com/SigNoz/signoz.git
synced 2026-11-04 15:31:08 +00:00
Compare commits
1 Commits
fix/aggreg
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0cc223fb5a |
@@ -80,7 +80,7 @@
|
||||
"crypto-js": "4.2.0",
|
||||
"d3-hierarchy": "3.1.2",
|
||||
"dayjs": "^1.10.7",
|
||||
"dompurify": "3.4.15",
|
||||
"dompurify": "3.4.17",
|
||||
"event-source-polyfill": "1.0.31",
|
||||
"eventemitter3": "5.0.1",
|
||||
"history": "4.10.1",
|
||||
@@ -200,17 +200,17 @@
|
||||
"oxfmt": "0.73.0",
|
||||
"oxlint": "1.88.0",
|
||||
"oxlint-tsgolint": "7.0.2004",
|
||||
"postcss": "8.5.26",
|
||||
"postcss": "8.5.32",
|
||||
"postcss-scss": "4.0.9",
|
||||
"react-resizable": "3.0.4",
|
||||
"redux-mock-store": "1.5.4",
|
||||
"sass": "1.97.3",
|
||||
"sharp": "0.35.4",
|
||||
"sharp": "0.35.5",
|
||||
"storybook": "10.6.3",
|
||||
"stylelint": "17.15.0",
|
||||
"svgo": "4.1.0",
|
||||
"ts-jest": "29.4.15",
|
||||
"typescript-plugin-css-modules": "5.2.0",
|
||||
"typescript-plugin-css-modules": "5.3.0",
|
||||
"use-sync-external-store": "1.6.0",
|
||||
"vite-plugin-checker": "0.12.0",
|
||||
"vite-plugin-compression": "0.5.1",
|
||||
|
||||
465
frontend/pnpm-lock.yaml
generated
465
frontend/pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@@ -37,9 +37,9 @@ overrides:
|
||||
cookie@<0.7.0: '>=0.7.1 <1'
|
||||
# via: @grafana/data@11.6.15 (3.4.0/3.2.4 exact);
|
||||
# @monaco-editor/react > monaco-editor@0.55.1 (3.2.7 exact)
|
||||
# remove: blocked. The direct dep is already on 3.4.15, but @grafana/data
|
||||
# remove: blocked. The direct dep is already on 3.4.17, but @grafana/data
|
||||
# (latest 13.1.0) and monaco-editor (latest 0.55.1) still pin vulnerable versions
|
||||
dompurify@<=3.4.12: '>=3.4.13 <4'
|
||||
dompurify@<=3.4.16: '>=3.4.17 <4'
|
||||
# via: rolldown-vite@7.3.1 (esbuild ^0.27.0); ts-jest@29.4.9 (~0.27.4)
|
||||
# remove: blocked on rolldown-vite (7.3.1 is latest, still ^0.27.0);
|
||||
# orval >=8.20.0 and ts-jest >=29.4.11 already fixed on their side
|
||||
@@ -60,17 +60,10 @@ overrides:
|
||||
# @commitlint/cli > @commitlint/load > cosmiconfig@9 (^4.1.0)
|
||||
# remove: blocked, both consumers cap js-yaml at ^4 and only 4.3.2 carries the fix
|
||||
'js-yaml@>=4.0.0 <4.3.2': '>=4.3.2 <5'
|
||||
# via: typescript-plugin-css-modules@5.2.0 (less ^4.2.0) > less@4.4.0 (image-size ~0.5.0)
|
||||
# remove: bump typescript-plugin-css-modules once it floors less itself; image-size has
|
||||
# no patched release at all, so dropping the dep is the only fix — less@4.5.0 did
|
||||
less@<4.5.0: '>=4.5.0 <5'
|
||||
# via: direct dep postcss (nanoid ^3.3.17)
|
||||
# remove: blocked — postcss 8.5.26 (latest) caps nanoid at ^3, fix landed in 3.3.18
|
||||
nanoid@<3.3.18: '>=3.3.18 <4'
|
||||
# via: react-syntax-highlighter@15.5.0 (prismjs ^1.27.0 + refractor@3 ~1.27.0 tilde-pinned)
|
||||
# remove: bump react-syntax-highlighter to >=16.1.1 (prismjs ^1.30.0, refractor@5)
|
||||
prismjs@<1.30.0: '>=1.30.0 <2'
|
||||
# via: typescript-plugin-css-modules@5.2.0 > postcss-modules-local-by-default@4.2.0
|
||||
# via: typescript-plugin-css-modules@5.3.0 > postcss-modules-local-by-default@4.2.0
|
||||
# (postcss-selector-parser ^7.0.0)
|
||||
# remove: blocked, 4.2.0 is latest and the range is open, so the floor is what pulls the fix
|
||||
postcss-selector-parser@>=7.1.0 <7.1.3: '>=7.1.3 <8'
|
||||
|
||||
@@ -5,22 +5,17 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
chparser "github.com/AfterShip/clickhouse-sql-parser/parser"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
grammar "github.com/SigNoz/signoz/pkg/parser/filterquery/grammar"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
)
|
||||
|
||||
const argumentPlaceholderPrefix = "__signoz_argument_"
|
||||
|
||||
type aggExprRewriter struct {
|
||||
logger *slog.Logger
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
@@ -62,13 +57,12 @@ func (r *aggExprRewriter) Rewrite(
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, []any, error) {
|
||||
|
||||
arguments := map[string]string{}
|
||||
wrapped := fmt.Sprintf("SELECT %s", extractArguments(expr, arguments))
|
||||
wrapped := fmt.Sprintf("SELECT %s", expr)
|
||||
p := chparser.NewParser(wrapped)
|
||||
stmts, err := p.ParseStmts()
|
||||
|
||||
if err != nil {
|
||||
return "", nil, errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to parse aggregation expression %q", expr)
|
||||
return "", nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to parse aggregation expression %q", expr)
|
||||
}
|
||||
|
||||
if len(stmts) == 0 {
|
||||
@@ -91,7 +85,6 @@ func (r *aggExprRewriter) Rewrite(
|
||||
fieldKeys: keys,
|
||||
fullTextColumn: r.fullTextColumn,
|
||||
storage: r.storage,
|
||||
arguments: arguments,
|
||||
}
|
||||
// Rewrite the first select item (our expression)
|
||||
if err := sel.SelectItems[0].Accept(visitor); err != nil {
|
||||
@@ -143,7 +136,6 @@ type exprVisitor struct {
|
||||
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
storage qbtypes.Storage
|
||||
arguments map[string]string
|
||||
Modified bool
|
||||
chArgs []any
|
||||
isRate bool
|
||||
@@ -163,16 +155,10 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
args = fn.Params.Items.Items
|
||||
}
|
||||
|
||||
// count(*) is count()
|
||||
if len(args) == 1 && aggFunc.MinArgs == 0 && v.arguments[chparser.Format(args[0])] == "*" {
|
||||
fn.Params.Items.Items = nil
|
||||
args = nil
|
||||
}
|
||||
|
||||
// if we know aggregation function, we must ensure that the number of arguments is correct
|
||||
if aggFunc.RequireArgs {
|
||||
if len(args) < aggFunc.MinArgs || len(args) > aggFunc.MaxArgs {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid number of arguments for %q: %d", name, len(args))
|
||||
return errors.NewInternalf(errors.CodeInternal, "invalid number of arguments for %q: %d", name, len(args))
|
||||
}
|
||||
}
|
||||
fn.Name.Name = aggFunc.FuncName
|
||||
@@ -188,10 +174,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
// Handle *If functions with predicate + values
|
||||
if aggFunc.FuncCombinator {
|
||||
// Map the predicate (last argument)
|
||||
origPred, err := v.argumentText(name, args[len(args)-1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
origPred := chparser.Format(args[len(args)-1])
|
||||
whereClause, err := PrepareWhereClause(
|
||||
origPred,
|
||||
FilterExprVisitorOpts{
|
||||
@@ -206,10 +189,6 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The search scan budget applies only to the filter.
|
||||
if whereClause.RequiresCostGuard {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "function `search` is not supported inside an aggregation").WithSuggestions("use `search` in the filter")
|
||||
}
|
||||
// not possible for whereClause to be empty here but still adding a check.
|
||||
if whereClause.IsEmpty() {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid predicate argument for %q: %q", name, origPred)
|
||||
@@ -223,18 +202,12 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
}
|
||||
args[len(args)-1] = parsedPred
|
||||
v.Modified = true
|
||||
v.chArgs = append(v.chArgs, chArgs...)
|
||||
v.chArgs = chArgs
|
||||
|
||||
// Map each value column argument
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
origVal, err := v.argumentText(name, args[i])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fieldKey, err := fieldKeyFromArgument(name, origVal)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
origVal := chparser.Format(args[i])
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(origVal)
|
||||
expr, err := ResolveColumn(v.ctx, v.query, v.storage, &fieldKey, dataType, v.fieldKeys)
|
||||
if err != nil {
|
||||
return errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to get table field name for %q", origVal)
|
||||
@@ -250,14 +223,8 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
} else {
|
||||
// Non-If functions: map every argument as a column/value
|
||||
for i, arg := range args {
|
||||
orig, err := v.argumentText(name, arg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fieldKey, err := fieldKeyFromArgument(name, orig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
orig := chparser.Format(arg)
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(orig)
|
||||
expr, err := ResolveColumn(v.ctx, v.query, v.storage, &fieldKey, dataType, v.fieldKeys)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -278,14 +245,6 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (v *exprVisitor) argumentText(name string, arg chparser.Expr) (string, error) {
|
||||
text, ok := v.arguments[chparser.Format(arg)]
|
||||
if !ok {
|
||||
return "", errors.NewInternalf(errors.CodeInternal, "argument of %q was not extracted before parsing", name)
|
||||
}
|
||||
return text, nil
|
||||
}
|
||||
|
||||
// parseFragment parses a SQL expression fragment by wrapping in SELECT.
|
||||
func parseFragment(sql string) (chparser.Expr, error) {
|
||||
wrapped := fmt.Sprintf("SELECT %s", sql)
|
||||
@@ -303,134 +262,3 @@ func parseFragment(sql string) (chparser.Expr, error) {
|
||||
}
|
||||
return sel.SelectItems[0].Expr, nil
|
||||
}
|
||||
|
||||
// extractArguments replaces every argument of an aggregation call with a
|
||||
// placeholder kept in arguments, because the ClickHouse parser cannot read
|
||||
// the predicates such as EXISTS or CONTAINS, and field keys
|
||||
// with hyphens or slashes. Unbalanced input is returned unchanged for the
|
||||
// parser to reject.
|
||||
func extractArguments(expr string, arguments map[string]string) string {
|
||||
var out strings.Builder
|
||||
written := 0
|
||||
for i := 0; i < len(expr); i++ {
|
||||
c := expr[i]
|
||||
if c == '\'' || c == '"' || c == '`' {
|
||||
end := skipQuoted(expr, i)
|
||||
if end < 0 {
|
||||
return expr
|
||||
}
|
||||
i = end
|
||||
continue
|
||||
}
|
||||
if !isIdentifierStart(c) || (i > 0 && (isIdentifierPart(expr[i-1]) || expr[i-1] == '.')) {
|
||||
continue
|
||||
}
|
||||
nameEnd := i
|
||||
for nameEnd < len(expr) && isIdentifierPart(expr[nameEnd]) {
|
||||
nameEnd++
|
||||
}
|
||||
open := nameEnd
|
||||
for open < len(expr) && strings.ContainsRune(" \t\r\n", rune(expr[open])) {
|
||||
open++
|
||||
}
|
||||
if _, ok := AggreFuncMap[valuer.NewString(strings.ToLower(expr[i:nameEnd]))]; !ok || open == len(expr) || expr[open] != '(' {
|
||||
i = nameEnd - 1
|
||||
continue
|
||||
}
|
||||
commas, closing := scanArguments(expr, open)
|
||||
if closing < 0 {
|
||||
return expr
|
||||
}
|
||||
out.WriteString(expr[written : open+1])
|
||||
bounds := append(append([]int{open}, commas...), closing)
|
||||
for j := 1; j < len(bounds); j++ {
|
||||
if j > 1 {
|
||||
out.WriteString(", ")
|
||||
}
|
||||
text := strings.TrimSpace(expr[bounds[j-1]+1 : bounds[j]])
|
||||
// DISTINCT modifies the whole argument list, so it stays for the parser.
|
||||
if end := strings.IndexFunc(text, unicode.IsSpace); j == 1 && end > 0 && strings.EqualFold(text[:end], "DISTINCT") {
|
||||
out.WriteString("DISTINCT ")
|
||||
text = strings.TrimSpace(text[end:])
|
||||
}
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
placeholder := fmt.Sprintf("%s%d", argumentPlaceholderPrefix, len(arguments))
|
||||
arguments[placeholder] = text
|
||||
out.WriteString(placeholder)
|
||||
}
|
||||
written = closing
|
||||
i = closing
|
||||
}
|
||||
out.WriteString(expr[written:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
// scanArguments returns the top-level commas and the closing parenthesis of
|
||||
// the argument list that opens at open, with -1 as the closing index when the
|
||||
// list is unterminated.
|
||||
func scanArguments(expr string, open int) ([]int, int) {
|
||||
var commas []int
|
||||
depth := 0
|
||||
for i := open + 1; i < len(expr); i++ {
|
||||
switch expr[i] {
|
||||
case '\'', '"', '`':
|
||||
end := skipQuoted(expr, i)
|
||||
if end < 0 {
|
||||
return nil, -1
|
||||
}
|
||||
i = end
|
||||
case '(', '[':
|
||||
depth++
|
||||
case ']':
|
||||
depth--
|
||||
case ')':
|
||||
if depth == 0 {
|
||||
return commas, i
|
||||
}
|
||||
depth--
|
||||
case ',':
|
||||
if depth == 0 {
|
||||
commas = append(commas, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, -1
|
||||
}
|
||||
|
||||
// fieldKeyFromArgument reads a value argument as a single key of the filter
|
||||
// grammar.
|
||||
func fieldKeyFromArgument(name string, text string) (telemetrytypes.TelemetryFieldKey, error) {
|
||||
lexer := grammar.NewFilterQueryLexer(antlr.NewInputStream(text))
|
||||
lexer.RemoveErrorListeners()
|
||||
if lexer.NextToken().GetTokenType() != grammar.FilterQueryLexerKEY || lexer.NextToken().GetTokenType() != antlr.TokenEOF {
|
||||
return telemetrytypes.TelemetryFieldKey{}, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid argument for %q: %q is not a field name", name, text)
|
||||
}
|
||||
return telemetrytypes.GetFieldKeyFromKeyText(text), nil
|
||||
}
|
||||
|
||||
// skipQuoted returns the index of the quote closing the one at start, honoring
|
||||
// backslash escapes and doubled quotes, or -1 when it is unterminated.
|
||||
func skipQuoted(expr string, start int) int {
|
||||
quote := expr[start]
|
||||
for i := start + 1; i < len(expr); i++ {
|
||||
switch {
|
||||
case expr[i] == '\\':
|
||||
i++
|
||||
case expr[i] == quote && i+1 < len(expr) && expr[i+1] == quote:
|
||||
i++
|
||||
case expr[i] == quote:
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func isIdentifierStart(c byte) bool {
|
||||
return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
|
||||
}
|
||||
|
||||
func isIdentifierPart(c byte) bool {
|
||||
return isIdentifierStart(c) || (c >= '0' && c <= '9')
|
||||
}
|
||||
|
||||
@@ -137,46 +137,6 @@ def test_logs_filter_expression_with_dollar_in_key(
|
||||
assert {row["data"]["body"] for row in get_rows(response)} == expected_bodies, f"{expression}: {response.text}"
|
||||
|
||||
|
||||
def test_logs_aggregation_with_dollar_in_key(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC)
|
||||
insert_logs(
|
||||
[
|
||||
Logs(timestamp=now - timedelta(seconds=3), resources={"service.name": SERVICE}, attributes={"at$0dollar": "alpha-dollar", "at${x}": "a$0b", "n$0um": 5}, body="alpha body"),
|
||||
Logs(timestamp=now - timedelta(seconds=2), resources={"service.name": SERVICE}, attributes={"at$0dollar": "beta-dollar", "at${x}": "it's", "n$0um": 7}, body="beta body"),
|
||||
Logs(timestamp=now - timedelta(seconds=1), resources={"service.name": SERVICE}, attributes={"plain": "gamma"}, body="gamma body"),
|
||||
]
|
||||
)
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
start_ms = int((now - timedelta(minutes=2)).timestamp() * 1000)
|
||||
end_ms = int((now + timedelta(seconds=1)).timestamp() * 1000)
|
||||
|
||||
for expression, expected in [
|
||||
("count_distinct(at$0dollar)", 2),
|
||||
("count(DISTINCT at$0dollar)", 2),
|
||||
("sum(n$0um)", 12),
|
||||
("countIf(at$0dollar EXISTS)", 2),
|
||||
("count_distinct(at${x})", 2),
|
||||
("countIf(at${x} = 'a$0b')", 1),
|
||||
("countIf(at${x} = 'it\\'s')", 1),
|
||||
("sumIf(n$0um, at$0dollar = 'beta-dollar')", 7),
|
||||
]:
|
||||
response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms,
|
||||
end_ms,
|
||||
[build_scalar_query("A", "logs", [build_aggregation(expression)], filter_expression=f"service.name = '{SERVICE}'")],
|
||||
request_type="scalar",
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, f"{expression}: {response.text}"
|
||||
assert get_scalar_table_data(response.json()) == [[expected]], f"{expression}: {response.text}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key_id", list(RESOURCE_KEYS), ids=list(RESOURCE_KEYS))
|
||||
def test_logs_group_by_hostile_resource_attribute(
|
||||
signoz: types.SigNoz,
|
||||
|
||||
@@ -128,172 +128,6 @@ def test_logs_aggregate_functions(
|
||||
assert by_service[service] == pytest.approx(expected), f"{service}: {by_service[service]} != {expected}"
|
||||
|
||||
|
||||
def test_logs_aggregate_if_predicates_use_filter_syntax(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
svc-a: 3 logs, two with a k8s.pod.name and an x-request-id header, two
|
||||
with "timeout" in the body; svc-b: 1 log without either, with "timeout".
|
||||
|
||||
Tests:
|
||||
countIf predicates accept the filter-only operators EXISTS, NOT EXISTS,
|
||||
CONTAINS and NOT CONTAINS, a combination of them, and a key with hyphens;
|
||||
two predicates in one expression each keep their own values.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
# (service, k8s.pod.name, x-request-id header, body)
|
||||
specs = [
|
||||
("svc-a", "pod-1", "abc", "request timeout"),
|
||||
("svc-a", None, None, "request ok"),
|
||||
("svc-a", "pod-2", "def", "upstream timeout"),
|
||||
("svc-b", None, None, "timeout"),
|
||||
]
|
||||
logs = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": service} | ({"k8s.pod.name": pod} if pod else {}),
|
||||
attributes={"http.request.header.x-request-id": request_id} if request_id else {},
|
||||
body=body,
|
||||
)
|
||||
for i, (service, pod, request_id, body) in enumerate(specs)
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
query = build_scalar_query(
|
||||
name="A",
|
||||
signal="logs",
|
||||
aggregations=[
|
||||
build_aggregation("count()", "cnt"),
|
||||
build_aggregation("countIf(k8s.pod.name EXISTS)", "with_pod"),
|
||||
build_aggregation("countIf(k8s.pod.name NOT EXISTS)", "without_pod"),
|
||||
build_aggregation("countIf(body CONTAINS 'timeout')", "timeouts"),
|
||||
build_aggregation("countIf(body NOT CONTAINS 'timeout')", "not_timeouts"),
|
||||
build_aggregation("countIf(body CONTAINS 'timeout' AND k8s.pod.name EXISTS)", "pod_timeouts"),
|
||||
build_aggregation("countIf(http.request.header.x-request-id = 'abc')", "request_abc"),
|
||||
build_aggregation("countIf(k8s.pod.name = 'pod-1') + countIf(body CONTAINS 'ok')", "pod_1_or_ok"),
|
||||
],
|
||||
group_by=[build_group_by_field("service.name", "string", "resource")],
|
||||
order=[build_order_by("count()", "desc")],
|
||||
)
|
||||
response = make_scalar_query_request(signoz, token, now, [query])
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
data = get_scalar_table_data(response.json())
|
||||
assert [row[0] for row in data] == ["svc-a", "svc-b"]
|
||||
by_service = {row[0]: row[1:] for row in data}
|
||||
|
||||
for service in ["svc-a", "svc-b"]:
|
||||
rows = [(pod, request_id, body) for svc, pod, request_id, body in specs if svc == service]
|
||||
expected = [
|
||||
len(rows),
|
||||
sum(1 for pod, _, _ in rows if pod),
|
||||
sum(1 for pod, _, _ in rows if not pod),
|
||||
sum(1 for _, _, body in rows if "timeout" in body),
|
||||
sum(1 for _, _, body in rows if "timeout" not in body),
|
||||
sum(1 for pod, _, body in rows if "timeout" in body and pod),
|
||||
sum(1 for _, request_id, _ in rows if request_id == "abc"),
|
||||
sum(1 for pod, _, _ in rows if pod == "pod-1") + sum(1 for _, _, body in rows if "ok" in body),
|
||||
]
|
||||
assert by_service[service] == expected, f"{service}: {by_service[service]} != {expected}"
|
||||
|
||||
|
||||
def test_logs_aggregate_arguments_are_field_names(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
svc-a: 3 logs, svc-b: 1 log, each with a body size under a key with a
|
||||
hyphen and an app label under a key with slashes.
|
||||
|
||||
Tests:
|
||||
Aggregation arguments are read as field names, so keys with hyphens,
|
||||
slashes and a type suffix resolve, count(*) counts every log, and DISTINCT
|
||||
keeps its meaning.
|
||||
"""
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
# (service, http.response.body-size, k8s.pod.labels.app.kubernetes.io/name)
|
||||
specs = [
|
||||
("svc-a", 100, "web"),
|
||||
("svc-a", 250, "web"),
|
||||
("svc-a", 400, "api"),
|
||||
("svc-b", 70, "worker"),
|
||||
]
|
||||
logs = [
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i + 1),
|
||||
resources={"service.name": service},
|
||||
attributes={"http.response.body-size": size, "k8s.pod.labels.app.kubernetes.io/name": app},
|
||||
body=f"{service} log {i}",
|
||||
)
|
||||
for i, (service, size, app) in enumerate(specs)
|
||||
]
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
query = build_scalar_query(
|
||||
name="A",
|
||||
signal="logs",
|
||||
aggregations=[
|
||||
build_aggregation("count(*)", "cnt"),
|
||||
build_aggregation("sum(http.response.body-size)", "body_size"),
|
||||
build_aggregation("max(http.response.body-size:float64)", "max_body_size"),
|
||||
build_aggregation("count_distinct(k8s.pod.labels.app.kubernetes.io/name)", "apps"),
|
||||
build_aggregation("count(DISTINCT k8s.pod.labels.app.kubernetes.io/name)", "distinct_apps"),
|
||||
],
|
||||
group_by=[build_group_by_field("service.name", "string", "resource")],
|
||||
order=[build_order_by("count(*)", "desc")],
|
||||
)
|
||||
response = make_scalar_query_request(signoz, token, now, [query])
|
||||
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
data = get_scalar_table_data(response.json())
|
||||
assert [row[0] for row in data] == ["svc-a", "svc-b"]
|
||||
by_service = {row[0]: row[1:] for row in data}
|
||||
|
||||
for service in ["svc-a", "svc-b"]:
|
||||
sizes = [size for svc, size, _ in specs if svc == service]
|
||||
apps = {app for svc, _, app in specs if svc == service}
|
||||
expected = [len(sizes), sum(sizes), max(sizes), len(apps), len(apps)]
|
||||
assert by_service[service] == expected, f"{service}: {by_service[service]} != {expected}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"expression",
|
||||
[
|
||||
pytest.param("countIf(k8s.pod.name EXISTS", id="unclosed_parenthesis"),
|
||||
pytest.param("countIf()", id="missing_predicate"),
|
||||
pytest.param("countIf(latency_ms >= 25, latency_ms < 100)", id="extra_argument"),
|
||||
pytest.param("sum(1)", id="literal_argument"),
|
||||
pytest.param("sum(latency_ms * 2)", id="arithmetic_in_argument"),
|
||||
pytest.param("sum(attributes_number['latency_ms'])", id="column_expression_argument"),
|
||||
pytest.param("countIf(search('timeout'))", id="search_in_predicate"),
|
||||
],
|
||||
)
|
||||
def test_logs_aggregate_invalid_expression_is_bad_request(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
expression: str,
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
query = build_scalar_query(name="A", signal="logs", aggregations=[build_aggregation(expression)])
|
||||
response = make_scalar_query_request(signoz, token, now, [query])
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"having_expression,aggregations,expected_services",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user