mirror of
https://github.com/SigNoz/signoz.git
synced 2026-11-20 15:21:19 +00:00
Compare commits
13 Commits
main
...
issue_nerv
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea10c2542e | ||
|
|
9436019094 | ||
|
|
da6c366e8e | ||
|
|
07b1ddf645 | ||
|
|
141f5cd1f4 | ||
|
|
9dabe4e2c3 | ||
|
|
690a2af781 | ||
|
|
82c8c1c43d | ||
|
|
047561e669 | ||
|
|
f557a6563f | ||
|
|
255b6fa1ac | ||
|
|
75686f4004 | ||
|
|
2df97d39e2 |
@@ -12680,6 +12680,19 @@ components:
|
||||
minimum: 0
|
||||
type: integer
|
||||
type: object
|
||||
SpantypesEventV2:
|
||||
properties:
|
||||
attributes:
|
||||
additionalProperties: {}
|
||||
type: object
|
||||
isError:
|
||||
type: boolean
|
||||
name:
|
||||
type: string
|
||||
timeUnixNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
type: object
|
||||
SpantypesFieldContext:
|
||||
enum:
|
||||
- attribute
|
||||
@@ -12872,6 +12885,13 @@ components:
|
||||
nullable: true
|
||||
type: array
|
||||
type: object
|
||||
SpantypesLink:
|
||||
properties:
|
||||
spanId:
|
||||
type: string
|
||||
traceId:
|
||||
type: string
|
||||
type: object
|
||||
SpantypesOtelSpanRef:
|
||||
properties:
|
||||
refType:
|
||||
@@ -13163,6 +13183,100 @@ components:
|
||||
nullable: true
|
||||
type: object
|
||||
type: object
|
||||
SpantypesSpanRecord:
|
||||
properties:
|
||||
attributes:
|
||||
additionalProperties: {}
|
||||
type: object
|
||||
events:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesEventV2'
|
||||
type: array
|
||||
links:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesLink'
|
||||
type: array
|
||||
resource:
|
||||
additionalProperties:
|
||||
type: string
|
||||
type: object
|
||||
spanId:
|
||||
type: string
|
||||
statusCode:
|
||||
type: string
|
||||
statusMessage:
|
||||
type: string
|
||||
truncated:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTruncatedField'
|
||||
type: array
|
||||
required:
|
||||
- spanId
|
||||
- statusCode
|
||||
- statusMessage
|
||||
- attributes
|
||||
- resource
|
||||
- events
|
||||
- links
|
||||
- truncated
|
||||
type: object
|
||||
SpantypesSpanSummary:
|
||||
properties:
|
||||
childCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
durationNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
hasError:
|
||||
type: boolean
|
||||
kind:
|
||||
type: string
|
||||
level:
|
||||
minimum: 0
|
||||
type: integer
|
||||
name:
|
||||
type: string
|
||||
ownTimeNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parentSpanId:
|
||||
type: string
|
||||
responseStatusCode:
|
||||
type: string
|
||||
serviceName:
|
||||
type: string
|
||||
spanId:
|
||||
type: string
|
||||
startOffsetNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
subtreeSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- spanId
|
||||
- name
|
||||
- serviceName
|
||||
- kind
|
||||
- startOffsetNano
|
||||
- durationNano
|
||||
- ownTimeNano
|
||||
- hasError
|
||||
- level
|
||||
- childCount
|
||||
- subtreeSpanCount
|
||||
type: object
|
||||
SpantypesSpanTextMatch:
|
||||
properties:
|
||||
field:
|
||||
type: string
|
||||
snippet:
|
||||
type: string
|
||||
required:
|
||||
- field
|
||||
- snippet
|
||||
type: object
|
||||
SpantypesThreadSpan:
|
||||
properties:
|
||||
attributes:
|
||||
@@ -13229,6 +13343,23 @@ components:
|
||||
- formattedOutput
|
||||
- formatterWarnings
|
||||
type: object
|
||||
SpantypesTraceAIIdentity:
|
||||
properties:
|
||||
modelCallCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
tokens:
|
||||
$ref: '#/components/schemas/SpantypesTraceAITokens'
|
||||
toolCallCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
totalCost:
|
||||
type: number
|
||||
required:
|
||||
- modelCallCount
|
||||
- toolCallCount
|
||||
- tokens
|
||||
type: object
|
||||
SpantypesTraceAISummary:
|
||||
properties:
|
||||
tokens:
|
||||
@@ -13262,6 +13393,383 @@ components:
|
||||
- cacheWrite
|
||||
- reasoning
|
||||
type: object
|
||||
SpantypesTraceErrorItem:
|
||||
properties:
|
||||
errorAncestorCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
statusMessage:
|
||||
type: string
|
||||
required:
|
||||
- span
|
||||
- statusMessage
|
||||
- errorAncestorCount
|
||||
type: object
|
||||
SpantypesTraceErrorList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceErrorItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceExpensiveItem:
|
||||
properties:
|
||||
cost:
|
||||
type: number
|
||||
inputTokens:
|
||||
minimum: 0
|
||||
type: integer
|
||||
outputTokens:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- inputTokens
|
||||
- outputTokens
|
||||
type: object
|
||||
SpantypesTraceExpensiveList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceExpensiveItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceIdentity:
|
||||
properties:
|
||||
ai:
|
||||
$ref: '#/components/schemas/SpantypesTraceAIIdentity'
|
||||
durationNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
errorSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
hasMissingSpans:
|
||||
type: boolean
|
||||
root:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
rootCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
serviceCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
services:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceServiceStats'
|
||||
type: array
|
||||
spanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
startTimestampMillis:
|
||||
minimum: 0
|
||||
type: integer
|
||||
traceId:
|
||||
type: string
|
||||
required:
|
||||
- traceId
|
||||
- startTimestampMillis
|
||||
- durationNano
|
||||
- spanCount
|
||||
- errorSpanCount
|
||||
- rootCount
|
||||
- hasMissingSpans
|
||||
- serviceCount
|
||||
- services
|
||||
type: object
|
||||
SpantypesTraceLinkItem:
|
||||
properties:
|
||||
linkCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
links:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesLink'
|
||||
type: array
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- linkCount
|
||||
- links
|
||||
type: object
|
||||
SpantypesTraceLinkList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceLinkItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceOverview:
|
||||
properties:
|
||||
errors:
|
||||
$ref: '#/components/schemas/SpantypesTraceErrorList'
|
||||
expensive:
|
||||
$ref: '#/components/schemas/SpantypesTraceExpensiveList'
|
||||
identity:
|
||||
$ref: '#/components/schemas/SpantypesTraceIdentity'
|
||||
links:
|
||||
$ref: '#/components/schemas/SpantypesTraceLinkList'
|
||||
slowest:
|
||||
$ref: '#/components/schemas/SpantypesTraceSlowestList'
|
||||
tree:
|
||||
$ref: '#/components/schemas/SpantypesTraceTree'
|
||||
unmarkedFailures:
|
||||
$ref: '#/components/schemas/SpantypesTraceUnmarkedFailureList'
|
||||
type: object
|
||||
SpantypesTraceServiceStats:
|
||||
properties:
|
||||
errorSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
ownTimeNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
serviceName:
|
||||
type: string
|
||||
spanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- serviceName
|
||||
- spanCount
|
||||
- errorSpanCount
|
||||
- ownTimeNano
|
||||
type: object
|
||||
SpantypesTraceSlowestItem:
|
||||
properties:
|
||||
errorSpanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
ownTimeNano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
spanCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- span
|
||||
- spanCount
|
||||
- ownTimeNano
|
||||
- errorSpanCount
|
||||
type: object
|
||||
SpantypesTraceSlowestList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceSlowestItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceSpanDetail:
|
||||
properties:
|
||||
ancestorCount:
|
||||
minimum: 0
|
||||
type: integer
|
||||
ancestors:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
type: array
|
||||
descendants:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanLevel'
|
||||
type: array
|
||||
percentile:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanPercentile'
|
||||
record:
|
||||
$ref: '#/components/schemas/SpantypesSpanRecord'
|
||||
summary:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- summary
|
||||
- record
|
||||
- ancestors
|
||||
- ancestorCount
|
||||
- descendants
|
||||
type: object
|
||||
SpantypesTraceSpanLevel:
|
||||
properties:
|
||||
level:
|
||||
minimum: 0
|
||||
type: integer
|
||||
spans:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
type: array
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- level
|
||||
- total
|
||||
- spans
|
||||
type: object
|
||||
SpantypesTraceSpanList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanMatch'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
warnings:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTraceSpanMatch:
|
||||
properties:
|
||||
matches:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanTextMatch'
|
||||
type: array
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- matches
|
||||
type: object
|
||||
SpantypesTraceSpanPercentile:
|
||||
properties:
|
||||
p50Nano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
p90Nano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
p99Nano:
|
||||
minimum: 0
|
||||
type: integer
|
||||
position:
|
||||
format: double
|
||||
type: number
|
||||
windowMinutes:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- p50Nano
|
||||
- p90Nano
|
||||
- p99Nano
|
||||
- position
|
||||
- windowMinutes
|
||||
type: object
|
||||
SpantypesTraceTree:
|
||||
properties:
|
||||
complete:
|
||||
type: boolean
|
||||
records:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanRecord'
|
||||
type: array
|
||||
spans:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
type: array
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- complete
|
||||
- spans
|
||||
type: object
|
||||
SpantypesTraceUnmarkedFailureItem:
|
||||
properties:
|
||||
parent:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
reason:
|
||||
$ref: '#/components/schemas/SpantypesUnmarkedFailureReason'
|
||||
span:
|
||||
$ref: '#/components/schemas/SpantypesSpanSummary'
|
||||
required:
|
||||
- span
|
||||
- reason
|
||||
type: object
|
||||
SpantypesTraceUnmarkedFailureList:
|
||||
properties:
|
||||
items:
|
||||
items:
|
||||
$ref: '#/components/schemas/SpantypesTraceUnmarkedFailureItem'
|
||||
type: array
|
||||
nextCursor:
|
||||
type: string
|
||||
total:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- total
|
||||
- items
|
||||
type: object
|
||||
SpantypesTruncatedField:
|
||||
properties:
|
||||
field:
|
||||
type: string
|
||||
length:
|
||||
minimum: 0
|
||||
type: integer
|
||||
required:
|
||||
- field
|
||||
- length
|
||||
type: object
|
||||
SpantypesUnmarkedFailureReason:
|
||||
enum:
|
||||
- http_4xx
|
||||
- grpc_status
|
||||
- exception_event
|
||||
type: string
|
||||
SpantypesUpdatableSpanMapper:
|
||||
properties:
|
||||
config:
|
||||
@@ -19284,6 +19792,280 @@ paths:
|
||||
tags:
|
||||
- tracedetail
|
||||
x-signoz-stability: alpha
|
||||
/api/v1/traces/{traceID}/inspect:
|
||||
get:
|
||||
deprecated: false
|
||||
description: 'Returns the overview of a trace of any size: its identity with
|
||||
per-service totals, the spans where each error started, the failures not marked
|
||||
as errors, the slowest spans by own time, the most expensive gen_ai spans,
|
||||
the spans that link to other spans, and the top of the span tree. A small
|
||||
trace also returns each span''s full record. Every list carries its total
|
||||
and a nextCursor when more items follow.'
|
||||
operationId: InspectTrace
|
||||
parameters:
|
||||
- description: The nextCursor of one list of the overview; returns the next
|
||||
page of that list and nothing else.
|
||||
in: query
|
||||
name: cursor
|
||||
schema:
|
||||
description: The nextCursor of one list of the overview; returns the next
|
||||
page of that list and nothing else.
|
||||
type: string
|
||||
- in: path
|
||||
name: traceID
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/SpantypesTraceOverview'
|
||||
status:
|
||||
type: string
|
||||
required:
|
||||
- status
|
||||
- data
|
||||
type: object
|
||||
description: OK
|
||||
"400":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Bad Request
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"404":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Not Found
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- VIEWER
|
||||
- tokenizer:
|
||||
- VIEWER
|
||||
summary: Inspect a trace
|
||||
tags:
|
||||
- tracedetail
|
||||
x-signoz-stability: alpha
|
||||
/api/v1/traces/{traceID}/inspect/spans:
|
||||
get:
|
||||
deprecated: false
|
||||
description: Returns the spans of a trace that match a search, each with its
|
||||
direct parent and where the words were found. Without a search it returns
|
||||
every span in tree order, or with under one span and the spans below it.
|
||||
operationId: InspectTraceSpans
|
||||
parameters:
|
||||
- description: 'Returns the spans that match, earliest first: words, a filter
|
||||
expression, or both, such as worldpay AND has_error = true. Words match
|
||||
the span name, status message, attributes, resource attributes and events,
|
||||
ignoring case. Without it every span is returned, in tree order.'
|
||||
in: query
|
||||
name: search
|
||||
schema:
|
||||
description: 'Returns the spans that match, earliest first: words, a filter
|
||||
expression, or both, such as worldpay AND has_error = true. Words match
|
||||
the span name, status message, attributes, resource attributes and events,
|
||||
ignoring case. Without it every span is returned, in tree order.'
|
||||
type: string
|
||||
- description: A span ID; returns that span and every span below it. Cannot
|
||||
be combined with search.
|
||||
in: query
|
||||
name: under
|
||||
schema:
|
||||
description: A span ID; returns that span and every span below it. Cannot
|
||||
be combined with search.
|
||||
type: string
|
||||
- description: Page size, at most 1000; when not given, 20 with search and 400
|
||||
without. A page holds fewer spans when it would pass the response size limit.
|
||||
in: query
|
||||
name: limit
|
||||
schema:
|
||||
description: Page size, at most 1000; when not given, 20 with search and
|
||||
400 without. A page holds fewer spans when it would pass the response
|
||||
size limit.
|
||||
type: integer
|
||||
- description: The nextCursor of an earlier response to the same request; returns
|
||||
the next page.
|
||||
in: query
|
||||
name: cursor
|
||||
schema:
|
||||
description: The nextCursor of an earlier response to the same request;
|
||||
returns the next page.
|
||||
type: string
|
||||
- in: path
|
||||
name: traceID
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanList'
|
||||
status:
|
||||
type: string
|
||||
required:
|
||||
- status
|
||||
- data
|
||||
type: object
|
||||
description: OK
|
||||
"400":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Bad Request
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"404":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Not Found
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- VIEWER
|
||||
- tokenizer:
|
||||
- VIEWER
|
||||
summary: Search or list the spans of a trace
|
||||
tags:
|
||||
- tracedetail
|
||||
x-signoz-stability: alpha
|
||||
/api/v1/traces/{traceID}/inspect/spans/{spanID}:
|
||||
get:
|
||||
deprecated: false
|
||||
description: 'Returns one span in full: its summary and record, its nearest
|
||||
ancestors with the root, the spans below it level by level, and where its
|
||||
duration sits among recent spans of the same name and service. Long values
|
||||
are cut and listed in truncated; selectFields returns named attributes without
|
||||
the cut.'
|
||||
operationId: InspectTraceSpan
|
||||
parameters:
|
||||
- description: The number of levels of ancestors and of descendants returned,
|
||||
1 to 10; 2 when not given. The root span is always returned.
|
||||
in: query
|
||||
name: depth
|
||||
schema:
|
||||
description: The number of levels of ancestors and of descendants returned,
|
||||
1 to 10; 2 when not given. The root span is always returned.
|
||||
type: integer
|
||||
- description: Attribute names returned without the 500 character cut; the record
|
||||
then holds only these attributes.
|
||||
in: query
|
||||
name: selectFields
|
||||
schema:
|
||||
description: Attribute names returned without the 500 character cut; the
|
||||
record then holds only these attributes.
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
- in: path
|
||||
name: traceID
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- in: path
|
||||
name: spanID
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/SpantypesTraceSpanDetail'
|
||||
status:
|
||||
type: string
|
||||
required:
|
||||
- status
|
||||
- data
|
||||
type: object
|
||||
description: OK
|
||||
"400":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Bad Request
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"404":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Not Found
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- VIEWER
|
||||
- tokenizer:
|
||||
- VIEWER
|
||||
summary: Inspect a span of a trace
|
||||
tags:
|
||||
- tracedetail
|
||||
x-signoz-stability: alpha
|
||||
/api/v1/traces/{traceID}/summary:
|
||||
get:
|
||||
deprecated: false
|
||||
|
||||
@@ -13817,6 +13817,28 @@ export interface SpantypesEventDTO {
|
||||
timeUnixNano?: number;
|
||||
}
|
||||
|
||||
export type SpantypesEventV2DTOAttributes = { [key: string]: unknown };
|
||||
|
||||
export interface SpantypesEventV2DTO {
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
attributes?: SpantypesEventV2DTOAttributes;
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
isError?: boolean;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
name?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
timeUnixNano?: number;
|
||||
}
|
||||
|
||||
export enum SpantypesFieldContextDTO {
|
||||
attribute = 'attribute',
|
||||
resource = 'resource',
|
||||
@@ -14507,6 +14529,17 @@ export interface SpantypesGettableWaterfallTraceDTO {
|
||||
uncollapsedSpans?: string[] | null;
|
||||
}
|
||||
|
||||
export interface SpantypesLinkDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
spanId?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
traceId?: string;
|
||||
}
|
||||
|
||||
export interface SpantypesPostableFlamegraphDTO {
|
||||
/**
|
||||
* @type array
|
||||
@@ -14593,6 +14626,497 @@ export interface SpantypesPostableWaterfallDTO {
|
||||
uncollapsedSpans?: string[] | null;
|
||||
}
|
||||
|
||||
export type SpantypesSpanRecordDTOAttributes = { [key: string]: unknown };
|
||||
|
||||
export type SpantypesSpanRecordDTOResource = { [key: string]: string };
|
||||
|
||||
export interface SpantypesTruncatedFieldDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
field: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
length: number;
|
||||
}
|
||||
|
||||
export interface SpantypesSpanRecordDTO {
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
attributes: SpantypesSpanRecordDTOAttributes;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
events: SpantypesEventV2DTO[];
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
links: SpantypesLinkDTO[];
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
resource: SpantypesSpanRecordDTOResource;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
spanId: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
statusCode: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
statusMessage: string;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
truncated: SpantypesTruncatedFieldDTO[];
|
||||
}
|
||||
|
||||
export interface SpantypesSpanSummaryDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
childCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
durationNano: number;
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
hasError: boolean;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
kind: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
level: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ownTimeNano: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
parentSpanId?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
responseStatusCode?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
serviceName: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
spanId: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
startOffsetNano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
subtreeSpanCount: number;
|
||||
}
|
||||
|
||||
export interface SpantypesSpanTextMatchDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
field: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
snippet: string;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceAIIdentityDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
modelCallCount: number;
|
||||
tokens: SpantypesTraceAITokensDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
toolCallCount: number;
|
||||
/**
|
||||
* @type number
|
||||
*/
|
||||
totalCost?: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceErrorItemDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorAncestorCount: number;
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
statusMessage: string;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceErrorListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceErrorItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceExpensiveItemDTO {
|
||||
/**
|
||||
* @type number
|
||||
*/
|
||||
cost?: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
inputTokens: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
outputTokens: number;
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceExpensiveListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceExpensiveItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceServiceStatsDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorSpanCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ownTimeNano: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
serviceName: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
spanCount: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceIdentityDTO {
|
||||
ai?: SpantypesTraceAIIdentityDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
durationNano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorSpanCount: number;
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
hasMissingSpans: boolean;
|
||||
root?: SpantypesSpanSummaryDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
rootCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
serviceCount: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
services: SpantypesTraceServiceStatsDTO[];
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
spanCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
startTimestampMillis: number;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
traceId: string;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceLinkItemDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
linkCount: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
links: SpantypesLinkDTO[];
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceLinkListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceLinkItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSlowestItemDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
errorSpanCount: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ownTimeNano: number;
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
spanCount: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSlowestListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceSlowestItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceTreeDTO {
|
||||
/**
|
||||
* @type boolean
|
||||
*/
|
||||
complete: boolean;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
records?: SpantypesSpanRecordDTO[];
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
spans: SpantypesSpanSummaryDTO[];
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export enum SpantypesUnmarkedFailureReasonDTO {
|
||||
http_4xx = 'http_4xx',
|
||||
grpc_status = 'grpc_status',
|
||||
exception_event = 'exception_event',
|
||||
}
|
||||
export interface SpantypesTraceUnmarkedFailureItemDTO {
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
reason: SpantypesUnmarkedFailureReasonDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceUnmarkedFailureListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceUnmarkedFailureItemDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceOverviewDTO {
|
||||
errors?: SpantypesTraceErrorListDTO;
|
||||
expensive?: SpantypesTraceExpensiveListDTO;
|
||||
identity?: SpantypesTraceIdentityDTO;
|
||||
links?: SpantypesTraceLinkListDTO;
|
||||
slowest?: SpantypesTraceSlowestListDTO;
|
||||
tree?: SpantypesTraceTreeDTO;
|
||||
unmarkedFailures?: SpantypesTraceUnmarkedFailureListDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanLevelDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
level: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
spans: SpantypesSpanSummaryDTO[];
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanPercentileDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
p50Nano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
p90Nano: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
p99Nano: number;
|
||||
/**
|
||||
* @type number
|
||||
* @format double
|
||||
*/
|
||||
position: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
windowMinutes: number;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanDetailDTO {
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
ancestorCount: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
ancestors: SpantypesSpanSummaryDTO[];
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
descendants: SpantypesTraceSpanLevelDTO[];
|
||||
percentile?: SpantypesTraceSpanPercentileDTO;
|
||||
record: SpantypesSpanRecordDTO;
|
||||
summary: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanMatchDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
matches: SpantypesSpanTextMatchDTO[];
|
||||
parent?: SpantypesSpanSummaryDTO;
|
||||
span: SpantypesSpanSummaryDTO;
|
||||
}
|
||||
|
||||
export interface SpantypesTraceSpanListDTO {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
items: SpantypesTraceSpanMatchDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
nextCursor?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
*/
|
||||
total: number;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
warnings?: string[];
|
||||
}
|
||||
|
||||
export interface SpantypesUpdatableSpanMapperDTO {
|
||||
config?: SpantypesSpanMapperConfigDTO;
|
||||
/**
|
||||
@@ -16035,6 +16559,84 @@ export type GetTraceAggregations200 = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type InspectTracePathParameters = {
|
||||
traceID: string;
|
||||
};
|
||||
export type InspectTraceParams = {
|
||||
/**
|
||||
* @type string
|
||||
* @description The nextCursor of one list of the overview; returns the next page of that list and nothing else.
|
||||
*/
|
||||
cursor?: string;
|
||||
};
|
||||
|
||||
export type InspectTrace200 = {
|
||||
data: SpantypesTraceOverviewDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type InspectTraceSpansPathParameters = {
|
||||
traceID: string;
|
||||
};
|
||||
export type InspectTraceSpansParams = {
|
||||
/**
|
||||
* @type string
|
||||
* @description Returns the spans that match, earliest first: words, a filter expression, or both, such as worldpay AND has_error = true. Words match the span name, status message, attributes, resource attributes and events, ignoring case. Without it every span is returned, in tree order.
|
||||
*/
|
||||
search?: string;
|
||||
/**
|
||||
* @type string
|
||||
* @description A span ID; returns that span and every span below it. Cannot be combined with search.
|
||||
*/
|
||||
under?: string;
|
||||
/**
|
||||
* @type integer
|
||||
* @description Page size, at most 1000; when not given, 20 with search and 400 without. A page holds fewer spans when it would pass the response size limit.
|
||||
*/
|
||||
limit?: number;
|
||||
/**
|
||||
* @type string
|
||||
* @description The nextCursor of an earlier response to the same request; returns the next page.
|
||||
*/
|
||||
cursor?: string;
|
||||
};
|
||||
|
||||
export type InspectTraceSpans200 = {
|
||||
data: SpantypesTraceSpanListDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type InspectTraceSpanPathParameters = {
|
||||
traceID: string;
|
||||
spanID: string;
|
||||
};
|
||||
export type InspectTraceSpanParams = {
|
||||
/**
|
||||
* @type integer
|
||||
* @description The number of levels of ancestors and of descendants returned, 1 to 10; 2 when not given. The root span is always returned.
|
||||
*/
|
||||
depth?: number;
|
||||
/**
|
||||
* @type array
|
||||
* @description Attribute names returned without the 500 character cut; the record then holds only these attributes.
|
||||
*/
|
||||
selectFields?: string[];
|
||||
};
|
||||
|
||||
export type InspectTraceSpan200 = {
|
||||
data: SpantypesTraceSpanDetailDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type GetTraceSummaryPathParameters = {
|
||||
traceID: string;
|
||||
};
|
||||
|
||||
@@ -29,6 +29,15 @@ import type {
|
||||
GetTraceThreadPathParameters,
|
||||
GetWaterfallV4200,
|
||||
GetWaterfallV4PathParameters,
|
||||
InspectTrace200,
|
||||
InspectTraceParams,
|
||||
InspectTracePathParameters,
|
||||
InspectTraceSpan200,
|
||||
InspectTraceSpanParams,
|
||||
InspectTraceSpanPathParameters,
|
||||
InspectTraceSpans200,
|
||||
InspectTraceSpansParams,
|
||||
InspectTraceSpansPathParameters,
|
||||
RenderErrorResponseDTO,
|
||||
SpantypesPostableFlamegraphDTO,
|
||||
SpantypesPostableTraceAggregationsDTO,
|
||||
@@ -158,6 +167,352 @@ export const useGetTraceAggregations = <
|
||||
> => {
|
||||
return useMutation(getGetTraceAggregationsMutationOptions(options));
|
||||
};
|
||||
/**
|
||||
* Returns the overview of a trace of any size: its identity with per-service totals, the spans where each error started, the failures not marked as errors, the slowest spans by own time, the most expensive gen_ai spans, the spans that link to other spans, and the top of the span tree. A small trace also returns each span's full record. Every list carries its total and a nextCursor when more items follow.
|
||||
* @summary Inspect a trace
|
||||
*/
|
||||
export const inspectTrace = (
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
signal?: AbortSignal,
|
||||
) => {
|
||||
return GeneratedAPIInstance<InspectTrace200>({
|
||||
url: `/api/v1/traces/${traceID}/inspect`,
|
||||
method: 'GET',
|
||||
params,
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
export const getInspectTraceQueryKey = (
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
) => {
|
||||
return [
|
||||
`/api/v1/traces/${traceID}/inspect`,
|
||||
...(params ? [params] : []),
|
||||
] as const;
|
||||
};
|
||||
|
||||
export const getInspectTraceQueryOptions = <
|
||||
TData = Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
) => {
|
||||
const { query: queryOptions } = options ?? {};
|
||||
|
||||
const queryKey =
|
||||
queryOptions?.queryKey ?? getInspectTraceQueryKey({ traceID }, params);
|
||||
|
||||
const queryFn: QueryFunction<Awaited<ReturnType<typeof inspectTrace>>> = ({
|
||||
signal,
|
||||
}) => inspectTrace({ traceID }, params, signal);
|
||||
|
||||
return {
|
||||
queryKey,
|
||||
queryFn,
|
||||
enabled: traceID !== null && traceID !== undefined,
|
||||
...queryOptions,
|
||||
} as UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError,
|
||||
TData
|
||||
> & { queryKey: QueryKey };
|
||||
};
|
||||
|
||||
export type InspectTraceQueryResult = NonNullable<
|
||||
Awaited<ReturnType<typeof inspectTrace>>
|
||||
>;
|
||||
export type InspectTraceQueryError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Inspect a trace
|
||||
*/
|
||||
|
||||
export function useInspectTrace<
|
||||
TData = Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTrace>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
|
||||
const queryOptions = getInspectTraceQueryOptions({ traceID }, params, options);
|
||||
|
||||
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
|
||||
queryKey: QueryKey;
|
||||
};
|
||||
|
||||
return withQueryKey(query, queryOptions.queryKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* @summary Inspect a trace
|
||||
*/
|
||||
export const invalidateInspectTrace = async (
|
||||
queryClient: QueryClient,
|
||||
{ traceID }: InspectTracePathParameters,
|
||||
params?: InspectTraceParams,
|
||||
options?: InvalidateOptions,
|
||||
): Promise<QueryClient> => {
|
||||
await queryClient.invalidateQueries(
|
||||
{ queryKey: getInspectTraceQueryKey({ traceID }, params) },
|
||||
options,
|
||||
);
|
||||
|
||||
return queryClient;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the spans of a trace that match a search, each with its direct parent and where the words were found. Without a search it returns every span in tree order, or with under one span and the spans below it.
|
||||
* @summary Search or list the spans of a trace
|
||||
*/
|
||||
export const inspectTraceSpans = (
|
||||
{ traceID }: InspectTraceSpansPathParameters,
|
||||
params?: InspectTraceSpansParams,
|
||||
signal?: AbortSignal,
|
||||
) => {
|
||||
return GeneratedAPIInstance<InspectTraceSpans200>({
|
||||
url: `/api/v1/traces/${traceID}/inspect/spans`,
|
||||
method: 'GET',
|
||||
params,
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
export const getInspectTraceSpansQueryKey = (
|
||||
{ traceID }: InspectTraceSpansPathParameters,
|
||||
params?: InspectTraceSpansParams,
|
||||
) => {
|
||||
return [
|
||||
`/api/v1/traces/${traceID}/inspect/spans`,
|
||||
...(params ? [params] : []),
|
||||
] as const;
|
||||
};
|
||||
|
||||
export const getInspectTraceSpansQueryOptions = <
|
||||
TData = Awaited<ReturnType<typeof inspectTraceSpans>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID }: InspectTraceSpansPathParameters,
|
||||
params?: InspectTraceSpansParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTraceSpans>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
) => {
|
||||
const { query: queryOptions } = options ?? {};
|
||||
|
||||
const queryKey =
|
||||
queryOptions?.queryKey ?? getInspectTraceSpansQueryKey({ traceID }, params);
|
||||
|
||||
const queryFn: QueryFunction<
|
||||
Awaited<ReturnType<typeof inspectTraceSpans>>
|
||||
> = ({ signal }) => inspectTraceSpans({ traceID }, params, signal);
|
||||
|
||||
return {
|
||||
queryKey,
|
||||
queryFn,
|
||||
enabled: traceID !== null && traceID !== undefined,
|
||||
...queryOptions,
|
||||
} as UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTraceSpans>>,
|
||||
TError,
|
||||
TData
|
||||
> & { queryKey: QueryKey };
|
||||
};
|
||||
|
||||
export type InspectTraceSpansQueryResult = NonNullable<
|
||||
Awaited<ReturnType<typeof inspectTraceSpans>>
|
||||
>;
|
||||
export type InspectTraceSpansQueryError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Search or list the spans of a trace
|
||||
*/
|
||||
|
||||
export function useInspectTraceSpans<
|
||||
TData = Awaited<ReturnType<typeof inspectTraceSpans>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID }: InspectTraceSpansPathParameters,
|
||||
params?: InspectTraceSpansParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTraceSpans>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
|
||||
const queryOptions = getInspectTraceSpansQueryOptions(
|
||||
{ traceID },
|
||||
params,
|
||||
options,
|
||||
);
|
||||
|
||||
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
|
||||
queryKey: QueryKey;
|
||||
};
|
||||
|
||||
return withQueryKey(query, queryOptions.queryKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* @summary Search or list the spans of a trace
|
||||
*/
|
||||
export const invalidateInspectTraceSpans = async (
|
||||
queryClient: QueryClient,
|
||||
{ traceID }: InspectTraceSpansPathParameters,
|
||||
params?: InspectTraceSpansParams,
|
||||
options?: InvalidateOptions,
|
||||
): Promise<QueryClient> => {
|
||||
await queryClient.invalidateQueries(
|
||||
{ queryKey: getInspectTraceSpansQueryKey({ traceID }, params) },
|
||||
options,
|
||||
);
|
||||
|
||||
return queryClient;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns one span in full: its summary and record, its nearest ancestors with the root, the spans below it level by level, and where its duration sits among recent spans of the same name and service. Long values are cut and listed in truncated; selectFields returns named attributes without the cut.
|
||||
* @summary Inspect a span of a trace
|
||||
*/
|
||||
export const inspectTraceSpan = (
|
||||
{ traceID, spanID }: InspectTraceSpanPathParameters,
|
||||
params?: InspectTraceSpanParams,
|
||||
signal?: AbortSignal,
|
||||
) => {
|
||||
return GeneratedAPIInstance<InspectTraceSpan200>({
|
||||
url: `/api/v1/traces/${traceID}/inspect/spans/${spanID}`,
|
||||
method: 'GET',
|
||||
params,
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
export const getInspectTraceSpanQueryKey = (
|
||||
{ traceID, spanID }: InspectTraceSpanPathParameters,
|
||||
params?: InspectTraceSpanParams,
|
||||
) => {
|
||||
return [
|
||||
`/api/v1/traces/${traceID}/inspect/spans/${spanID}`,
|
||||
...(params ? [params] : []),
|
||||
] as const;
|
||||
};
|
||||
|
||||
export const getInspectTraceSpanQueryOptions = <
|
||||
TData = Awaited<ReturnType<typeof inspectTraceSpan>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID, spanID }: InspectTraceSpanPathParameters,
|
||||
params?: InspectTraceSpanParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTraceSpan>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
) => {
|
||||
const { query: queryOptions } = options ?? {};
|
||||
|
||||
const queryKey =
|
||||
queryOptions?.queryKey ??
|
||||
getInspectTraceSpanQueryKey({ traceID, spanID }, params);
|
||||
|
||||
const queryFn: QueryFunction<Awaited<ReturnType<typeof inspectTraceSpan>>> = ({
|
||||
signal,
|
||||
}) => inspectTraceSpan({ traceID, spanID }, params, signal);
|
||||
|
||||
return {
|
||||
queryKey,
|
||||
queryFn,
|
||||
enabled:
|
||||
traceID !== null &&
|
||||
traceID !== undefined &&
|
||||
spanID !== null &&
|
||||
spanID !== undefined,
|
||||
...queryOptions,
|
||||
} as UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTraceSpan>>,
|
||||
TError,
|
||||
TData
|
||||
> & { queryKey: QueryKey };
|
||||
};
|
||||
|
||||
export type InspectTraceSpanQueryResult = NonNullable<
|
||||
Awaited<ReturnType<typeof inspectTraceSpan>>
|
||||
>;
|
||||
export type InspectTraceSpanQueryError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Inspect a span of a trace
|
||||
*/
|
||||
|
||||
export function useInspectTraceSpan<
|
||||
TData = Awaited<ReturnType<typeof inspectTraceSpan>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(
|
||||
{ traceID, spanID }: InspectTraceSpanPathParameters,
|
||||
params?: InspectTraceSpanParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof inspectTraceSpan>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
|
||||
const queryOptions = getInspectTraceSpanQueryOptions(
|
||||
{ traceID, spanID },
|
||||
params,
|
||||
options,
|
||||
);
|
||||
|
||||
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
|
||||
queryKey: QueryKey;
|
||||
};
|
||||
|
||||
return withQueryKey(query, queryOptions.queryKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* @summary Inspect a span of a trace
|
||||
*/
|
||||
export const invalidateInspectTraceSpan = async (
|
||||
queryClient: QueryClient,
|
||||
{ traceID, spanID }: InspectTraceSpanPathParameters,
|
||||
params?: InspectTraceSpanParams,
|
||||
options?: InvalidateOptions,
|
||||
): Promise<QueryClient> => {
|
||||
await queryClient.invalidateQueries(
|
||||
{ queryKey: getInspectTraceSpanQueryKey({ traceID, spanID }, params) },
|
||||
options,
|
||||
);
|
||||
|
||||
return queryClient;
|
||||
};
|
||||
|
||||
/**
|
||||
* Returns the trace's time range, root span, span and error counts, and whether any spans are missing. AI traces also include token and cost totals.
|
||||
* @summary Get summary for a trace
|
||||
|
||||
@@ -102,5 +102,59 @@ func (provider *provider) addTraceDetailRoutes(router *mux.Router) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/traces/{traceID}/inspect", handler.New(
|
||||
provider.authzMiddleware.ViewAccess(provider.traceDetailHandler.InspectTrace),
|
||||
handler.OpenAPIDef{
|
||||
ID: "InspectTrace",
|
||||
Tags: []string{"tracedetail"},
|
||||
Summary: "Inspect a trace",
|
||||
Description: "Returns the overview of a trace of any size: its identity with per-service totals, the spans where each error started, the failures not marked as errors, the slowest spans by own time, the most expensive gen_ai spans, the spans that link to other spans, and the top of the span tree. A small trace also returns each span's full record. Every list carries its total and a nextCursor when more items follow.",
|
||||
RequestQuery: new(spantypes.InspectTraceParams),
|
||||
Response: new(spantypes.TraceOverview),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
|
||||
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
|
||||
},
|
||||
)).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/traces/{traceID}/inspect/spans", handler.New(
|
||||
provider.authzMiddleware.ViewAccess(provider.traceDetailHandler.InspectTraceSpans),
|
||||
handler.OpenAPIDef{
|
||||
ID: "InspectTraceSpans",
|
||||
Tags: []string{"tracedetail"},
|
||||
Summary: "Search or list the spans of a trace",
|
||||
Description: "Returns the spans of a trace that match a search, each with its direct parent and where the words were found. Without a search it returns every span in tree order, or with under one span and the spans below it.",
|
||||
RequestQuery: new(spantypes.InspectTraceSpansParams),
|
||||
Response: new(spantypes.TraceSpanList),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
|
||||
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
|
||||
},
|
||||
)).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/traces/{traceID}/inspect/spans/{spanID}", handler.New(
|
||||
provider.authzMiddleware.ViewAccess(provider.traceDetailHandler.InspectTraceSpan),
|
||||
handler.OpenAPIDef{
|
||||
ID: "InspectTraceSpan",
|
||||
Tags: []string{"tracedetail"},
|
||||
Summary: "Inspect a span of a trace",
|
||||
Description: "Returns one span in full: its summary and record, its nearest ancestors with the root, the spans below it level by level, and where its duration sits among recent spans of the same name and service. Long values are cut and listed in truncated; selectFields returns named attributes without the cut.",
|
||||
RequestQuery: new(spantypes.InspectTraceSpanParams),
|
||||
Response: new(spantypes.TraceSpanDetail),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
|
||||
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
|
||||
},
|
||||
)).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -127,3 +127,87 @@ func (h *handler) GetThread(rw http.ResponseWriter, r *http.Request) {
|
||||
|
||||
render.Success(rw, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (h *handler) InspectTrace(rw http.ResponseWriter, r *http.Request) {
|
||||
claims, err := authtypes.ClaimsFromContext(r.Context())
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
params := new(spantypes.InspectTraceParams)
|
||||
if err := binding.Query.BindQuery(r.URL.Query(), params); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
query, err := spantypes.NewTraceOverviewQuery(params)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.module.GetTraceOverview(r.Context(), valuer.MustNewUUID(claims.OrgID), mux.Vars(r)["traceID"], query)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (h *handler) InspectTraceSpan(rw http.ResponseWriter, r *http.Request) {
|
||||
claims, err := authtypes.ClaimsFromContext(r.Context())
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
params := new(spantypes.InspectTraceSpanParams)
|
||||
if err := binding.Query.BindQuery(r.URL.Query(), params); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
query, err := spantypes.NewTraceSpanQuery(mux.Vars(r)["spanID"], params)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.module.GetTraceSpan(r.Context(), valuer.MustNewUUID(claims.OrgID), mux.Vars(r)["traceID"], query)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, result)
|
||||
}
|
||||
|
||||
func (h *handler) InspectTraceSpans(rw http.ResponseWriter, r *http.Request) {
|
||||
claims, err := authtypes.ClaimsFromContext(r.Context())
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
params := new(spantypes.InspectTraceSpansParams)
|
||||
if err := binding.Query.BindQuery(r.URL.Query(), params); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
query, err := spantypes.NewTraceSpanListQuery(params)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.module.ListTraceSpans(r.Context(), valuer.MustNewUUID(claims.OrgID), mux.Vars(r)["traceID"], query)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, result)
|
||||
}
|
||||
|
||||
@@ -5,12 +5,22 @@ import (
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/modules/spanpercentile"
|
||||
"github.com/SigNoz/signoz/pkg/modules/tracedetail"
|
||||
"github.com/SigNoz/signoz/pkg/types/spanpercentiletypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/spantypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// The span percentile compares a span with those of the hour before it, and gives up after 2 seconds.
|
||||
const (
|
||||
spanPercentileWindow = time.Hour
|
||||
spanPercentileTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// formattedSpansLogThreshold is the number of formatted gen_ai spans in one waterfall response above which
|
||||
@@ -18,13 +28,14 @@ import (
|
||||
const formattedSpansLogThreshold = 200
|
||||
|
||||
type module struct {
|
||||
store spantypes.TraceStore
|
||||
settings factory.ScopedProviderSettings
|
||||
config tracedetail.Config
|
||||
metrics *moduleMetrics
|
||||
store spantypes.TraceStore
|
||||
spanPercentile spanpercentile.Module
|
||||
settings factory.ScopedProviderSettings
|
||||
config tracedetail.Config
|
||||
metrics *moduleMetrics
|
||||
}
|
||||
|
||||
func NewModule(traceStore spantypes.TraceStore, providerSettings factory.ProviderSettings, cfg tracedetail.Config) *module {
|
||||
func NewModule(traceStore spantypes.TraceStore, spanPercentile spanpercentile.Module, providerSettings factory.ProviderSettings, cfg tracedetail.Config) *module {
|
||||
scopedProviderSettings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/modules/tracedetail/impltracedetail")
|
||||
|
||||
metrics, err := newModuleMetrics(scopedProviderSettings.Meter())
|
||||
@@ -33,10 +44,11 @@ func NewModule(traceStore spantypes.TraceStore, providerSettings factory.Provide
|
||||
}
|
||||
|
||||
m := &module{
|
||||
config: cfg,
|
||||
store: traceStore,
|
||||
settings: scopedProviderSettings,
|
||||
metrics: metrics,
|
||||
config: cfg,
|
||||
store: traceStore,
|
||||
spanPercentile: spanPercentile,
|
||||
settings: scopedProviderSettings,
|
||||
metrics: metrics,
|
||||
}
|
||||
|
||||
m.metrics.waterfallSpanLimit.Record(context.Background(), int64(cfg.Waterfall.MaxLimitToSelectAllSpans), metric.WithAttributes(attrResponseType.String(attrResponseTypeWindowed)))
|
||||
@@ -292,3 +304,229 @@ func (m *module) getWindowedFlamegraph(ctx context.Context, traceID, selectedSpa
|
||||
enrichedSpans := flamegraphTrace.EnrichSelectedSpans(selectedSpans, fullSpans, selectFields)
|
||||
return spantypes.NewGettableFlamegraphTrace(enrichedSpans, start, end, true), nil
|
||||
}
|
||||
|
||||
// The tree and the span's own data are loaded side by side.
|
||||
func (m *module) GetTraceSpan(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.TraceSpanQuery) (*spantypes.TraceSpanDetail, error) {
|
||||
bounds, err := m.store.GetTraceBounds(ctx, traceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var (
|
||||
skeleton *spantypes.TraceSkeleton
|
||||
spans []spantypes.StorableSpan
|
||||
percentile *spantypes.TraceSpanPercentile
|
||||
)
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
skeleton, err = m.loadSkeleton(groupCtx, orgID, traceID, bounds, false, false)
|
||||
return err
|
||||
})
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
spans, err = m.store.GetTraceSpansByIDs(groupCtx, traceID, bounds.Start, bounds.End, []string{query.SpanID})
|
||||
// A call with selectFields reads more of a span that was already opened with its percentile.
|
||||
if err != nil || len(spans) == 0 || len(query.SelectFields) > 0 {
|
||||
return err
|
||||
}
|
||||
percentile = m.getSpanPercentile(groupCtx, orgID, &spans[0])
|
||||
return nil
|
||||
})
|
||||
if err := group.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
idx, ok := skeleton.Index(query.SpanID)
|
||||
if !ok || len(spans) == 0 {
|
||||
return nil, errors.NewNotFoundf(spantypes.ErrCodeInspectSpanNotFound, "span %s not found in trace %s", query.SpanID, traceID)
|
||||
}
|
||||
|
||||
detail := skeleton.SpanDetail(idx, query.Depth)
|
||||
detail.Record = spantypes.NewSpanRecord(&spans[0], query.SelectFields)
|
||||
detail.Percentile = percentile
|
||||
spantypes.FitSpan(detail)
|
||||
return detail, nil
|
||||
}
|
||||
|
||||
// ListTraceSpans returns a page of the spans matching a search, or without one of every span.
|
||||
func (m *module) ListTraceSpans(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.TraceSpanListQuery) (*spantypes.TraceSpanList, error) {
|
||||
bounds, err := m.store.GetTraceBounds(ctx, traceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if query.Search == "" {
|
||||
skeleton, err := m.loadSkeleton(ctx, orgID, traceID, bounds, false, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if query.Under == "" {
|
||||
return skeleton.SpanPage(nil, query.Offset, query.Limit), nil
|
||||
}
|
||||
root, ok := skeleton.Index(query.Under)
|
||||
if !ok {
|
||||
return nil, errors.NewNotFoundf(spantypes.ErrCodeInspectSpanNotFound, "span %s not found in trace %s", query.Under, traceID)
|
||||
}
|
||||
return skeleton.SpanPage(&root, query.Offset, query.Limit), nil
|
||||
}
|
||||
|
||||
// The tree gives each match its place; the search and the matched spans' own data load beside it.
|
||||
var (
|
||||
skeleton *spantypes.TraceSkeleton
|
||||
search *spantypes.TraceSpanSearch
|
||||
spans []spantypes.StorableSpan
|
||||
)
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
skeleton, err = m.loadSkeleton(groupCtx, orgID, traceID, bounds, false, false)
|
||||
return err
|
||||
})
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
search, err = m.store.SearchTraceSpans(groupCtx, orgID, traceID, bounds, &spantypes.SpansQuery{Search: query.Search, Limit: query.Limit, After: query.After})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
spans, err = m.store.GetTraceSpansByIDs(groupCtx, traceID, bounds.Start, bounds.End, search.SpanIDs())
|
||||
return err
|
||||
})
|
||||
if err := group.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return spantypes.NewTraceSpanList(skeleton, search, spans), nil
|
||||
}
|
||||
|
||||
// getSpanPercentile places the span among those of the same name and service that started in the
|
||||
// hour before it. It reads across traces, so it is cut short and left out when slow or failing.
|
||||
func (m *module) getSpanPercentile(ctx context.Context, orgID valuer.UUID, span *spantypes.StorableSpan) *spantypes.TraceSpanPercentile {
|
||||
if span.Name == "" || span.ServiceName == "" || span.DurationNano == 0 {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, spanPercentileTimeout)
|
||||
defer cancel()
|
||||
|
||||
startMillis := uint64(span.StartTime.UnixMilli())
|
||||
response, err := m.spanPercentile.GetSpanPercentile(ctx, orgID, &spanpercentiletypes.SpanPercentileRequest{
|
||||
DurationNano: int64(span.DurationNano),
|
||||
Name: span.Name,
|
||||
ServiceName: span.ServiceName,
|
||||
Start: startMillis - uint64(spanPercentileWindow.Milliseconds()),
|
||||
End: startMillis + 1,
|
||||
})
|
||||
if err != nil {
|
||||
if !errors.Ast(err, errors.TypeNotFound) {
|
||||
m.settings.Logger().WarnContext(ctx, "span percentile left out of trace inspect", errors.Attr(err), slog.String("span_name", span.Name), slog.String("service_name", span.ServiceName))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return &spantypes.TraceSpanPercentile{
|
||||
P50Nano: uint64(response.Percentiles.P50),
|
||||
P90Nano: uint64(response.Percentiles.P90),
|
||||
P99Nano: uint64(response.Percentiles.P99),
|
||||
Position: response.Position.Percentile,
|
||||
WindowMinutes: uint64(spanPercentileWindow.Minutes()),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *module) GetTraceOverview(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.TraceOverviewQuery) (*spantypes.TraceOverview, error) {
|
||||
bounds, err := m.store.GetTraceBounds(ctx, traceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// A page of one list loads only what that list is built from.
|
||||
withAISpans := query.List == "" || query.List == spantypes.OverviewListExpensive
|
||||
withLinkedSpans := query.List == "" || query.List == spantypes.OverviewListLinks
|
||||
skeleton, err := m.loadSkeleton(ctx, orgID, traceID, bounds, withAISpans, withLinkedSpans)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch query.List {
|
||||
case spantypes.OverviewListErrors:
|
||||
errorList, err := m.errorPage(ctx, traceID, bounds, skeleton, query.Offset)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &spantypes.TraceOverview{Errors: errorList}, nil
|
||||
case spantypes.OverviewListUnmarkedFailures:
|
||||
return &spantypes.TraceOverview{UnmarkedFailures: skeleton.UnmarkedFailurePage(query.Offset)}, nil
|
||||
case spantypes.OverviewListSlowest:
|
||||
return &spantypes.TraceOverview{Slowest: skeleton.SlowestPage(query.Offset)}, nil
|
||||
case spantypes.OverviewListExpensive:
|
||||
return &spantypes.TraceOverview{Expensive: skeleton.ExpensivePage(query.Offset)}, nil
|
||||
case spantypes.OverviewListLinks:
|
||||
return &spantypes.TraceOverview{Links: skeleton.LinkPage(query.Offset)}, nil
|
||||
}
|
||||
|
||||
errorList, err := m.errorPage(ctx, traceID, bounds, skeleton, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
overview := &spantypes.TraceOverview{
|
||||
Identity: skeleton.Identity(),
|
||||
Errors: errorList,
|
||||
UnmarkedFailures: skeleton.UnmarkedFailurePage(0),
|
||||
Slowest: skeleton.SlowestPage(0),
|
||||
Expensive: skeleton.ExpensivePage(0),
|
||||
Links: skeleton.LinkPage(0),
|
||||
Tree: skeleton.Tree(),
|
||||
}
|
||||
if spantypes.IsSmallTrace(skeleton.Len()) {
|
||||
spans, err := m.store.GetTraceSpans(ctx, traceID, bounds)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if records, ok := spantypes.SmallTraceRecords(spans); ok {
|
||||
overview.Tree.Records = records
|
||||
}
|
||||
}
|
||||
skeleton.FitOverview(overview)
|
||||
return overview, nil
|
||||
}
|
||||
|
||||
func (m *module) errorPage(ctx context.Context, traceID string, bounds *spantypes.TraceBounds, skeleton *spantypes.TraceSkeleton, offset int) (*spantypes.TraceErrorList, error) {
|
||||
errorList, spanIDs := skeleton.ErrorPage(offset)
|
||||
messages, err := m.store.GetSpanStatusMessages(ctx, traceID, bounds, spanIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
errorList.SetStatusMessages(messages)
|
||||
return errorList, nil
|
||||
}
|
||||
|
||||
// loadSkeleton builds the span tree of the trace, with the gen_ai usage and the span links when asked for.
|
||||
func (m *module) loadSkeleton(ctx context.Context, orgID valuer.UUID, traceID string, bounds *spantypes.TraceBounds, withAISpans, withLinkedSpans bool) (*spantypes.TraceSkeleton, error) {
|
||||
var (
|
||||
skeleton *spantypes.TraceSkeleton
|
||||
aiSpans []spantypes.AISpanUsage
|
||||
linkedSpans []spantypes.LinkedSpan
|
||||
)
|
||||
group, groupCtx := errgroup.WithContext(ctx)
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
skeleton, err = m.store.GetTraceSkeleton(groupCtx, traceID, bounds)
|
||||
return err
|
||||
})
|
||||
if withAISpans {
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
aiSpans, err = m.store.GetAISpanUsage(groupCtx, orgID, traceID, bounds)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if withLinkedSpans {
|
||||
group.Go(func() error {
|
||||
var err error
|
||||
linkedSpans, err = m.store.GetLinkedSpans(groupCtx, traceID, bounds)
|
||||
return err
|
||||
})
|
||||
}
|
||||
if err := group.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if skeleton.Len() == 0 {
|
||||
return nil, spantypes.ErrTraceNotFound
|
||||
}
|
||||
return skeleton.WithAISpans(aiSpans).WithLinkedSpans(linkedSpans), nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
@@ -24,6 +25,18 @@ import (
|
||||
|
||||
const colServiceName = `resource_string_service$$$$name` // $ gets escaped so $$$$ converts to $$.
|
||||
|
||||
// The collector marks an exception event with isError when it writes the span.
|
||||
const colHasExceptionEvent = `(NOT has_error AND arrayExists(e -> position(e, '"isError":true') > 0, events)) AS has_exception_event`
|
||||
|
||||
var genAIUsageValues = []struct{ key, alias string }{
|
||||
{aiobservabilitytypes.GenAIUsageInputTokens, "input_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageOutputTokens, "output_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheReadInputTokens, "cache_read_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, "cache_write_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageReasoningOutputTokens, "reasoning_tokens_value"},
|
||||
{aiobservabilitytypes.SignozGenAITotalCost, "total_cost_value"},
|
||||
}
|
||||
|
||||
// storableSpanColumns are the columns a waterfall span scans. A row's legacy maps are blanked when its
|
||||
// JSON holds the attributes.
|
||||
var storableSpanColumns = []string{
|
||||
@@ -57,6 +70,22 @@ type spanDurationRow struct {
|
||||
TotalNs uint64 `ch:"total_ns"`
|
||||
}
|
||||
|
||||
type spanStatusMessageRow struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
StatusMessage string `ch:"status_message"`
|
||||
}
|
||||
|
||||
type spanLinksRow struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
Links string `ch:"links"`
|
||||
}
|
||||
|
||||
type spanSearchRow struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
TimeUnixNano int64 `ch:"time_unix_nano"`
|
||||
Total uint64 `ch:"total"`
|
||||
}
|
||||
|
||||
type traceStore struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
@@ -172,20 +201,7 @@ func (s *traceStore) GetTraceStats(ctx context.Context, orgID valuer.UUID, trace
|
||||
|
||||
// genAISpanColumns returns the gen_ai columns aggregated per span, resolved across attribute evolutions.
|
||||
func (s *traceStore) genAISpanColumns(ctx context.Context, orgID valuer.UUID, bounds *spantypes.TraceBounds, sb *sqlbuilder.SelectBuilder) ([]string, error) {
|
||||
values := []struct{ key, alias string }{
|
||||
{aiobservabilitytypes.GenAIUsageInputTokens, "input_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageOutputTokens, "output_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheReadInputTokens, "cache_read_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, "cache_write_tokens_value"},
|
||||
{aiobservabilitytypes.GenAIUsageReasoningOutputTokens, "reasoning_tokens_value"},
|
||||
{aiobservabilitytypes.SignozGenAITotalCost, "total_cost_value"},
|
||||
}
|
||||
|
||||
names := slices.Clone(aiobservabilitytypes.GenAISpanGateKeys)
|
||||
for _, value := range values {
|
||||
names = append(names, value.key)
|
||||
}
|
||||
keys, err := s.genAIFieldKeys(ctx, orgID, names)
|
||||
keys, err := s.genAIFieldKeys(ctx, orgID, genAIUsageKeyNames())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -195,9 +211,26 @@ func (s *traceStore) genAISpanColumns(ctx context.Context, orgID valuer.UUID, bo
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
columns := []string{isGenAI + " AS is_gen_ai"}
|
||||
usage, err := s.genAIUsageColumns(ctx, q, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]string{isGenAI + " AS is_gen_ai"}, usage...), nil
|
||||
}
|
||||
|
||||
for _, value := range values {
|
||||
// genAIUsageKeyNames lists the gate keys and the usage keys a gen_ai query resolves.
|
||||
func genAIUsageKeyNames() []string {
|
||||
names := slices.Clone(aiobservabilitytypes.GenAISpanGateKeys)
|
||||
for _, value := range genAIUsageValues {
|
||||
names = append(names, value.key)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// genAIUsageColumns returns the token and cost columns of a span, resolved across attribute evolutions.
|
||||
func (s *traceStore) genAIUsageColumns(ctx context.Context, q qbtypes.QueryInfo, keys map[string][]*telemetrytypes.TelemetryFieldKey) ([]string, error) {
|
||||
columns := make([]string, 0, len(genAIUsageValues))
|
||||
for _, value := range genAIUsageValues {
|
||||
// lookup by number, the type metadata stores numeric attributes under; float64 is only the output cast
|
||||
// fixed gen_ai keys that many traces lack, so a not-found warning means nothing here
|
||||
expr, _, err := querybuilder.ResolveColumn(ctx, q, s.storage, attributeKey(value.key, telemetrytypes.FieldDataTypeNumber), telemetrytypes.FieldDataTypeFloat64, keys)
|
||||
@@ -523,3 +556,250 @@ func (s *traceStore) anyExistsCondition(ctx context.Context, q qbtypes.QueryInfo
|
||||
}
|
||||
return sb.Or(conds...), nil
|
||||
}
|
||||
|
||||
func (s *traceStore) GetTraceSkeleton(ctx context.Context, traceID string, bounds *spantypes.TraceBounds) (*spantypes.TraceSkeleton, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(
|
||||
"DISTINCT ON (span_id) span_id", "parent_span_id", "timestamp", "duration_nano", "has_error",
|
||||
"name", colServiceName, "kind_string", "response_status_code", colHasExceptionEvent,
|
||||
)
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
)
|
||||
sb.OrderByAsc("timestamp")
|
||||
sb.OrderByAsc("name")
|
||||
sb.OrderByAsc("span_id")
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
rows, err := s.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying trace skeleton")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
builder := spantypes.NewTraceSkeletonBuilder(traceID, int(bounds.NumSpans))
|
||||
var span spantypes.SkeletonSpan
|
||||
// Scanning into one reused list of fields, not ScanStruct, keeps the per-row cost to the strings.
|
||||
fields := []any{
|
||||
&span.SpanID, &span.ParentSpanID, &span.StartTime, &span.DurationNano, &span.HasError,
|
||||
&span.Name, &span.ServiceName, &span.KindString, &span.ResponseStatusCode, &span.HasExceptionEvent,
|
||||
}
|
||||
for rows.Next() {
|
||||
if err := rows.Scan(fields...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error scanning trace skeleton")
|
||||
}
|
||||
builder.Add(&span)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error reading trace skeleton")
|
||||
}
|
||||
return builder.Build(), nil
|
||||
}
|
||||
|
||||
func (s *traceStore) GetAISpanUsage(ctx context.Context, orgID valuer.UUID, traceID string, bounds *spantypes.TraceBounds) ([]spantypes.AISpanUsage, error) {
|
||||
keys, err := s.genAIFieldKeys(ctx, orgID, genAIUsageKeyNames())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
q := querybuilder.NewQueryInfo(ctx, orgID, s.flagger, telemetrytypes.SignalTraces, nil, uint64(bounds.Start.UnixNano()), uint64(bounds.End.UnixNano()))
|
||||
// Like the waterfall, read the JSON attributes wherever that column is registered for the trace's
|
||||
// time range and the legacy maps otherwise, without waiting for the use_trace_attributes_json flag.
|
||||
q.TraceAttrsJSONOn = true
|
||||
isGenAI, err := s.anyExistsCondition(ctx, q, aiobservabilitytypes.GenAISpanGateKeys, keys, sb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isModelCall, err := s.anyExistsCondition(ctx, q, []string{aiobservabilitytypes.GenAIRequestModel}, keys, sb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
isToolCall, err := s.anyExistsCondition(ctx, q, []string{aiobservabilitytypes.GenAIToolName}, keys, sb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
usage, err := s.genAIUsageColumns(ctx, q, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sb.Select("DISTINCT ON (span_id) span_id", isModelCall+" AS is_model_call", isToolCall+" AS is_tool_call")
|
||||
sb.SelectMore(usage...)
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
isGenAI,
|
||||
)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
spans := []spantypes.AISpanUsage{}
|
||||
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &spans, query, args...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying gen_ai span usage")
|
||||
}
|
||||
return spans, nil
|
||||
}
|
||||
|
||||
func (s *traceStore) GetLinkedSpans(ctx context.Context, traceID string, bounds *spantypes.TraceBounds) ([]spantypes.LinkedSpan, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("DISTINCT ON (span_id) span_id", "links")
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
// The collector writes a link as a FOLLOWS_FROM reference; the parent is a CHILD_OF one.
|
||||
"position(links, 'FOLLOWS_FROM') > 0",
|
||||
)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var rows []spanLinksRow
|
||||
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &rows, query, args...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying linked spans")
|
||||
}
|
||||
linkedSpans := make([]spantypes.LinkedSpan, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if linked, ok := spantypes.NewLinkedSpan(row.SpanID, row.Links); ok {
|
||||
linkedSpans = append(linkedSpans, linked)
|
||||
}
|
||||
}
|
||||
return linkedSpans, nil
|
||||
}
|
||||
|
||||
func (s *traceStore) GetSpanStatusMessages(ctx context.Context, traceID string, bounds *spantypes.TraceBounds, spanIDs []string) (map[string]string, error) {
|
||||
messages := make(map[string]string, len(spanIDs))
|
||||
if len(spanIDs) == 0 {
|
||||
return messages, nil
|
||||
}
|
||||
ids := make([]any, len(spanIDs))
|
||||
for i, id := range spanIDs {
|
||||
ids[i] = id
|
||||
}
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("DISTINCT ON (span_id) span_id", "status_message")
|
||||
sb.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
sb.Where(
|
||||
sb.E("trace_id", traceID),
|
||||
sb.In("span_id", ids...),
|
||||
sb.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
sb.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var rows []spanStatusMessageRow
|
||||
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &rows, query, args...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error querying span status messages")
|
||||
}
|
||||
for _, row := range rows {
|
||||
messages[row.SpanID] = row.StatusMessage
|
||||
}
|
||||
return messages, nil
|
||||
}
|
||||
|
||||
// SearchTraceSpans returns one page of the spans of a trace matching a search: plain words, a
|
||||
// filter expression, or both.
|
||||
func (s *traceStore) SearchTraceSpans(ctx context.Context, orgID valuer.UUID, traceID string, bounds *spantypes.TraceBounds, query *spantypes.SpansQuery) (*spantypes.TraceSpanSearch, error) {
|
||||
selectors := querybuilder.QueryStringToKeysSelectors(query.Search)
|
||||
for _, selector := range selectors {
|
||||
selector.Signal = telemetrytypes.SignalTraces
|
||||
selector.SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact
|
||||
}
|
||||
keys, _, err := s.metadataStore.GetKeysMulti(ctx, orgID, querybuilder.ExpandKeySelectorsForFamilies(ctx, orgID, s.flagger, selectors))
|
||||
if err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error fetching search field keys")
|
||||
}
|
||||
|
||||
search := &spantypes.TraceSpanSearch{Hits: []spantypes.ThreadCursor{}, Words: []string{}}
|
||||
matches := sqlbuilder.NewSelectBuilder()
|
||||
matches.Select("DISTINCT ON (span_id) span_id", "toUnixTimestamp64Nano(timestamp) AS time_unix_nano")
|
||||
matches.From(fmt.Sprintf("%s.%s", spantypes.TraceDB, spantypes.TraceTable))
|
||||
matches.Where(
|
||||
matches.E("trace_id", traceID),
|
||||
matches.GE("ts_bucket_start", bounds.Start.Unix()-1800),
|
||||
matches.LE("ts_bucket_start", bounds.End.Unix()),
|
||||
)
|
||||
q := querybuilder.NewQueryInfo(ctx, orgID, s.flagger, telemetrytypes.SignalTraces, nil, uint64(bounds.Start.UnixNano()), uint64(bounds.End.UnixNano()))
|
||||
// As in GetAISpanUsage: the JSON attributes are read without waiting for the feature flag.
|
||||
q.TraceAttrsJSONOn = true
|
||||
prepared, err := querybuilder.PrepareWhereClause(query.Search, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Query: q,
|
||||
Storage: s.storage,
|
||||
FieldKeys: keys,
|
||||
Builder: matches,
|
||||
FullTextCondition: func(sb *sqlbuilder.SelectBuilder, text string) string {
|
||||
search.Words = append(search.Words, text)
|
||||
return spanTextCondition(sb, text)
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !prepared.IsEmpty() {
|
||||
matches.AddWhereClause(prepared.WhereClause)
|
||||
}
|
||||
search.Warnings = prepared.Warnings
|
||||
|
||||
// The total counts every match, so the page cursor applies outside the counted set.
|
||||
counted := sqlbuilder.NewSelectBuilder()
|
||||
counted.Select("span_id", "time_unix_nano", "count() OVER () AS total")
|
||||
counted.From(counted.BuilderAs(matches, "matches"))
|
||||
|
||||
page := sqlbuilder.NewSelectBuilder()
|
||||
page.Select("span_id", "time_unix_nano", "total")
|
||||
page.From(page.BuilderAs(counted, "counted"))
|
||||
if query.After != nil {
|
||||
page.Where(page.GT("(time_unix_nano, span_id)", sqlbuilder.Tuple(int64(query.After.TimeUnixNano), query.After.SpanID)))
|
||||
}
|
||||
page.OrderByAsc("time_unix_nano")
|
||||
page.OrderByAsc("span_id")
|
||||
// One extra row tells whether another page follows.
|
||||
page.Limit(query.Limit + 1)
|
||||
stmt, args := page.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var rows []spanSearchRow
|
||||
if err := s.telemetryStore.ClickhouseDB().Select(ctx, &rows, stmt, args...); err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error searching trace spans")
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
search.Total = rows[0].Total
|
||||
}
|
||||
if len(rows) > query.Limit {
|
||||
rows = rows[:query.Limit]
|
||||
search.HasMore = true
|
||||
}
|
||||
for _, row := range rows {
|
||||
search.Hits = append(search.Hits, spantypes.ThreadCursor{TimeUnixNano: uint64(row.TimeUnixNano), SpanID: row.SpanID})
|
||||
}
|
||||
return search, nil
|
||||
}
|
||||
|
||||
// spanTextCondition matches text, ignoring case, in the span name, status message, attributes,
|
||||
// resource attributes and events. A span's JSON attributes are read when filled, the legacy maps otherwise.
|
||||
func spanTextCondition(sb *sqlbuilder.SelectBuilder, text string) string {
|
||||
needle := sb.Var(text)
|
||||
in := func(haystack string) string {
|
||||
return fmt.Sprintf("positionCaseInsensitiveUTF8(%s, %s) > 0", haystack, needle)
|
||||
}
|
||||
anyOf := func(array, element string) string {
|
||||
return fmt.Sprintf("arrayExists(x -> %s, %s)", in(element), array)
|
||||
}
|
||||
maps := []string{
|
||||
anyOf("mapKeys(attributes_string)", "x"), anyOf("mapValues(attributes_string)", "x"),
|
||||
anyOf("mapKeys(attributes_number)", "x"), anyOf("mapValues(attributes_number)", "toString(x)"),
|
||||
anyOf("mapKeys(attributes_bool)", "x"), anyOf("mapValues(attributes_bool)", "toString(x)"),
|
||||
}
|
||||
return "(" + strings.Join([]string{
|
||||
in("name"),
|
||||
in("status_message"),
|
||||
// The JSON column nests a dotted name, so names are matched as paths and values in the text.
|
||||
fmt.Sprintf("if(notEmpty(attributes), %s OR %s, %s)", anyOf("JSONAllPaths(attributes)", "x"), in("toJSONString(attributes)"), strings.Join(maps, " OR ")),
|
||||
anyOf("mapKeys(resources_string)", "x"), anyOf("mapValues(resources_string)", "x"),
|
||||
anyOf("events", "x"),
|
||||
}, " OR ") + ")"
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -253,3 +254,33 @@ func TestGetThreadSpans(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetLinkedSpans(t *testing.T) {
|
||||
columns := []cmock.ColumnType{{Name: "span_id", Type: "String"}, {Name: "links", Type: "String"}}
|
||||
rows := [][]any{
|
||||
{"consumer", `[{"traceId":"trace-abc123","spanId":"parent","refType":"CHILD_OF"},{"traceId":"producer-trace","spanId":"producer","refType":"FOLLOWS_FROM"}]`},
|
||||
{"parent-named-follows-from", `[{"traceId":"trace-abc123","spanId":"FOLLOWS_FROM","refType":"CHILD_OF"}]`},
|
||||
{"malformed", `not json`},
|
||||
}
|
||||
s := newTestStore(sqlmock.QueryMatcherEqual)
|
||||
s.Mock().ExpectSelect("SELECT DISTINCT ON (span_id) span_id, links FROM signoz_traces.distributed_signoz_index_v3 WHERE trace_id = ? AND ts_bucket_start >= ? AND ts_bucket_start <= ? AND position(links, 'FOLLOWS_FROM') > 0").
|
||||
WillReturnRows(cmock.NewRows(columns, rows))
|
||||
|
||||
linkedSpans, err := s.Store().GetLinkedSpans(context.Background(), testTraceID, testBounds)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []spantypes.LinkedSpan{{SpanID: "consumer", Links: []spantypes.Link{{TraceID: "producer-trace", SpanID: "producer"}}}}, linkedSpans)
|
||||
assert.NoError(t, s.Mock().ExpectationsWereMet())
|
||||
}
|
||||
|
||||
// The JSON attributes are read as soon as that column is registered, with no feature flag set.
|
||||
func TestGetAISpanUsageReadsJSONAttributesWithoutFlag(t *testing.T) {
|
||||
jsonInsideTrace := testStart.Add(500 * time.Second)
|
||||
s := newTestStoreWithMetadata(sqlmock.QueryMatcherRegexp, genAIMetadataStore(&jsonInsideTrace), nil)
|
||||
s.Mock().ExpectSelect("attributes\\.`gen_ai\\.request\\.model` IS NOT NULL").WillReturnRows(cmock.NewRows(nil, nil))
|
||||
|
||||
_, err := s.Store().GetAISpanUsage(context.Background(), valuer.GenerateUUID(), testTraceID, testBounds)
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NoError(t, s.Mock().ExpectationsWereMet())
|
||||
}
|
||||
|
||||
@@ -16,6 +16,9 @@ type Handler interface {
|
||||
GetTraceAggregations(http.ResponseWriter, *http.Request)
|
||||
GetFlamegraph(http.ResponseWriter, *http.Request)
|
||||
GetThread(http.ResponseWriter, *http.Request)
|
||||
InspectTrace(http.ResponseWriter, *http.Request)
|
||||
InspectTraceSpan(http.ResponseWriter, *http.Request)
|
||||
InspectTraceSpans(http.ResponseWriter, *http.Request)
|
||||
}
|
||||
|
||||
// Module defines the business logic for trace detail operations.
|
||||
@@ -25,4 +28,7 @@ type Module interface {
|
||||
GetTraceAggregations(ctx context.Context, traceID string, req *spantypes.PostableTraceAggregations) (*spantypes.GettableTraceAggregations, error)
|
||||
GetFlamegraph(ctx context.Context, traceID string, selectedSpanID string, selectFields []telemetrytypes.TelemetryFieldKey) (*spantypes.GettableFlamegraphTrace, error)
|
||||
GetThread(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.ThreadQuery) (*spantypes.GettableTraceThread, error)
|
||||
GetTraceOverview(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.TraceOverviewQuery) (*spantypes.TraceOverview, error)
|
||||
GetTraceSpan(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.TraceSpanQuery) (*spantypes.TraceSpanDetail, error)
|
||||
ListTraceSpans(ctx context.Context, orgID valuer.UUID, traceID string, query *spantypes.TraceSpanListQuery) (*spantypes.TraceSpanList, error)
|
||||
}
|
||||
|
||||
@@ -35,6 +35,7 @@ type filterExpressionVisitor struct {
|
||||
mainErrorURL string
|
||||
builder *sqlbuilder.SelectBuilder
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
fullTextCondition func(sb *sqlbuilder.SelectBuilder, text string) string
|
||||
skipResourceFilter bool
|
||||
skipFullTextFilter bool
|
||||
variables map[string]qbtypes.VariableItem
|
||||
@@ -48,12 +49,14 @@ type FilterExprVisitorOpts struct {
|
||||
Context context.Context
|
||||
// Query is the request's context with the query-path flags evaluated
|
||||
// one time. Storage answers the signal's part of every term.
|
||||
Query qbtypes.QueryInfo
|
||||
Storage qbtypes.Storage
|
||||
Logger *slog.Logger
|
||||
FieldKeys map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
Builder *sqlbuilder.SelectBuilder
|
||||
FullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
Query qbtypes.QueryInfo
|
||||
Storage qbtypes.Storage
|
||||
Logger *slog.Logger
|
||||
FieldKeys map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
Builder *sqlbuilder.SelectBuilder
|
||||
FullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
// FullTextCondition compiles a free-text term for a signal without a full-text column.
|
||||
FullTextCondition func(sb *sqlbuilder.SelectBuilder, text string) string
|
||||
SkipResourceFilter bool
|
||||
SkipFullTextFilter bool
|
||||
Variables map[string]qbtypes.VariableItem
|
||||
@@ -68,6 +71,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
|
||||
fieldKeys: opts.FieldKeys,
|
||||
builder: opts.Builder,
|
||||
fullTextColumn: opts.FullTextColumn,
|
||||
fullTextCondition: opts.FullTextCondition,
|
||||
skipResourceFilter: opts.SkipResourceFilter,
|
||||
skipFullTextFilter: opts.SkipFullTextFilter,
|
||||
variables: opts.Variables,
|
||||
@@ -330,7 +334,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
return SkipConditionLiteral
|
||||
}
|
||||
|
||||
if v.fullTextColumn == nil {
|
||||
if v.fullTextColumn == nil && v.fullTextCondition == nil {
|
||||
v.errors = append(v.errors, "full text search is not supported")
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
@@ -353,6 +357,9 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
}
|
||||
if v.fullTextColumn == nil {
|
||||
return v.fullTextCondition(v.builder, searchText)
|
||||
}
|
||||
conds, ok := v.compile(storageKey(v.fullTextColumn), qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText))
|
||||
if !ok {
|
||||
return ErrorConditionLiteral
|
||||
@@ -664,6 +671,9 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
|
||||
}
|
||||
|
||||
if v.fullTextColumn == nil {
|
||||
if v.fullTextCondition != nil {
|
||||
return v.fullTextCondition(v.builder, text)
|
||||
}
|
||||
v.errors = append(v.errors, "full text search is not supported")
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
sqlbuilder "github.com/huandu/go-sqlbuilder"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestPrepareWhereClause_EmptyVariableList ensures PrepareWhereClause errors when a variable has an empty list value.
|
||||
@@ -1461,6 +1462,50 @@ func TestVisitComparison_FullText(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestVisitComparison_FullTextCondition covers a caller without a full-text column that compiles
|
||||
// free-text terms itself, as the search inside one trace does.
|
||||
func TestVisitComparison_FullTextCondition(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
expr string
|
||||
expectedSQL string
|
||||
expectedTerms []string
|
||||
}{
|
||||
{name: "BareWord", expr: "hello", expectedSQL: "WHERE text_cond", expectedTerms: []string{"hello"}},
|
||||
{name: "QuotedPhrase", expr: "'hello world'", expectedSQL: "WHERE text_cond", expectedTerms: []string{"hello world"}},
|
||||
{name: "WordAndAttribute", expr: "hello AND a = 'a'", expectedSQL: "WHERE (text_cond AND a_cond)", expectedTerms: []string{"hello"}},
|
||||
{name: "NotWord", expr: "NOT hello", expectedSQL: "WHERE NOT (text_cond)", expectedTerms: []string{"hello"}},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
_, opts := visitComparisonOpts(t)
|
||||
opts.FullTextColumn = nil
|
||||
terms := []string{}
|
||||
opts.FullTextCondition = func(_ *sqlbuilder.SelectBuilder, text string) string {
|
||||
terms = append(terms, text)
|
||||
return "text_cond"
|
||||
}
|
||||
|
||||
result, err := PrepareWhereClause(testCase.expr, opts)
|
||||
|
||||
require.NoError(t, err)
|
||||
sql, _ := result.WhereClause.Build()
|
||||
assert.Equal(t, testCase.expectedSQL, sql)
|
||||
assert.Equal(t, testCase.expectedTerms, terms)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("NoColumnAndNoCondition_IsAnError", func(t *testing.T) {
|
||||
_, opts := visitComparisonOpts(t)
|
||||
opts.FullTextColumn = nil
|
||||
|
||||
_, err := PrepareWhereClause("hello", opts)
|
||||
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
// TestVisitComparison_AllVariable covers the __all__ dynamic variable.
|
||||
// IN/NOT IN $service where service="__all__" resolves to TrueConditionLiteral in both
|
||||
// RSB and SB, short-circuiting OR and being stripped from AND.
|
||||
|
||||
@@ -135,6 +135,7 @@ func NewModules(
|
||||
userSetter := impluser.NewSetter(impluser.NewStore(sqlstore, providerSettings), tokenizer, emailing, providerSettings, orgSetter, authz, analytics, config.User, userRoleStore, userGetter, onDeleteUser)
|
||||
ruleStore := sqlrulestore.NewRuleStore(sqlstore, queryParser, providerSettings)
|
||||
authDomainModule := implauthdomain.NewModule(implauthdomain.NewStore(sqlstore), authNs, authz)
|
||||
spanPercentile := implspanpercentile.NewModule(querier, providerSettings)
|
||||
|
||||
return Modules{
|
||||
OrgGetter: orgGetter,
|
||||
@@ -151,7 +152,7 @@ func NewModules(
|
||||
RawDataExport: implrawdataexport.NewModule(querier),
|
||||
AuthDomain: authDomainModule,
|
||||
Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, authDomainModule, tokenizer, orgGetter, authz, config.Global),
|
||||
SpanPercentile: implspanpercentile.NewModule(querier, providerSettings),
|
||||
SpanPercentile: spanPercentile,
|
||||
Services: implservices.NewModule(querier, telemetryStore),
|
||||
MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, fl, providerSettings, config.MetricsExplorer),
|
||||
MetricReductionRule: metricReductionRule,
|
||||
@@ -162,7 +163,7 @@ func NewModules(
|
||||
LogsPipeline: impllogspipeline.NewModule(sqlstore),
|
||||
RuleStateHistory: implrulestatehistory.NewModule(implrulestatehistory.NewStore(telemetryStore, telemetryMetadataStore, providerSettings.Logger), ruleStore, providerSettings.Logger),
|
||||
CloudIntegration: cloudIntegrationModule,
|
||||
TraceDetail: impltracedetail.NewModule(impltracedetail.NewTraceStore(telemetryStore, telemetryMetadataStore, schemaMetadataStore, fl), providerSettings, config.TraceDetail),
|
||||
TraceDetail: impltracedetail.NewModule(impltracedetail.NewTraceStore(telemetryStore, telemetryMetadataStore, schemaMetadataStore, fl), spanPercentile, providerSettings, config.TraceDetail),
|
||||
SpanMapper: spanMapper,
|
||||
LLMPricingRule: impllmpricingrule.NewModule(impllmpricingrule.NewStore(sqlstore), querier, telemetryMetadataStore, providerSettings),
|
||||
Tag: tagModule,
|
||||
|
||||
@@ -40,6 +40,13 @@ type TraceStore interface {
|
||||
GetThreadSpans(ctx context.Context, orgID valuer.UUID, traceID string, bounds *TraceBounds, page ThreadPage) ([]StorableSpan, error)
|
||||
GetThreadCursor(ctx context.Context, traceID string, bounds *TraceBounds, spanID string) (*ThreadCursor, error)
|
||||
|
||||
GetTraceSkeleton(ctx context.Context, traceID string, bounds *TraceBounds) (*TraceSkeleton, error)
|
||||
GetAISpanUsage(ctx context.Context, orgID valuer.UUID, traceID string, bounds *TraceBounds) ([]AISpanUsage, error)
|
||||
// GetLinkedSpans returns the spans of the trace that link to other spans.
|
||||
GetLinkedSpans(ctx context.Context, traceID string, bounds *TraceBounds) ([]LinkedSpan, error)
|
||||
GetSpanStatusMessages(ctx context.Context, traceID string, bounds *TraceBounds, spanIDs []string) (map[string]string, error)
|
||||
SearchTraceSpans(ctx context.Context, orgID valuer.UUID, traceID string, bounds *TraceBounds, query *SpansQuery) (*TraceSpanSearch, error)
|
||||
|
||||
GetSpanCountByField(ctx context.Context, traceID string, bounds *TraceBounds, fieldKey telemetrytypes.TelemetryFieldKey) (map[string]uint64, error)
|
||||
GetSpanDurationByField(ctx context.Context, traceID string, bounds *TraceBounds, fieldKey telemetrytypes.TelemetryFieldKey) (map[string]uint64, error)
|
||||
}
|
||||
|
||||
419
pkg/types/spantypes/trace_inspect.go
Normal file
419
pkg/types/spantypes/trace_inspect.go
Normal file
@@ -0,0 +1,419 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"math"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
const (
|
||||
inspectListSize = 10
|
||||
inspectServiceLimit = 20
|
||||
inspectLinkTargets = 5
|
||||
inspectLevelSize = 20
|
||||
inspectDefaultDepth = 2
|
||||
inspectMaxDepth = 10
|
||||
// Page sizes of a search and of the list of every span.
|
||||
inspectSearchLimit = 20
|
||||
inspectAllSpansLimit = 400
|
||||
inspectMaxLimit = 1000
|
||||
// inspectNameChars cuts a span name in a summary; a statement used as a name can run to kilobytes.
|
||||
inspectNameChars = 200
|
||||
// inspectTreeSpans is the number of spans the tree shows, and the largest trace returned with its records.
|
||||
inspectTreeSpans = 50
|
||||
inspectSmallTraceChars = 20_000
|
||||
// inspectMaxResponseChars keeps a response under 50,000 characters with its envelope.
|
||||
inspectMaxResponseChars = 48_000
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCodeInspectInvalidCursor = errors.MustNewCode("trace_inspect_invalid_cursor")
|
||||
ErrCodeInspectInvalidDepth = errors.MustNewCode("trace_inspect_invalid_depth")
|
||||
ErrCodeInspectInvalidLimit = errors.MustNewCode("trace_inspect_invalid_limit")
|
||||
ErrCodeInspectInvalidParams = errors.MustNewCode("trace_inspect_invalid_params")
|
||||
ErrCodeInspectSpanNotFound = errors.MustNewCode("trace_inspect_span_not_found")
|
||||
)
|
||||
|
||||
// The lists of the overview, as a cursor names them.
|
||||
const (
|
||||
OverviewListErrors = "errors"
|
||||
OverviewListUnmarkedFailures = "unmarkedFailures"
|
||||
OverviewListSlowest = "slowest"
|
||||
OverviewListExpensive = "expensive"
|
||||
OverviewListLinks = "links"
|
||||
)
|
||||
|
||||
var overviewLists = []string{OverviewListErrors, OverviewListUnmarkedFailures, OverviewListSlowest, OverviewListExpensive, OverviewListLinks}
|
||||
|
||||
var (
|
||||
UnmarkedFailureHTTP4xx = UnmarkedFailureReason{valuer.NewString("http_4xx")}
|
||||
UnmarkedFailureGRPCStatus = UnmarkedFailureReason{valuer.NewString("grpc_status")}
|
||||
UnmarkedFailureExceptionEvent = UnmarkedFailureReason{valuer.NewString("exception_event")}
|
||||
)
|
||||
|
||||
// UnmarkedFailureReason says why a span without the error flag is listed as a failure.
|
||||
type UnmarkedFailureReason struct{ valuer.String }
|
||||
|
||||
func (UnmarkedFailureReason) Enum() []any {
|
||||
return []any{UnmarkedFailureHTTP4xx, UnmarkedFailureGRPCStatus, UnmarkedFailureExceptionEvent}
|
||||
}
|
||||
|
||||
type InspectTraceParams struct {
|
||||
Cursor string `query:"cursor" description:"The nextCursor of one list of the overview; returns the next page of that list and nothing else."`
|
||||
}
|
||||
|
||||
type InspectTraceSpanParams struct {
|
||||
Depth int `query:"depth" description:"The number of levels of ancestors and of descendants returned, 1 to 10; 2 when not given. The root span is always returned."`
|
||||
SelectFields []string `query:"selectFields" description:"Attribute names returned without the 500 character cut; the record then holds only these attributes."`
|
||||
}
|
||||
|
||||
type InspectTraceSpansParams struct {
|
||||
Search string `query:"search" description:"Returns the spans that match, earliest first: words, a filter expression, or both, such as worldpay AND has_error = true. Words match the span name, status message, attributes, resource attributes and events, ignoring case. Without it every span is returned, in tree order."`
|
||||
Under string `query:"under" description:"A span ID; returns that span and every span below it. Cannot be combined with search."`
|
||||
Limit int `query:"limit" description:"Page size, at most 1000; when not given, 20 with search and 400 without. A page holds fewer spans when it would pass the response size limit."`
|
||||
Cursor string `query:"cursor" description:"The nextCursor of an earlier response to the same request; returns the next page."`
|
||||
}
|
||||
|
||||
// SpanSummary is the short form of a span: enough to place it in the trace and decide whether to open it.
|
||||
type SpanSummary struct {
|
||||
SpanID string `json:"spanId" required:"true"`
|
||||
ParentSpanID string `json:"parentSpanId,omitempty"`
|
||||
Name string `json:"name" required:"true"`
|
||||
ServiceName string `json:"serviceName" required:"true"`
|
||||
Kind string `json:"kind" required:"true"`
|
||||
// StartOffsetNano is the time from the start of the trace to the start of the span.
|
||||
StartOffsetNano uint64 `json:"startOffsetNano" required:"true"`
|
||||
DurationNano uint64 `json:"durationNano" required:"true"`
|
||||
// OwnTimeNano is the duration minus the time the span's children cover.
|
||||
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
|
||||
ResponseStatusCode string `json:"responseStatusCode,omitempty"`
|
||||
HasError bool `json:"hasError" required:"true"`
|
||||
Level uint32 `json:"level" required:"true"`
|
||||
ChildCount uint32 `json:"childCount" required:"true"`
|
||||
// SubtreeSpanCount is the number of spans below this one.
|
||||
SubtreeSpanCount uint32 `json:"subtreeSpanCount" required:"true"`
|
||||
}
|
||||
|
||||
type TraceServiceStats struct {
|
||||
ServiceName string `json:"serviceName" required:"true"`
|
||||
SpanCount uint64 `json:"spanCount" required:"true"`
|
||||
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
|
||||
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
|
||||
}
|
||||
|
||||
type TraceAIIdentity struct {
|
||||
ModelCallCount uint64 `json:"modelCallCount" required:"true"`
|
||||
ToolCallCount uint64 `json:"toolCallCount" required:"true"`
|
||||
Tokens TraceAITokens `json:"tokens" required:"true"`
|
||||
// TotalCost is omitted when no span carries a cost attribute.
|
||||
TotalCost *float64 `json:"totalCost,omitempty" nullable:"false"`
|
||||
}
|
||||
|
||||
type TraceIdentity struct {
|
||||
TraceID string `json:"traceId" required:"true"`
|
||||
StartTimestampMillis uint64 `json:"startTimestampMillis" required:"true"`
|
||||
DurationNano uint64 `json:"durationNano" required:"true"`
|
||||
// Root is the earliest span without a parent; omitted when every span's parent is missing.
|
||||
Root *SpanSummary `json:"root,omitempty"`
|
||||
SpanCount uint64 `json:"spanCount" required:"true"`
|
||||
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
|
||||
RootCount uint64 `json:"rootCount" required:"true"`
|
||||
HasMissingSpans bool `json:"hasMissingSpans" required:"true"`
|
||||
ServiceCount uint64 `json:"serviceCount" required:"true"`
|
||||
Services []TraceServiceStats `json:"services" required:"true" nullable:"false"`
|
||||
AI *TraceAIIdentity `json:"ai,omitempty"`
|
||||
}
|
||||
|
||||
// Every list has a total, items and a nextCursor when more items follow. Every item starts with
|
||||
// its span and that span's direct parent, omitted for a root or when the parent span is missing.
|
||||
type TraceErrorItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
StatusMessage string `json:"statusMessage" required:"true"`
|
||||
// ErrorAncestorCount is the number of consecutive ancestors, from the parent up, that carry the error flag.
|
||||
ErrorAncestorCount uint32 `json:"errorAncestorCount" required:"true"`
|
||||
}
|
||||
|
||||
type TraceErrorList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceErrorItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceUnmarkedFailureItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
Reason UnmarkedFailureReason `json:"reason" required:"true"`
|
||||
}
|
||||
|
||||
type TraceUnmarkedFailureList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceUnmarkedFailureItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
// TraceSlowestItem is one span, or the same-name spans under one parent taken together.
|
||||
type TraceSlowestItem struct {
|
||||
// Span is the longest span of the group.
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
SpanCount uint64 `json:"spanCount" required:"true"`
|
||||
OwnTimeNano uint64 `json:"ownTimeNano" required:"true"`
|
||||
ErrorSpanCount uint64 `json:"errorSpanCount" required:"true"`
|
||||
}
|
||||
|
||||
type TraceSlowestList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceSlowestItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceExpensiveItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
Cost *float64 `json:"cost,omitempty" nullable:"false"`
|
||||
InputTokens uint64 `json:"inputTokens" required:"true"`
|
||||
OutputTokens uint64 `json:"outputTokens" required:"true"`
|
||||
}
|
||||
|
||||
type TraceExpensiveList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceExpensiveItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceLinkItem struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
// LinkCount is the number of spans this span links to; Links holds the first of them.
|
||||
LinkCount uint64 `json:"linkCount" required:"true"`
|
||||
Links []Link `json:"links" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceLinkList lists the spans of this trace that link to other spans, in this or another trace.
|
||||
type TraceLinkList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceLinkItem `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
}
|
||||
|
||||
type TraceTree struct {
|
||||
// Total is the number of spans in the trace; Spans holds every one of them when Complete is set.
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Complete bool `json:"complete" required:"true"`
|
||||
// Spans is in tree order: a span is followed by the spans below it. A span's childCount and
|
||||
// subtreeSpanCount say how much of the trace sits under it.
|
||||
Spans []SpanSummary `json:"spans" required:"true" nullable:"false"`
|
||||
// Records holds the full record of every span of a small trace.
|
||||
Records []SpanRecord `json:"records,omitempty" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceOverview holds every part on a first call and one list on a call with a cursor.
|
||||
type TraceOverview struct {
|
||||
Identity *TraceIdentity `json:"identity,omitempty"`
|
||||
Errors *TraceErrorList `json:"errors,omitempty"`
|
||||
UnmarkedFailures *TraceUnmarkedFailureList `json:"unmarkedFailures,omitempty"`
|
||||
Slowest *TraceSlowestList `json:"slowest,omitempty"`
|
||||
// Expensive is present when the trace has gen_ai spans.
|
||||
Expensive *TraceExpensiveList `json:"expensive,omitempty"`
|
||||
Links *TraceLinkList `json:"links,omitempty"`
|
||||
Tree *TraceTree `json:"tree,omitempty"`
|
||||
}
|
||||
|
||||
type TraceSpanLevel struct {
|
||||
// Level counts down from the span: 1 holds its children.
|
||||
Level uint32 `json:"level" required:"true"`
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Spans []SpanSummary `json:"spans" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceSpanPercentile places a span's duration among the spans of the same name and service that
|
||||
// started in the window before it.
|
||||
type TraceSpanPercentile struct {
|
||||
P50Nano uint64 `json:"p50Nano" required:"true"`
|
||||
P90Nano uint64 `json:"p90Nano" required:"true"`
|
||||
P99Nano uint64 `json:"p99Nano" required:"true"`
|
||||
// Position is the share of those spans, 0 to 100, that were no slower than this one.
|
||||
Position float64 `json:"position" required:"true"`
|
||||
WindowMinutes uint64 `json:"windowMinutes" required:"true"`
|
||||
}
|
||||
|
||||
type TraceSpanDetail struct {
|
||||
Summary SpanSummary `json:"summary" required:"true"`
|
||||
Record SpanRecord `json:"record" required:"true"`
|
||||
// Percentile is omitted when it could not be computed in time.
|
||||
Percentile *TraceSpanPercentile `json:"percentile,omitempty"`
|
||||
// Ancestors holds the nearest ancestors, the parent first, then the root when it is further up.
|
||||
Ancestors []SpanSummary `json:"ancestors" required:"true" nullable:"false"`
|
||||
AncestorCount uint64 `json:"ancestorCount" required:"true"`
|
||||
Descendants []TraceSpanLevel `json:"descendants" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
type SpanTextMatch struct {
|
||||
Field string `json:"field" required:"true"`
|
||||
Snippet string `json:"snippet" required:"true"`
|
||||
}
|
||||
|
||||
type TraceSpanMatch struct {
|
||||
Span SpanSummary `json:"span" required:"true"`
|
||||
Parent *SpanSummary `json:"parent,omitempty"`
|
||||
// Matches says where the searched words were found; empty for a filter without words.
|
||||
Matches []SpanTextMatch `json:"matches" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceSpanList holds the spans matching a search, or every span when there is none.
|
||||
type TraceSpanList struct {
|
||||
Total uint64 `json:"total" required:"true"`
|
||||
Items []TraceSpanMatch `json:"items" required:"true" nullable:"false"`
|
||||
NextCursor string `json:"nextCursor,omitempty"`
|
||||
// Warnings says why a search may have matched less than expected, such as a key that is not known.
|
||||
Warnings []string `json:"warnings,omitempty" nullable:"false"`
|
||||
}
|
||||
|
||||
// TraceOverviewQuery is a validated overview request.
|
||||
type TraceOverviewQuery struct {
|
||||
// List and Offset page one list of the overview; List is empty on a first call.
|
||||
List string
|
||||
Offset int
|
||||
}
|
||||
|
||||
// TraceSpanQuery is a validated request for one span.
|
||||
type TraceSpanQuery struct {
|
||||
SpanID string
|
||||
Depth int
|
||||
SelectFields []string
|
||||
}
|
||||
|
||||
// TraceSpanListQuery is a validated request for the spans of a trace: those matching Search, or
|
||||
// without one every span, or Under and the spans below it.
|
||||
type TraceSpanListQuery struct {
|
||||
Search string
|
||||
Under string
|
||||
Limit int
|
||||
// A search is paged by After, the list of every span by Offset.
|
||||
Offset int
|
||||
After *ThreadCursor
|
||||
}
|
||||
|
||||
// SpansQuery is one page of a search inside a trace.
|
||||
type SpansQuery struct {
|
||||
Search string
|
||||
Limit int
|
||||
After *ThreadCursor
|
||||
}
|
||||
|
||||
// TraceSpanSearch is one page of the spans matching a search, in start time order.
|
||||
type TraceSpanSearch struct {
|
||||
Hits []ThreadCursor
|
||||
Total uint64
|
||||
// Words holds the free-text terms of the search.
|
||||
Words []string
|
||||
Warnings []string
|
||||
HasMore bool
|
||||
}
|
||||
|
||||
func NewTraceOverviewQuery(params *InspectTraceParams) (*TraceOverviewQuery, error) {
|
||||
query := new(TraceOverviewQuery)
|
||||
if params.Cursor == "" {
|
||||
return query, nil
|
||||
}
|
||||
|
||||
cursor := new(overviewCursor)
|
||||
if err := decodeCursor(params.Cursor, cursor); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !slices.Contains(overviewLists, cursor.List) || cursor.Offset <= 0 || cursor.Offset > math.MaxInt32 {
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidCursor, "invalid cursor")
|
||||
}
|
||||
query.List, query.Offset = cursor.List, cursor.Offset
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func NewTraceSpanQuery(spanID string, params *InspectTraceSpanParams) (*TraceSpanQuery, error) {
|
||||
// A blank name selects nothing; a selection of only blanks is no selection.
|
||||
selectFields := slices.DeleteFunc(slices.Clone(params.SelectFields), func(field string) bool { return strings.TrimSpace(field) == "" })
|
||||
query := &TraceSpanQuery{SpanID: spanID, Depth: inspectDefaultDepth, SelectFields: selectFields}
|
||||
switch {
|
||||
case params.Depth < 0 || params.Depth > inspectMaxDepth:
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidDepth, "depth must be between 1 and %d, got %d", inspectMaxDepth, params.Depth)
|
||||
case params.Depth != 0:
|
||||
query.Depth = params.Depth
|
||||
}
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func NewTraceSpanListQuery(params *InspectTraceSpansParams) (*TraceSpanListQuery, error) {
|
||||
query := &TraceSpanListQuery{Search: strings.TrimSpace(params.Search), Under: params.Under, Limit: params.Limit}
|
||||
switch {
|
||||
case query.Search != "" && query.Under != "":
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidParams, "search and under cannot be combined")
|
||||
case params.Limit < 0 || params.Limit > inspectMaxLimit:
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidLimit, "limit must be between 1 and %d, got %d", inspectMaxLimit, params.Limit)
|
||||
case params.Limit == 0 && query.Search == "":
|
||||
query.Limit = inspectAllSpansLimit
|
||||
case params.Limit == 0:
|
||||
query.Limit = inspectSearchLimit
|
||||
}
|
||||
if params.Cursor == "" {
|
||||
return query, nil
|
||||
}
|
||||
|
||||
cursor := new(spanListCursor)
|
||||
if err := decodeCursor(params.Cursor, cursor); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch {
|
||||
case query.Search == "" && cursor.After == nil && cursor.Offset > 0 && cursor.Offset <= math.MaxInt32:
|
||||
query.Offset = cursor.Offset
|
||||
case query.Search != "" && cursor.Offset == 0 && cursor.After != nil && cursor.After.SpanID != "":
|
||||
query.After = cursor.After
|
||||
default:
|
||||
return nil, errors.NewInvalidInputf(ErrCodeInspectInvalidCursor, "invalid cursor")
|
||||
}
|
||||
return query, nil
|
||||
}
|
||||
|
||||
func (search *TraceSpanSearch) SpanIDs() []string {
|
||||
spanIDs := make([]string, len(search.Hits))
|
||||
for i := range search.Hits {
|
||||
spanIDs[i] = search.Hits[i].SpanID
|
||||
}
|
||||
return spanIDs
|
||||
}
|
||||
|
||||
// overviewCursor is the content of a nextCursor: the list it belongs to and where its next page starts.
|
||||
type overviewCursor struct {
|
||||
List string `json:"list"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
// spanListCursor is an offset into every span, or the last span of a page of matches.
|
||||
type spanListCursor struct {
|
||||
Offset int `json:"offset,omitempty"`
|
||||
After *ThreadCursor `json:"after,omitempty"`
|
||||
}
|
||||
|
||||
// decodeCursor rejects a field the cursor does not have, so a cursor of another route is not accepted.
|
||||
func decodeCursor(encoded string, cursor any) error {
|
||||
data, err := base64.RawURLEncoding.DecodeString(encoded)
|
||||
if err != nil {
|
||||
return errors.WrapInvalidInputf(err, ErrCodeInspectInvalidCursor, "invalid cursor")
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(cursor); err != nil {
|
||||
return errors.WrapInvalidInputf(err, ErrCodeInspectInvalidCursor, "invalid cursor")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func encodeCursor(cursor any) string {
|
||||
data, _ := json.Marshal(cursor)
|
||||
return base64.RawURLEncoding.EncodeToString(data)
|
||||
}
|
||||
128
pkg/types/spantypes/trace_inspect_test.go
Normal file
128
pkg/types/spantypes/trace_inspect_test.go
Normal file
@@ -0,0 +1,128 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
)
|
||||
|
||||
func TestNewTraceOverviewQuery(t *testing.T) {
|
||||
validCases := []struct {
|
||||
name string
|
||||
params InspectTraceParams
|
||||
expected TraceOverviewQuery
|
||||
}{
|
||||
{name: "NoCursor_WholeOverview", params: InspectTraceParams{}, expected: TraceOverviewQuery{}},
|
||||
{name: "ErrorsCursor_PageOfErrors", params: InspectTraceParams{Cursor: encodeCursor(overviewCursor{List: OverviewListErrors, Offset: 10})}, expected: TraceOverviewQuery{List: OverviewListErrors, Offset: 10}},
|
||||
{name: "SlowestCursor_PageOfSlowest", params: InspectTraceParams{Cursor: encodeCursor(overviewCursor{List: OverviewListSlowest, Offset: 20})}, expected: TraceOverviewQuery{List: OverviewListSlowest, Offset: 20}},
|
||||
}
|
||||
for _, testCase := range validCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
query, err := NewTraceOverviewQuery(&testCase.params)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testCase.expected, *query)
|
||||
})
|
||||
}
|
||||
|
||||
invalidCases := []struct {
|
||||
name string
|
||||
cursor string
|
||||
}{
|
||||
{name: "Cursor_NotBase64", cursor: "%%%"},
|
||||
{name: "Cursor_NotJSON", cursor: "bm90IGpzb24"},
|
||||
{name: "Cursor_UnknownList", cursor: encodeCursor(overviewCursor{List: "tree", Offset: 10})},
|
||||
{name: "Cursor_WithoutList", cursor: encodeCursor(overviewCursor{Offset: 10})},
|
||||
{name: "Cursor_WithoutOffset", cursor: encodeCursor(overviewCursor{List: OverviewListErrors})},
|
||||
{name: "Cursor_OffsetBeyondAnyTrace", cursor: encodeCursor(overviewCursor{List: OverviewListSlowest, Offset: math.MaxInt})},
|
||||
{name: "Cursor_WithAFieldOfAnotherRoute", cursor: encodeCursor(map[string]any{"list": OverviewListErrors, "offset": 10, "after": map[string]any{"spanId": "abc"}})},
|
||||
}
|
||||
for _, testCase := range invalidCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
_, err := NewTraceOverviewQuery(&InspectTraceParams{Cursor: testCase.cursor})
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTraceSpanQuery(t *testing.T) {
|
||||
validCases := []struct {
|
||||
name string
|
||||
params InspectTraceSpanParams
|
||||
expected TraceSpanQuery
|
||||
}{
|
||||
{name: "NoParams_DefaultDepth", params: InspectTraceSpanParams{}, expected: TraceSpanQuery{SpanID: "abc", Depth: inspectDefaultDepth}},
|
||||
{name: "DepthAndFields", params: InspectTraceSpanParams{Depth: 4, SelectFields: []string{"db.statement"}}, expected: TraceSpanQuery{SpanID: "abc", Depth: 4, SelectFields: []string{"db.statement"}}},
|
||||
{name: "BlankFields_SelectNothing", params: InspectTraceSpanParams{SelectFields: []string{"", " "}}, expected: TraceSpanQuery{SpanID: "abc", Depth: inspectDefaultDepth, SelectFields: []string{}}},
|
||||
}
|
||||
for _, testCase := range validCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
query, err := NewTraceSpanQuery("abc", &testCase.params)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testCase.expected, *query)
|
||||
})
|
||||
}
|
||||
|
||||
invalidCases := []struct {
|
||||
name string
|
||||
depth int
|
||||
}{
|
||||
{name: "Depth_Negative", depth: -1},
|
||||
{name: "Depth_OverMaximum", depth: 11},
|
||||
}
|
||||
for _, testCase := range invalidCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
_, err := NewTraceSpanQuery("abc", &InspectTraceSpanParams{Depth: testCase.depth})
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewTraceSpanListQuery(t *testing.T) {
|
||||
afterCursor := encodeCursor(spanListCursor{After: &ThreadCursor{TimeUnixNano: 42, SpanID: "abc"}})
|
||||
offsetCursor := encodeCursor(spanListCursor{Offset: 400})
|
||||
|
||||
validCases := []struct {
|
||||
name string
|
||||
params InspectTraceSpansParams
|
||||
expected TraceSpanListQuery
|
||||
}{
|
||||
{name: "NoParams_EverySpanWithDefaultLimit", params: InspectTraceSpansParams{}, expected: TraceSpanListQuery{Limit: inspectAllSpansLimit}},
|
||||
{name: "Search_DefaultLimit", params: InspectTraceSpansParams{Search: "worldpay"}, expected: TraceSpanListQuery{Search: "worldpay", Limit: inspectSearchLimit}},
|
||||
{name: "BlankSearch_EverySpan", params: InspectTraceSpansParams{Search: " "}, expected: TraceSpanListQuery{Limit: inspectAllSpansLimit}},
|
||||
{name: "SearchLimitCursor_NextPageOfMatches", params: InspectTraceSpansParams{Search: "worldpay", Limit: 5, Cursor: afterCursor}, expected: TraceSpanListQuery{Search: "worldpay", Limit: 5, After: &ThreadCursor{TimeUnixNano: 42, SpanID: "abc"}}},
|
||||
{name: "UnderLimitCursor_NextPageBelowThatSpan", params: InspectTraceSpansParams{Under: "abc", Limit: 50, Cursor: offsetCursor}, expected: TraceSpanListQuery{Under: "abc", Limit: 50, Offset: 400}},
|
||||
}
|
||||
for _, testCase := range validCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
query, err := NewTraceSpanListQuery(&testCase.params)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, testCase.expected, *query)
|
||||
})
|
||||
}
|
||||
|
||||
invalidCases := []struct {
|
||||
name string
|
||||
params InspectTraceSpansParams
|
||||
}{
|
||||
{name: "SearchWithUnder", params: InspectTraceSpansParams{Search: "worldpay", Under: "abc"}},
|
||||
{name: "Limit_Negative", params: InspectTraceSpansParams{Search: "worldpay", Limit: -1}},
|
||||
{name: "Limit_OverMaximum", params: InspectTraceSpansParams{Limit: 1001}},
|
||||
{name: "SearchCursor_FromEverySpan", params: InspectTraceSpansParams{Search: "worldpay", Cursor: offsetCursor}},
|
||||
{name: "EverySpanCursor_FromASearch", params: InspectTraceSpansParams{Cursor: afterCursor}},
|
||||
{name: "Cursor_FromAnOverviewList", params: InspectTraceSpansParams{Cursor: encodeCursor(overviewCursor{List: OverviewListErrors, Offset: 10})}},
|
||||
{name: "Cursor_OffsetBeyondAnyTrace", params: InspectTraceSpansParams{Cursor: encodeCursor(spanListCursor{Offset: math.MaxInt})}},
|
||||
}
|
||||
for _, testCase := range invalidCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
_, err := NewTraceSpanListQuery(&testCase.params)
|
||||
require.Error(t, err)
|
||||
assert.True(t, errors.Ast(err, errors.TypeInvalidInput))
|
||||
})
|
||||
}
|
||||
}
|
||||
526
pkg/types/spantypes/trace_overview.go
Normal file
526
pkg/types/spantypes/trace_overview.go
Normal file
@@ -0,0 +1,526 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// ErrorPage returns one page of error origins and the ids of its spans, for their status messages.
|
||||
func (s *TraceSkeleton) ErrorPage(offset int) (*TraceErrorList, []string) {
|
||||
origins := s.errorOrigins()
|
||||
list := &TraceErrorList{Total: uint64(len(origins)), Items: []TraceErrorItem{}, NextCursor: nextListCursor(OverviewListErrors, offset, len(origins))}
|
||||
spanIDs := make([]string, 0, inspectListSize)
|
||||
for _, i := range pageOf(origins, offset, inspectListSize) {
|
||||
item := TraceErrorItem{Span: s.Summary(i), Parent: s.parentSummary(i)}
|
||||
for p := s.Parents[i]; p != noParent && s.HasError(p); p = s.Parents[p] {
|
||||
item.ErrorAncestorCount++
|
||||
}
|
||||
list.Items = append(list.Items, item)
|
||||
spanIDs = append(spanIDs, item.Span.SpanID)
|
||||
}
|
||||
return list, spanIDs
|
||||
}
|
||||
|
||||
// SetStatusMessages fills the status message of each item, cut like any other long value.
|
||||
func (list *TraceErrorList) SetStatusMessages(messages map[string]string) {
|
||||
for i := range list.Items {
|
||||
message := messages[list.Items[i].Span.SpanID]
|
||||
if cut, _, ok := cutString(message, spanRecordValueChars); ok {
|
||||
message = cut
|
||||
}
|
||||
list.Items[i].StatusMessage = message
|
||||
}
|
||||
}
|
||||
|
||||
// UnmarkedFailurePage returns one page of the spans that failed without the error flag: a 4xx
|
||||
// response, a gRPC status other than OK, or an exception event.
|
||||
func (s *TraceSkeleton) UnmarkedFailurePage(offset int) *TraceUnmarkedFailureList {
|
||||
type failure struct {
|
||||
index int32
|
||||
reason UnmarkedFailureReason
|
||||
}
|
||||
failures := make([]failure, 0)
|
||||
for i := range int32(s.Len()) {
|
||||
if s.HasError(i) {
|
||||
continue
|
||||
}
|
||||
if reason, ok := unmarkedFailureReason(s.ResponseStatusCode(i), s.HasExceptionEvent(i)); ok {
|
||||
failures = append(failures, failure{index: i, reason: reason})
|
||||
}
|
||||
}
|
||||
list := &TraceUnmarkedFailureList{Total: uint64(len(failures)), Items: []TraceUnmarkedFailureItem{}, NextCursor: nextListCursor(OverviewListUnmarkedFailures, offset, len(failures))}
|
||||
for _, f := range pageOf(failures, offset, inspectListSize) {
|
||||
list.Items = append(list.Items, TraceUnmarkedFailureItem{Span: s.Summary(f.index), Parent: s.parentSummary(f.index), Reason: f.reason})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// LinkPage returns one page of the spans that link to other spans, in start time order.
|
||||
func (s *TraceSkeleton) LinkPage(offset int) *TraceLinkList {
|
||||
list := &TraceLinkList{Total: uint64(len(s.LinkedSpans)), Items: []TraceLinkItem{}, NextCursor: nextListCursor(OverviewListLinks, offset, len(s.LinkedSpans))}
|
||||
for _, linked := range pageOf(s.LinkedSpans, offset, inspectListSize) {
|
||||
idx, ok := s.Index(linked.SpanID)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
list.Items = append(list.Items, TraceLinkItem{
|
||||
Span: s.Summary(idx),
|
||||
Parent: s.parentSummary(idx),
|
||||
LinkCount: uint64(len(linked.Links)),
|
||||
Links: linked.Links[:min(len(linked.Links), inspectLinkTargets)],
|
||||
})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// SlowestPage returns one page of the ranking by own time, same-name spans under one parent
|
||||
// counted as one entry. Only the entries up to the end of the page are kept while ranking.
|
||||
func (s *TraceSkeleton) SlowestPage(offset int) *TraceSlowestList {
|
||||
keep := offset + inspectListSize
|
||||
top := &slowestHeap{}
|
||||
total := 0
|
||||
offer := func(e slowestEntry) {
|
||||
total++
|
||||
switch {
|
||||
case top.Len() < keep:
|
||||
heap.Push(top, e)
|
||||
case e.ranksBefore((*top)[0]):
|
||||
(*top)[0] = e
|
||||
heap.Fix(top, 0)
|
||||
}
|
||||
}
|
||||
single := func(i int32) slowestEntry {
|
||||
e := slowestEntry{longest: i, spanCount: 1, ownTime: s.ownTime[i]}
|
||||
if s.HasError(i) {
|
||||
e.errCount = 1
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
for _, root := range s.roots {
|
||||
offer(single(root))
|
||||
}
|
||||
var siblings []int32
|
||||
for parent := range int32(s.Len()) {
|
||||
children := s.Children(parent)
|
||||
if len(children) == 1 {
|
||||
offer(single(children[0]))
|
||||
continue
|
||||
}
|
||||
siblings = append(siblings[:0], children...)
|
||||
slices.SortStableFunc(siblings, func(a, b int32) int {
|
||||
if s.Names[a] != s.Names[b] {
|
||||
return int(s.Names[a]) - int(s.Names[b])
|
||||
}
|
||||
return int(s.Services[a]) - int(s.Services[b])
|
||||
})
|
||||
for k := 0; k < len(siblings); {
|
||||
e := single(siblings[k])
|
||||
next := k + 1
|
||||
for ; next < len(siblings) && s.Names[siblings[next]] == s.Names[siblings[k]] && s.Services[siblings[next]] == s.Services[siblings[k]]; next++ {
|
||||
member := single(siblings[next])
|
||||
e.spanCount++
|
||||
e.ownTime += member.ownTime
|
||||
e.errCount += member.errCount
|
||||
if s.DurationNano[siblings[next]] > s.DurationNano[e.longest] {
|
||||
e.longest = siblings[next]
|
||||
}
|
||||
}
|
||||
offer(e)
|
||||
k = next
|
||||
}
|
||||
}
|
||||
|
||||
ranked := []slowestEntry(*top)
|
||||
slices.SortFunc(ranked, func(a, b slowestEntry) int {
|
||||
if a.ranksBefore(b) {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
})
|
||||
list := &TraceSlowestList{Total: uint64(total), Items: []TraceSlowestItem{}, NextCursor: nextListCursor(OverviewListSlowest, offset, total)}
|
||||
for _, e := range pageOf(ranked, offset, inspectListSize) {
|
||||
list.Items = append(list.Items, TraceSlowestItem{Span: s.Summary(e.longest), Parent: s.parentSummary(e.longest), SpanCount: e.spanCount, OwnTimeNano: e.ownTime, ErrorSpanCount: e.errCount})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// ExpensivePage returns one page of the gen_ai spans ranked by cost, or by tokens when no span
|
||||
// carries a cost; nil without gen_ai spans.
|
||||
func (s *TraceSkeleton) ExpensivePage(offset int) *TraceExpensiveList {
|
||||
if len(s.AISpans) == 0 {
|
||||
return nil
|
||||
}
|
||||
hasCost := false
|
||||
for i := range s.AISpans {
|
||||
if s.AISpans[i].Cost != nil {
|
||||
hasCost = true
|
||||
break
|
||||
}
|
||||
}
|
||||
type rankedSpan struct {
|
||||
index int32
|
||||
weight float64
|
||||
usage *AISpanUsage
|
||||
}
|
||||
ranked := make([]rankedSpan, 0, len(s.AISpans))
|
||||
for i := range s.AISpans {
|
||||
usage := &s.AISpans[i]
|
||||
index, ok := s.Index(usage.SpanID)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
weight := floatValue(usage.InputTokens) + floatValue(usage.OutputTokens)
|
||||
if hasCost {
|
||||
weight = floatValue(usage.Cost)
|
||||
}
|
||||
ranked = append(ranked, rankedSpan{index: index, weight: weight, usage: usage})
|
||||
}
|
||||
// Equal weights fall back to span order, so that every page cuts the same ranking.
|
||||
slices.SortFunc(ranked, func(a, b rankedSpan) int {
|
||||
switch {
|
||||
case a.weight > b.weight:
|
||||
return -1
|
||||
case a.weight < b.weight:
|
||||
return 1
|
||||
}
|
||||
return int(a.index) - int(b.index)
|
||||
})
|
||||
|
||||
list := &TraceExpensiveList{Total: uint64(len(ranked)), Items: []TraceExpensiveItem{}, NextCursor: nextListCursor(OverviewListExpensive, offset, len(ranked))}
|
||||
for _, span := range pageOf(ranked, offset, inspectListSize) {
|
||||
list.Items = append(list.Items, TraceExpensiveItem{
|
||||
Span: s.Summary(span.index),
|
||||
Parent: s.parentSummary(span.index),
|
||||
Cost: span.usage.Cost,
|
||||
InputTokens: uint64(floatValue(span.usage.InputTokens)),
|
||||
OutputTokens: uint64(floatValue(span.usage.OutputTokens)),
|
||||
})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) Identity() *TraceIdentity {
|
||||
identity := &TraceIdentity{
|
||||
TraceID: s.TraceID,
|
||||
StartTimestampMillis: uint64(s.startNs / 1_000_000),
|
||||
DurationNano: uint64(s.endNs - s.startNs),
|
||||
SpanCount: uint64(s.Len()),
|
||||
ErrorSpanCount: s.errCount,
|
||||
RootCount: uint64(len(s.roots)),
|
||||
HasMissingSpans: s.HasMissingSpans(),
|
||||
}
|
||||
if root, ok := s.PrimaryRoot(); ok && !s.HasMissingParent(root) {
|
||||
summary := s.Summary(root)
|
||||
identity.Root = &summary
|
||||
}
|
||||
|
||||
stats := make([]TraceServiceStats, len(s.Strings))
|
||||
for i := range int32(s.Len()) {
|
||||
service := &stats[s.Services[i]]
|
||||
service.SpanCount++
|
||||
service.OwnTimeNano += s.ownTime[i]
|
||||
if s.HasError(i) {
|
||||
service.ErrorSpanCount++
|
||||
}
|
||||
}
|
||||
services := make([]TraceServiceStats, 0)
|
||||
for idx := range stats {
|
||||
if stats[idx].SpanCount > 0 {
|
||||
stats[idx].ServiceName = s.Strings[idx]
|
||||
services = append(services, stats[idx])
|
||||
}
|
||||
}
|
||||
slices.SortStableFunc(services, func(a, b TraceServiceStats) int {
|
||||
switch {
|
||||
case a.OwnTimeNano > b.OwnTimeNano:
|
||||
return -1
|
||||
case a.OwnTimeNano < b.OwnTimeNano:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
identity.ServiceCount = uint64(len(services))
|
||||
identity.Services = services[:min(len(services), inspectServiceLimit)]
|
||||
|
||||
if len(s.AISpans) > 0 {
|
||||
ai := &TraceAIIdentity{}
|
||||
for i := range s.AISpans {
|
||||
usage := &s.AISpans[i]
|
||||
if usage.IsModelCall {
|
||||
ai.ModelCallCount++
|
||||
}
|
||||
if usage.IsToolCall {
|
||||
ai.ToolCallCount++
|
||||
}
|
||||
ai.Tokens.Input += uint64(floatValue(usage.InputTokens))
|
||||
ai.Tokens.Output += uint64(floatValue(usage.OutputTokens))
|
||||
ai.Tokens.CacheRead += uint64(floatValue(usage.CacheReadTokens))
|
||||
ai.Tokens.CacheWrite += uint64(floatValue(usage.CacheWriteTokens))
|
||||
ai.Tokens.Reasoning += uint64(floatValue(usage.ReasoningTokens))
|
||||
if usage.Cost != nil {
|
||||
if ai.TotalCost == nil {
|
||||
ai.TotalCost = new(float64)
|
||||
}
|
||||
*ai.TotalCost += *usage.Cost
|
||||
}
|
||||
}
|
||||
identity.AI = ai
|
||||
}
|
||||
return identity
|
||||
}
|
||||
|
||||
// Tree lists the first 50 spans of the trace: level by level from the roots, and in time order on
|
||||
// the level that does not fit. It is complete when the trace has no more spans than that.
|
||||
func (s *TraceSkeleton) Tree() *TraceTree {
|
||||
return s.tree(inspectTreeSpans)
|
||||
}
|
||||
|
||||
// FitOverview drops the span records and then shrinks the tree until the overview fits the
|
||||
// response size limit.
|
||||
func (s *TraceSkeleton) FitOverview(overview *TraceOverview) {
|
||||
for limit := len(overview.Tree.Spans); jsonChars(overview) > inspectMaxResponseChars && limit > 0; {
|
||||
if overview.Tree.Records != nil {
|
||||
overview.Tree.Records = nil
|
||||
continue
|
||||
}
|
||||
over := jsonChars(overview) - inspectMaxResponseChars
|
||||
perSpan := jsonChars(overview.Tree.Spans)/len(overview.Tree.Spans) + 1
|
||||
limit = max(limit-over/perSpan-1, 0)
|
||||
overview.Tree = s.tree(limit)
|
||||
}
|
||||
}
|
||||
|
||||
// SpanDetail places span i in the trace: its nearest ancestors with the root, and the first
|
||||
// spans of each level below it. The record and the percentile are filled by the caller.
|
||||
func (s *TraceSkeleton) SpanDetail(i int32, depth int) *TraceSpanDetail {
|
||||
ancestors := s.Ancestors(i)
|
||||
detail := &TraceSpanDetail{
|
||||
Summary: s.Summary(i),
|
||||
Ancestors: make([]SpanSummary, 0, depth+1),
|
||||
AncestorCount: uint64(len(ancestors)),
|
||||
Descendants: make([]TraceSpanLevel, 0, depth),
|
||||
}
|
||||
for _, ancestor := range ancestors[:min(len(ancestors), depth)] {
|
||||
detail.Ancestors = append(detail.Ancestors, s.Summary(ancestor))
|
||||
}
|
||||
if len(ancestors) > depth {
|
||||
detail.Ancestors = append(detail.Ancestors, s.Summary(ancestors[len(ancestors)-1]))
|
||||
}
|
||||
|
||||
spans := []int32{i}
|
||||
for level := 1; level <= depth; level++ {
|
||||
next := make([]int32, 0)
|
||||
for _, parent := range spans {
|
||||
next = append(next, s.Children(parent)...)
|
||||
}
|
||||
if len(next) == 0 {
|
||||
break
|
||||
}
|
||||
// Positions are in start time order; sorting puts the children of different parents in time order.
|
||||
slices.Sort(next)
|
||||
shown := make([]SpanSummary, 0, min(len(next), inspectLevelSize))
|
||||
for _, child := range next[:min(len(next), inspectLevelSize)] {
|
||||
shown = append(shown, s.Summary(child))
|
||||
}
|
||||
detail.Descendants = append(detail.Descendants, TraceSpanLevel{Level: uint32(level), Total: uint64(len(next)), Spans: shown})
|
||||
spans = next
|
||||
}
|
||||
return detail
|
||||
}
|
||||
|
||||
// SpanPage returns one page of every span in tree order, or of root and the spans below it, ended
|
||||
// early when the response would pass its size limit. A span's parent is earlier in the list.
|
||||
func (s *TraceSkeleton) SpanPage(root *int32, offset, limit int) *TraceSpanList {
|
||||
order := s.preorder
|
||||
if root != nil {
|
||||
first := slices.Index(s.preorder, *root)
|
||||
order = s.preorder[first : first+1+int(s.subtree[*root])]
|
||||
}
|
||||
list := &TraceSpanList{Total: uint64(len(order)), Items: []TraceSpanMatch{}}
|
||||
chars := 0
|
||||
for _, i := range pageOf(order, offset, limit) {
|
||||
item := TraceSpanMatch{Span: s.Summary(i), Matches: []SpanTextMatch{}}
|
||||
chars += jsonChars(item)
|
||||
if chars > inspectMaxResponseChars && len(list.Items) > 0 {
|
||||
break
|
||||
}
|
||||
list.Items = append(list.Items, item)
|
||||
}
|
||||
if next := offset + len(list.Items); next < len(order) {
|
||||
list.NextCursor = encodeCursor(spanListCursor{Offset: next})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// NewTraceSpanList turns a page of search hits into matches: the span, its parent and where the
|
||||
// words were found. It ends early when the response would pass its size limit.
|
||||
func NewTraceSpanList(skeleton *TraceSkeleton, search *TraceSpanSearch, spans []StorableSpan) *TraceSpanList {
|
||||
byID := make(map[string]*StorableSpan, len(spans))
|
||||
for i := range spans {
|
||||
byID[spans[i].SpanID] = &spans[i]
|
||||
}
|
||||
list := &TraceSpanList{Total: search.Total, Items: []TraceSpanMatch{}, Warnings: search.Warnings}
|
||||
chars, hasMore := 0, search.HasMore
|
||||
var last *ThreadCursor
|
||||
for i := range search.Hits {
|
||||
span, hasRecord := byID[search.Hits[i].SpanID]
|
||||
idx, inTree := skeleton.Index(search.Hits[i].SpanID)
|
||||
if !hasRecord || !inTree {
|
||||
last = &search.Hits[i]
|
||||
continue
|
||||
}
|
||||
item := TraceSpanMatch{Span: skeleton.Summary(idx), Parent: skeleton.parentSummary(idx), Matches: findWords(span, search.Words)}
|
||||
chars += jsonChars(item)
|
||||
if chars > inspectMaxResponseChars && len(list.Items) > 0 {
|
||||
hasMore = true
|
||||
break
|
||||
}
|
||||
list.Items = append(list.Items, item)
|
||||
last = &search.Hits[i]
|
||||
}
|
||||
if hasMore && last != nil {
|
||||
list.NextCursor = encodeCursor(spanListCursor{After: last})
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
// FitSpan drops the deepest levels below the span until the response fits its size limit.
|
||||
func FitSpan(detail *TraceSpanDetail) {
|
||||
for len(detail.Descendants) > 0 && jsonChars(detail) > inspectMaxResponseChars {
|
||||
detail.Descendants = detail.Descendants[:len(detail.Descendants)-1]
|
||||
}
|
||||
}
|
||||
|
||||
// slowestEntry is one span, or the same-name spans under one parent taken together.
|
||||
type slowestEntry struct {
|
||||
longest int32
|
||||
spanCount uint64
|
||||
ownTime uint64
|
||||
errCount uint64
|
||||
}
|
||||
|
||||
// ranksBefore orders entries by own time, the largest first, and by span order between equals,
|
||||
// so that every page cuts the same ranking.
|
||||
func (e slowestEntry) ranksBefore(other slowestEntry) bool {
|
||||
if e.ownTime != other.ownTime {
|
||||
return e.ownTime > other.ownTime
|
||||
}
|
||||
return e.longest < other.longest
|
||||
}
|
||||
|
||||
// slowestHeap holds the best-ranked entries seen so far, the last of them on top.
|
||||
type slowestHeap []slowestEntry
|
||||
|
||||
func (h slowestHeap) Len() int { return len(h) }
|
||||
func (h slowestHeap) Less(i, j int) bool { return h[j].ranksBefore(h[i]) }
|
||||
func (h slowestHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *slowestHeap) Push(x any) { *h = append(*h, x.(slowestEntry)) }
|
||||
|
||||
func (h *slowestHeap) Pop() any {
|
||||
old := *h
|
||||
last := old[len(old)-1]
|
||||
*h = old[:len(old)-1]
|
||||
return last
|
||||
}
|
||||
|
||||
// errorOrigins lists the error spans with no error span among their children, in start time order.
|
||||
func (s *TraceSkeleton) errorOrigins() []int32 {
|
||||
origins := make([]int32, 0)
|
||||
for i := range int32(s.Len()) {
|
||||
if !s.HasError(i) {
|
||||
continue
|
||||
}
|
||||
hasErrorChild := false
|
||||
for _, child := range s.Children(i) {
|
||||
if s.HasError(child) {
|
||||
hasErrorChild = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasErrorChild {
|
||||
origins = append(origins, i)
|
||||
}
|
||||
}
|
||||
return origins
|
||||
}
|
||||
|
||||
// unmarkedFailureReason reads the response status column, which holds an HTTP status or a gRPC status.
|
||||
func unmarkedFailureReason(responseStatusCode string, hasExceptionEvent bool) (UnmarkedFailureReason, bool) {
|
||||
if code, err := strconv.Atoi(responseStatusCode); err == nil {
|
||||
switch {
|
||||
case code >= 400 && code < 500:
|
||||
return UnmarkedFailureHTTP4xx, true
|
||||
case code >= 1 && code <= 16:
|
||||
return UnmarkedFailureGRPCStatus, true
|
||||
}
|
||||
}
|
||||
if hasExceptionEvent {
|
||||
return UnmarkedFailureExceptionEvent, true
|
||||
}
|
||||
return UnmarkedFailureReason{}, false
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) tree(limit int) *TraceTree {
|
||||
tree := &TraceTree{Total: uint64(s.Len()), Complete: s.Len() <= limit}
|
||||
shown := make(map[int32]struct{}, min(limit, s.Len()))
|
||||
for level := s.roots; len(level) > 0 && len(shown) < limit; {
|
||||
level = level[:min(len(level), limit-len(shown))]
|
||||
next := make([]int32, 0)
|
||||
for _, i := range level {
|
||||
shown[i] = struct{}{}
|
||||
next = append(next, s.Children(i)...)
|
||||
}
|
||||
// Positions are in start time order; sorting puts the children of different parents in time order.
|
||||
slices.Sort(next)
|
||||
level = next
|
||||
}
|
||||
|
||||
// The shown spans are returned in tree order: a span is followed by the spans below it.
|
||||
tree.Spans = make([]SpanSummary, 0, len(shown))
|
||||
for _, i := range s.preorder {
|
||||
if _, ok := shown[i]; ok {
|
||||
tree.Spans = append(tree.Spans, s.Summary(i))
|
||||
}
|
||||
}
|
||||
return tree
|
||||
}
|
||||
|
||||
// parentSummary returns the short form of span i's direct parent; nil for a root or a missing parent.
|
||||
func (s *TraceSkeleton) parentSummary(i int32) *SpanSummary {
|
||||
parent := s.Parents[i]
|
||||
if parent == noParent {
|
||||
return nil
|
||||
}
|
||||
summary := s.Summary(parent)
|
||||
return &summary
|
||||
}
|
||||
|
||||
// nextListCursor returns the cursor of the page after the one starting at offset; empty on the last page.
|
||||
func nextListCursor(list string, offset, total int) string {
|
||||
if offset+inspectListSize >= total {
|
||||
return ""
|
||||
}
|
||||
return encodeCursor(overviewCursor{List: list, Offset: offset + inspectListSize})
|
||||
}
|
||||
|
||||
func pageOf[T any](items []T, offset, limit int) []T {
|
||||
if offset >= len(items) {
|
||||
return nil
|
||||
}
|
||||
return items[offset:min(offset+limit, len(items))]
|
||||
}
|
||||
|
||||
func jsonChars(value any) int {
|
||||
data, _ := json.Marshal(value)
|
||||
return len(data)
|
||||
}
|
||||
|
||||
func floatValue(value *float64) float64 {
|
||||
if value == nil {
|
||||
return 0
|
||||
}
|
||||
return *value
|
||||
}
|
||||
742
pkg/types/spantypes/trace_overview_test.go
Normal file
742
pkg/types/spantypes/trace_overview_test.go
Normal file
@@ -0,0 +1,742 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func summaryIDs(summaries []SpanSummary) []string {
|
||||
ids := make([]string, 0, len(summaries))
|
||||
for _, summary := range summaries {
|
||||
ids = append(ids, summary.SpanID)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
// nextOffset reads the offset out of a list's nextCursor, as a request carrying it would.
|
||||
func nextOffset(t *testing.T, list, cursor string) int {
|
||||
t.Helper()
|
||||
query, err := NewTraceOverviewQuery(&InspectTraceParams{Cursor: cursor})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, list, query.List)
|
||||
return query.Offset
|
||||
}
|
||||
|
||||
func TestErrorPage(t *testing.T) {
|
||||
t.Run("ErrorCarriedUpToRoot_ListsWhereItStarted", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "level-0", startMs: 0, durationMs: 100, hasError: true}}
|
||||
for depth := 1; depth <= 12; depth++ {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("level-%d", depth), parent: fmt.Sprintf("level-%d", depth-1), startMs: int64(depth), durationMs: 50, hasError: true})
|
||||
}
|
||||
specs = append(specs, skeletonSpec{id: "healthy", parent: "level-0", startMs: 60, durationMs: 5})
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
list, spanIDs := skeleton.ErrorPage(0)
|
||||
|
||||
assert.Equal(t, uint64(1), list.Total)
|
||||
assert.Equal(t, []string{"level-12"}, spanIDs)
|
||||
require.Len(t, list.Items, 1)
|
||||
assert.Equal(t, uint32(12), list.Items[0].ErrorAncestorCount)
|
||||
require.NotNil(t, list.Items[0].Parent)
|
||||
assert.Equal(t, "level-11", list.Items[0].Parent.SpanID)
|
||||
assert.Empty(t, list.NextCursor)
|
||||
})
|
||||
|
||||
t.Run("ParentFailsOnItsOwn_ListsChildOnly", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "parent", parent: "root", startMs: 1, durationMs: 50, hasError: true},
|
||||
skeletonSpec{id: "child", parent: "parent", startMs: 2, durationMs: 10, hasError: true},
|
||||
skeletonSpec{id: "other", parent: "root", startMs: 60, durationMs: 10, hasError: true},
|
||||
)
|
||||
|
||||
list, spanIDs := skeleton.ErrorPage(0)
|
||||
|
||||
assert.Equal(t, uint64(2), list.Total)
|
||||
assert.Equal(t, []string{"child", "other"}, spanIDs)
|
||||
})
|
||||
|
||||
t.Run("MoreThanOnePage_PagesByCursorInStartOrder", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
for i := range 23 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("err-%02d", i), parent: "root", startMs: int64(i + 1), durationMs: 1, hasError: true})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
seen := make([]string, 0, 23)
|
||||
offset := 0
|
||||
for range 3 {
|
||||
list, spanIDs := skeleton.ErrorPage(offset)
|
||||
assert.Equal(t, uint64(23), list.Total)
|
||||
seen = append(seen, spanIDs...)
|
||||
if list.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
offset = nextOffset(t, OverviewListErrors, list.NextCursor)
|
||||
}
|
||||
|
||||
require.Len(t, seen, 23)
|
||||
assert.Equal(t, "err-00", seen[0])
|
||||
assert.Equal(t, "err-10", seen[10])
|
||||
assert.Equal(t, "err-22", seen[22])
|
||||
})
|
||||
|
||||
t.Run("RootError_HasNoParentAndItsMessageIsCut", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(skeletonSpec{id: "root", startMs: 0, durationMs: 10, hasError: true})
|
||||
list, _ := skeleton.ErrorPage(0)
|
||||
message := ""
|
||||
for range 60 {
|
||||
message += "0123456789"
|
||||
}
|
||||
|
||||
list.SetStatusMessages(map[string]string{"root": message})
|
||||
|
||||
assert.Nil(t, list.Items[0].Parent)
|
||||
assert.Len(t, []rune(list.Items[0].StatusMessage), spanRecordValueChars+1)
|
||||
})
|
||||
}
|
||||
|
||||
func TestUnmarkedFailurePage(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100, statusCode: "200"},
|
||||
skeletonSpec{id: "not-found", parent: "root", startMs: 1, durationMs: 1, statusCode: "404"},
|
||||
skeletonSpec{id: "flagged-4xx", parent: "root", startMs: 2, durationMs: 1, statusCode: "409", hasError: true},
|
||||
skeletonSpec{id: "grpc-ok", parent: "root", startMs: 3, durationMs: 1, statusCode: "0"},
|
||||
skeletonSpec{id: "grpc-not-found", parent: "root", startMs: 4, durationMs: 1, statusCode: "5"},
|
||||
skeletonSpec{id: "handled-exception", parent: "root", startMs: 5, durationMs: 1, hasException: true},
|
||||
skeletonSpec{id: "server-error", parent: "root", startMs: 6, durationMs: 1, statusCode: "503"},
|
||||
skeletonSpec{id: "json-rpc", parent: "root", startMs: 7, durationMs: 1, statusCode: "-32601"},
|
||||
)
|
||||
|
||||
list := skeleton.UnmarkedFailurePage(0)
|
||||
|
||||
require.Len(t, list.Items, 3)
|
||||
assert.Equal(t, uint64(3), list.Total)
|
||||
assert.Equal(t, "not-found", list.Items[0].Span.SpanID)
|
||||
assert.Equal(t, UnmarkedFailureHTTP4xx, list.Items[0].Reason)
|
||||
require.NotNil(t, list.Items[0].Parent)
|
||||
assert.Equal(t, "root", list.Items[0].Parent.SpanID)
|
||||
assert.Equal(t, "grpc-not-found", list.Items[1].Span.SpanID)
|
||||
assert.Equal(t, UnmarkedFailureGRPCStatus, list.Items[1].Reason)
|
||||
assert.Equal(t, "handled-exception", list.Items[2].Span.SpanID)
|
||||
assert.Equal(t, UnmarkedFailureExceptionEvent, list.Items[2].Reason)
|
||||
}
|
||||
|
||||
func TestLinkPage(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
linkedSpans := make([]LinkedSpan, 0, 14)
|
||||
for i := range 13 {
|
||||
id := fmt.Sprintf("consumer-%02d", i)
|
||||
specs = append(specs, skeletonSpec{id: id, parent: "root", startMs: int64(i + 1), durationMs: 1})
|
||||
// Given in reverse so the page order has to come from the spans, not from the input.
|
||||
linkedSpans = append([]LinkedSpan{{SpanID: id, Links: []Link{{TraceID: "producer-trace", SpanID: id + "-producer"}}}}, linkedSpans...)
|
||||
}
|
||||
batch := LinkedSpan{SpanID: "consumer-00"}
|
||||
for i := range 8 {
|
||||
batch.Links = append(batch.Links, Link{TraceID: fmt.Sprintf("producer-trace-%d", i), SpanID: "producer"})
|
||||
}
|
||||
linkedSpans[len(linkedSpans)-1] = batch
|
||||
linkedSpans = append(linkedSpans, LinkedSpan{SpanID: "not-loaded", Links: []Link{{TraceID: "t", SpanID: "s"}}})
|
||||
skeleton := buildSkeleton(specs...).WithLinkedSpans(linkedSpans)
|
||||
|
||||
first := skeleton.LinkPage(0)
|
||||
|
||||
assert.Equal(t, uint64(13), first.Total)
|
||||
require.Len(t, first.Items, inspectListSize)
|
||||
assert.Equal(t, "consumer-00", first.Items[0].Span.SpanID)
|
||||
require.NotNil(t, first.Items[0].Parent)
|
||||
assert.Equal(t, "root", first.Items[0].Parent.SpanID)
|
||||
assert.Equal(t, uint64(8), first.Items[0].LinkCount)
|
||||
assert.Len(t, first.Items[0].Links, inspectLinkTargets)
|
||||
assert.Equal(t, "consumer-09", first.Items[9].Span.SpanID)
|
||||
|
||||
second := skeleton.LinkPage(nextOffset(t, OverviewListLinks, first.NextCursor))
|
||||
require.Len(t, second.Items, 3)
|
||||
assert.Equal(t, "consumer-12", second.Items[2].Span.SpanID)
|
||||
assert.Empty(t, second.NextCursor)
|
||||
|
||||
empty := buildSkeleton(skeletonSpec{id: "root", startMs: 0, durationMs: 1}).LinkPage(0)
|
||||
assert.Equal(t, uint64(0), empty.Total)
|
||||
assert.Empty(t, empty.Items)
|
||||
}
|
||||
|
||||
func TestSlowestPage(t *testing.T) {
|
||||
t.Run("SameNameUnderOneParent_CountsAsOneEntry", func(t *testing.T) {
|
||||
specs := []skeletonSpec{
|
||||
{id: "root", name: "POST /graphql", service: "api", startMs: 0, durationMs: 10_000},
|
||||
{id: "resolve", parent: "root", name: "resolve_plans", service: "api", startMs: 1, durationMs: 9_000},
|
||||
{id: "slow-single", parent: "root", name: "render", service: "api", startMs: 9_100, durationMs: 300},
|
||||
}
|
||||
for i := range 416 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("cost-%03d", i), parent: "resolve", name: "calculate_annual_cost", service: "pricing", startMs: int64(2 + i*20), durationMs: int64(10 + i%3), hasError: i < 3})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
list := skeleton.SlowestPage(0)
|
||||
|
||||
require.NotEmpty(t, list.Items)
|
||||
assert.Equal(t, uint64(4), list.Total)
|
||||
assert.Empty(t, list.NextCursor)
|
||||
group := list.Items[0]
|
||||
assert.Equal(t, "calculate_annual_cost", group.Span.Name)
|
||||
assert.Equal(t, uint64(416), group.SpanCount)
|
||||
assert.Equal(t, uint64(3), group.ErrorSpanCount)
|
||||
assert.Equal(t, uint64(12*time.Millisecond), group.Span.DurationNano)
|
||||
require.NotNil(t, group.Parent)
|
||||
assert.Equal(t, "resolve", group.Parent.SpanID)
|
||||
assert.Equal(t, "resolve", list.Items[1].Span.SpanID)
|
||||
assert.Equal(t, uint64(1), list.Items[1].SpanCount)
|
||||
})
|
||||
|
||||
t.Run("SameNameUnderDifferentParents_StaysSeparate", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", name: "root", service: "api", startMs: 0, durationMs: 107},
|
||||
skeletonSpec{id: "left", parent: "root", name: "branch", service: "api", startMs: 0, durationMs: 40},
|
||||
skeletonSpec{id: "left-query", parent: "left", name: "SELECT", service: "db", startMs: 1, durationMs: 30},
|
||||
skeletonSpec{id: "right", parent: "root", name: "branch", service: "worker", startMs: 50, durationMs: 40},
|
||||
skeletonSpec{id: "right-query", parent: "right", name: "SELECT", service: "db", startMs: 51, durationMs: 15},
|
||||
)
|
||||
|
||||
list := skeleton.SlowestPage(0)
|
||||
|
||||
assert.Equal(t, uint64(5), list.Total)
|
||||
assert.Equal(t, []string{"left-query", "root", "right", "right-query", "left"}, []string{
|
||||
list.Items[0].Span.SpanID, list.Items[1].Span.SpanID, list.Items[2].Span.SpanID, list.Items[3].Span.SpanID, list.Items[4].Span.SpanID,
|
||||
})
|
||||
})
|
||||
|
||||
t.Run("MoreThanOnePage_PagesOneRankingByCursor", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", name: "root", service: "api", startMs: 0, durationMs: 1}}
|
||||
for i := range 30 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("op-%02d", i), parent: "root", name: fmt.Sprintf("op-%02d", i), service: "api", startMs: 10, durationMs: int64(100 + i)})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
seen := make([]string, 0, 31)
|
||||
offset := 0
|
||||
for range 5 {
|
||||
list := skeleton.SlowestPage(offset)
|
||||
assert.Equal(t, uint64(31), list.Total)
|
||||
for _, item := range list.Items {
|
||||
seen = append(seen, item.Span.SpanID)
|
||||
}
|
||||
if list.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
offset = nextOffset(t, OverviewListSlowest, list.NextCursor)
|
||||
}
|
||||
|
||||
require.Len(t, seen, 31)
|
||||
assert.Equal(t, "op-29", seen[0])
|
||||
assert.Equal(t, "op-20", seen[9])
|
||||
assert.Equal(t, "op-19", seen[10])
|
||||
assert.Equal(t, "op-00", seen[29])
|
||||
assert.Equal(t, "root", seen[30])
|
||||
})
|
||||
|
||||
t.Run("EqualOwnTime_PagesInSpanOrder", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", name: "root", service: "api", startMs: 0, durationMs: 1}}
|
||||
for i := range 25 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("op-%02d", i), parent: "root", name: fmt.Sprintf("op-%02d", i), service: "api", startMs: int64(10 + i), durationMs: 5})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
first, second, third := skeleton.SlowestPage(0), skeleton.SlowestPage(10), skeleton.SlowestPage(20)
|
||||
|
||||
assert.Equal(t, "op-00", first.Items[0].Span.SpanID)
|
||||
assert.Equal(t, "op-09", first.Items[9].Span.SpanID)
|
||||
assert.Equal(t, "op-10", second.Items[0].Span.SpanID)
|
||||
assert.Equal(t, "op-19", second.Items[9].Span.SpanID)
|
||||
require.Len(t, third.Items, 6)
|
||||
assert.Equal(t, "op-24", third.Items[4].Span.SpanID)
|
||||
assert.Equal(t, "root", third.Items[5].Span.SpanID)
|
||||
assert.Empty(t, skeleton.SlowestPage(40).Items)
|
||||
})
|
||||
}
|
||||
|
||||
func TestTree(t *testing.T) {
|
||||
t.Run("TraceWithinLimit_ListsEverySpanInTreeOrder", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "a", parent: "root", startMs: 1, durationMs: 40},
|
||||
skeletonSpec{id: "b", parent: "root", startMs: 50, durationMs: 40},
|
||||
skeletonSpec{id: "a-child", parent: "a", startMs: 60, durationMs: 1},
|
||||
)
|
||||
|
||||
tree := skeleton.tree(4)
|
||||
|
||||
assert.True(t, tree.Complete)
|
||||
assert.Equal(t, uint64(4), tree.Total)
|
||||
assert.Equal(t, []string{"root", "a", "a-child", "b"}, summaryIDs(tree.Spans))
|
||||
})
|
||||
|
||||
t.Run("WideLevelOverLimit_ShowsItsFirstSpansInTimeOrder", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
for i := range 200 {
|
||||
spec := skeletonSpec{id: fmt.Sprintf("s-%03d", i), parent: "root", startMs: int64(i), durationMs: 1}
|
||||
// A late error and a late long span do not change which spans are shown.
|
||||
switch i {
|
||||
case 150:
|
||||
spec.hasError = true
|
||||
case 170:
|
||||
spec.durationMs = 900
|
||||
}
|
||||
specs = append(specs, spec)
|
||||
}
|
||||
|
||||
tree := buildSkeleton(specs...).Tree()
|
||||
|
||||
assert.False(t, tree.Complete)
|
||||
assert.Equal(t, uint64(201), tree.Total)
|
||||
ids := summaryIDs(tree.Spans)
|
||||
require.Len(t, ids, inspectTreeSpans)
|
||||
assert.Equal(t, "root", ids[0])
|
||||
assert.Equal(t, "s-000", ids[1])
|
||||
assert.Equal(t, "s-048", ids[49])
|
||||
assert.Equal(t, uint32(200), tree.Spans[0].ChildCount)
|
||||
})
|
||||
|
||||
t.Run("NarrowTopLevels_DescendsUntilLimit", func(t *testing.T) {
|
||||
specs := []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 1000},
|
||||
{id: "gateway", parent: "root", startMs: 1, durationMs: 900},
|
||||
{id: "handler", parent: "gateway", startMs: 2, durationMs: 800},
|
||||
}
|
||||
for i := range 150 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("q-%03d", i), parent: "handler", startMs: int64(3 + i), durationMs: 1})
|
||||
}
|
||||
|
||||
tree := buildSkeleton(specs...).tree(10)
|
||||
|
||||
assert.Equal(t, []string{"root", "gateway", "handler", "q-000", "q-001", "q-002", "q-003", "q-004", "q-005", "q-006"}, summaryIDs(tree.Spans))
|
||||
assert.Equal(t, uint32(150), tree.Spans[2].ChildCount)
|
||||
})
|
||||
|
||||
t.Run("LevelCutAcrossParents_KeepsTreeOrder", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "a", parent: "root", startMs: 1, durationMs: 40},
|
||||
skeletonSpec{id: "b", parent: "root", startMs: 50, durationMs: 40},
|
||||
skeletonSpec{id: "a-1", parent: "a", startMs: 2, durationMs: 1},
|
||||
skeletonSpec{id: "a-2", parent: "a", startMs: 3, durationMs: 1},
|
||||
skeletonSpec{id: "b-1", parent: "b", startMs: 51, durationMs: 1},
|
||||
)
|
||||
|
||||
tree := skeleton.tree(5)
|
||||
|
||||
assert.False(t, tree.Complete)
|
||||
assert.Equal(t, []string{"root", "a", "a-1", "a-2", "b"}, summaryIDs(tree.Spans))
|
||||
})
|
||||
|
||||
t.Run("LevelCutAcrossParents_TakesItsEarliestSpans", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 1000},
|
||||
skeletonSpec{id: "first", parent: "root", startMs: 1, durationMs: 900},
|
||||
skeletonSpec{id: "second", parent: "root", startMs: 2, durationMs: 900},
|
||||
skeletonSpec{id: "second-early", parent: "second", startMs: 10, durationMs: 1},
|
||||
skeletonSpec{id: "second-next", parent: "second", startMs: 20, durationMs: 1},
|
||||
skeletonSpec{id: "first-late", parent: "first", startMs: 500, durationMs: 1},
|
||||
)
|
||||
|
||||
tree := skeleton.tree(5)
|
||||
|
||||
assert.Equal(t, []string{"root", "first", "second", "second-early", "second-next"}, summaryIDs(tree.Spans))
|
||||
})
|
||||
}
|
||||
|
||||
func TestFitOverview(t *testing.T) {
|
||||
// A root with 99 children that each fail with a 404; a long service name makes every line large.
|
||||
traceWith := func(service string) *TraceSkeleton {
|
||||
specs := []skeletonSpec{{id: "root", name: "root", service: service, startMs: 0, durationMs: 1000}}
|
||||
for i := range 99 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("span-%03d", i), parent: "root", name: fmt.Sprintf("operation %03d", i), service: service, startMs: int64(i), durationMs: 1, statusCode: "404"})
|
||||
}
|
||||
return buildSkeleton(specs...)
|
||||
}
|
||||
newOverview := func(skeleton *TraceSkeleton) *TraceOverview {
|
||||
overview := &TraceOverview{Identity: skeleton.Identity(), UnmarkedFailures: skeleton.UnmarkedFailurePage(0), Slowest: skeleton.SlowestPage(0), Tree: skeleton.Tree()}
|
||||
overview.Tree.Records = []SpanRecord{{SpanID: "root"}}
|
||||
return overview
|
||||
}
|
||||
|
||||
t.Run("OverviewOverSizeLimit_DropsRecordsThenShortensTheTree", func(t *testing.T) {
|
||||
skeleton := traceWith(strings.Repeat("checkout-", 40))
|
||||
overview := newOverview(skeleton)
|
||||
require.Greater(t, jsonChars(overview), inspectMaxResponseChars)
|
||||
|
||||
skeleton.FitOverview(overview)
|
||||
|
||||
// The tree is cut to what fits and no further: two more spans would pass the limit.
|
||||
assert.LessOrEqual(t, jsonChars(overview), inspectMaxResponseChars)
|
||||
assert.Greater(t, jsonChars(overview), inspectMaxResponseChars-2*jsonChars(overview.Tree.Spans[1]))
|
||||
assert.Nil(t, overview.Tree.Records)
|
||||
assert.Less(t, len(overview.Tree.Spans), inspectTreeSpans)
|
||||
assert.False(t, overview.Tree.Complete)
|
||||
assert.Equal(t, uint64(100), overview.Tree.Total)
|
||||
assert.Equal(t, "root", overview.Tree.Spans[0].SpanID)
|
||||
assert.Len(t, overview.Slowest.Items, inspectListSize)
|
||||
})
|
||||
|
||||
t.Run("OverviewWithinSizeLimit_IsLeftAsItIs", func(t *testing.T) {
|
||||
skeleton := traceWith("checkout")
|
||||
overview := newOverview(skeleton)
|
||||
require.LessOrEqual(t, jsonChars(overview), inspectMaxResponseChars)
|
||||
|
||||
skeleton.FitOverview(overview)
|
||||
|
||||
assert.Len(t, overview.Tree.Spans, inspectTreeSpans)
|
||||
assert.NotNil(t, overview.Tree.Records)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSpanDetail(t *testing.T) {
|
||||
specs := []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 1000},
|
||||
{id: "l1", parent: "root", startMs: 1, durationMs: 900},
|
||||
{id: "l2", parent: "l1", startMs: 2, durationMs: 800},
|
||||
{id: "l3", parent: "l2", startMs: 3, durationMs: 700},
|
||||
{id: "target", parent: "l3", startMs: 4, durationMs: 600},
|
||||
}
|
||||
for i := range 30 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("child-%02d", i), parent: "target", startMs: int64(5 + i), durationMs: 2})
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("grandchild-%02d", i), parent: fmt.Sprintf("child-%02d", i), startMs: int64(5 + i), durationMs: 1})
|
||||
}
|
||||
specs = append(specs, skeletonSpec{id: "great-grandchild", parent: "grandchild-00", startMs: 5, durationMs: 1})
|
||||
skeleton := buildSkeleton(specs...)
|
||||
position := func(spanID string) int32 {
|
||||
idx, ok := skeleton.Index(spanID)
|
||||
require.True(t, ok, spanID)
|
||||
return idx
|
||||
}
|
||||
|
||||
t.Run("DeepSpan_ReturnsNearestAncestorsThenRootAndTheLevelsBelow", func(t *testing.T) {
|
||||
detail := skeleton.SpanDetail(position("target"), 2)
|
||||
|
||||
assert.Equal(t, "target", detail.Summary.SpanID)
|
||||
assert.Equal(t, []string{"l3", "l2", "root"}, summaryIDs(detail.Ancestors))
|
||||
assert.Equal(t, uint64(4), detail.AncestorCount)
|
||||
require.Len(t, detail.Descendants, 2)
|
||||
assert.Equal(t, uint32(1), detail.Descendants[0].Level)
|
||||
assert.Equal(t, uint64(30), detail.Descendants[0].Total)
|
||||
// A level shows its first 20 spans in time order.
|
||||
require.Len(t, detail.Descendants[0].Spans, inspectLevelSize)
|
||||
assert.Equal(t, "child-00", detail.Descendants[0].Spans[0].SpanID)
|
||||
assert.Equal(t, "child-19", detail.Descendants[0].Spans[19].SpanID)
|
||||
assert.Equal(t, uint64(30), detail.Descendants[1].Total)
|
||||
assert.Equal(t, "grandchild-00", detail.Descendants[1].Spans[0].SpanID)
|
||||
})
|
||||
|
||||
t.Run("DepthCoversEveryAncestor_ListsThemOnce", func(t *testing.T) {
|
||||
detail := skeleton.SpanDetail(position("target"), 4)
|
||||
|
||||
assert.Equal(t, []string{"l3", "l2", "l1", "root"}, summaryIDs(detail.Ancestors))
|
||||
require.Len(t, detail.Descendants, 3)
|
||||
assert.Equal(t, uint64(1), detail.Descendants[2].Total)
|
||||
})
|
||||
|
||||
t.Run("RootSpan_HasNoAncestors", func(t *testing.T) {
|
||||
detail := skeleton.SpanDetail(position("root"), 1)
|
||||
|
||||
assert.Empty(t, detail.Ancestors)
|
||||
assert.Equal(t, uint64(0), detail.AncestorCount)
|
||||
require.Len(t, detail.Descendants, 1)
|
||||
assert.Equal(t, []string{"l1"}, summaryIDs(detail.Descendants[0].Spans))
|
||||
})
|
||||
|
||||
t.Run("LeafSpan_HasNoLevelsBelow", func(t *testing.T) {
|
||||
detail := skeleton.SpanDetail(position("great-grandchild"), 2)
|
||||
|
||||
assert.Empty(t, detail.Descendants)
|
||||
assert.Equal(t, uint64(7), detail.AncestorCount)
|
||||
})
|
||||
|
||||
t.Run("LevelBelowAcrossParents_IsInTimeOrder", func(t *testing.T) {
|
||||
twoBranches := buildSkeleton(
|
||||
skeletonSpec{id: "top", startMs: 0, durationMs: 1000},
|
||||
skeletonSpec{id: "first", parent: "top", startMs: 1, durationMs: 900},
|
||||
skeletonSpec{id: "second", parent: "top", startMs: 2, durationMs: 900},
|
||||
skeletonSpec{id: "second-early", parent: "second", startMs: 10, durationMs: 1},
|
||||
skeletonSpec{id: "first-late", parent: "first", startMs: 500, durationMs: 1},
|
||||
)
|
||||
|
||||
detail := twoBranches.SpanDetail(0, 2)
|
||||
|
||||
require.Len(t, detail.Descendants, 2)
|
||||
assert.Equal(t, []string{"second-early", "first-late"}, summaryIDs(detail.Descendants[1].Spans))
|
||||
})
|
||||
}
|
||||
|
||||
func TestSpanPage(t *testing.T) {
|
||||
t.Run("EverySpan_PagesInTreeOrderByCursor", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
for i := range 24 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("s-%02d", i), parent: "root", startMs: int64(i), durationMs: 1})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
seen := make([]string, 0, 25)
|
||||
cursor := ""
|
||||
for range 4 {
|
||||
query, err := NewTraceSpanListQuery(&InspectTraceSpansParams{Limit: 10, Cursor: cursor})
|
||||
require.NoError(t, err)
|
||||
page := skeleton.SpanPage(nil, query.Offset, query.Limit)
|
||||
assert.Equal(t, uint64(25), page.Total)
|
||||
for _, item := range page.Items {
|
||||
assert.Nil(t, item.Parent)
|
||||
assert.Empty(t, item.Matches)
|
||||
seen = append(seen, item.Span.SpanID)
|
||||
}
|
||||
cursor = page.NextCursor
|
||||
if cursor == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
require.Len(t, seen, 25)
|
||||
assert.Equal(t, "root", seen[0])
|
||||
assert.Equal(t, "s-00", seen[1])
|
||||
assert.Equal(t, "s-23", seen[24])
|
||||
})
|
||||
|
||||
t.Run("PageOverSizeLimit_EndsEarlyAndTheCursorContinuesThere", func(t *testing.T) {
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
for i := range 999 {
|
||||
specs = append(specs, skeletonSpec{id: fmt.Sprintf("span-with-a-long-identifier-%04d", i), parent: "root", name: "an operation name of ordinary length", service: "checkout-service", startMs: int64(i), durationMs: 1})
|
||||
}
|
||||
skeleton := buildSkeleton(specs...)
|
||||
|
||||
page := skeleton.SpanPage(nil, 0, inspectMaxLimit)
|
||||
|
||||
assert.Less(t, len(page.Items), 1000)
|
||||
assert.LessOrEqual(t, jsonChars(page.Items), inspectMaxResponseChars)
|
||||
query, err := NewTraceSpanListQuery(&InspectTraceSpansParams{Cursor: page.NextCursor})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(page.Items), query.Offset)
|
||||
})
|
||||
|
||||
t.Run("WithRoot_ListsThatSpanAndTheSpansBelowIt", func(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "before", parent: "root", startMs: 1, durationMs: 1},
|
||||
skeletonSpec{id: "branch", parent: "root", startMs: 2, durationMs: 50},
|
||||
skeletonSpec{id: "leaf-a", parent: "branch", startMs: 3, durationMs: 1},
|
||||
skeletonSpec{id: "leaf-b", parent: "branch", startMs: 4, durationMs: 1},
|
||||
skeletonSpec{id: "after", parent: "root", startMs: 60, durationMs: 1},
|
||||
)
|
||||
branch, ok := skeleton.Index("branch")
|
||||
require.True(t, ok)
|
||||
|
||||
page := skeleton.SpanPage(&branch, 0, 2)
|
||||
query, err := NewTraceSpanListQuery(&InspectTraceSpansParams{Under: "branch", Limit: 2, Cursor: page.NextCursor})
|
||||
require.NoError(t, err)
|
||||
rest := skeleton.SpanPage(&branch, query.Offset, query.Limit)
|
||||
|
||||
assert.Equal(t, uint64(3), page.Total)
|
||||
assert.Equal(t, []string{"branch", "leaf-a"}, []string{page.Items[0].Span.SpanID, page.Items[1].Span.SpanID})
|
||||
require.Len(t, rest.Items, 1)
|
||||
assert.Equal(t, "leaf-b", rest.Items[0].Span.SpanID)
|
||||
assert.Empty(t, rest.NextCursor)
|
||||
})
|
||||
}
|
||||
|
||||
func TestNewTraceSpanList(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", name: "POST /checkout", service: "gateway", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "orders", parent: "root", name: "POST /orders", service: "orders", startMs: 1, durationMs: 90},
|
||||
skeletonSpec{id: "charge", parent: "orders", name: "POST /charge", service: "payments", startMs: 3, durationMs: 70, hasError: true},
|
||||
skeletonSpec{id: "refund", parent: "orders", name: "POST /refund", service: "payments", startMs: 80, durationMs: 5},
|
||||
)
|
||||
charge := recordSpan("charge", map[string]string{"payment.provider": "Worldpay", "payment.note": strings.Repeat("padding ", 20) + "sent to worldpay gateway" + strings.Repeat(" padding", 20)})
|
||||
refund := recordSpan("refund", map[string]string{"payment.provider": "worldpay"})
|
||||
|
||||
t.Run("Matches_CarryTheSpanItsParentAndWhereTheWordsWereFound", func(t *testing.T) {
|
||||
search := &TraceSpanSearch{
|
||||
// The second hit has no record and the third is not in the tree; both are left out.
|
||||
Hits: []ThreadCursor{{TimeUnixNano: 3, SpanID: "charge"}, {TimeUnixNano: 8, SpanID: "orders"}, {TimeUnixNano: 9, SpanID: "not-in-tree"}, {TimeUnixNano: 80, SpanID: "refund"}},
|
||||
Total: 57,
|
||||
Words: []string{"worldpay"},
|
||||
HasMore: true,
|
||||
}
|
||||
|
||||
list := NewTraceSpanList(skeleton, search, []StorableSpan{refund, recordSpan("not-in-tree", nil), charge})
|
||||
|
||||
assert.Equal(t, uint64(57), list.Total)
|
||||
require.Len(t, list.Items, 2)
|
||||
first := list.Items[0]
|
||||
assert.Equal(t, "charge", first.Span.SpanID)
|
||||
assert.Equal(t, uint32(2), first.Span.Level)
|
||||
require.NotNil(t, first.Parent)
|
||||
assert.Equal(t, "orders", first.Parent.SpanID)
|
||||
require.Len(t, first.Matches, 3)
|
||||
assert.Equal(t, SpanTextMatch{Field: "statusMessage", Snippet: "worldpay declined the card"}, first.Matches[0])
|
||||
assert.Equal(t, "attributes.payment.note", first.Matches[1].Field)
|
||||
assert.True(t, strings.HasPrefix(first.Matches[1].Snippet, truncationMark))
|
||||
assert.Contains(t, first.Matches[1].Snippet, "sent to worldpay gateway")
|
||||
assert.Equal(t, SpanTextMatch{Field: "attributes.payment.provider", Snippet: "Worldpay"}, first.Matches[2])
|
||||
assert.Equal(t, "refund", list.Items[1].Span.SpanID)
|
||||
|
||||
// The cursor continues after the last span shown.
|
||||
query, err := NewTraceSpanListQuery(&InspectTraceSpansParams{Search: "worldpay", Cursor: list.NextCursor})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, &ThreadCursor{TimeUnixNano: 80, SpanID: "refund"}, query.After)
|
||||
})
|
||||
|
||||
t.Run("WordInAnAttributeNameOrAResourceAttribute_IsFound", func(t *testing.T) {
|
||||
search := &TraceSpanSearch{Hits: []ThreadCursor{{TimeUnixNano: 80, SpanID: "refund"}}, Total: 1, Words: []string{"provider", "payments"}, Warnings: []string{"key `x` not found"}}
|
||||
|
||||
list := NewTraceSpanList(skeleton, search, []StorableSpan{refund})
|
||||
|
||||
require.Len(t, list.Items, 1)
|
||||
assert.Equal(t, []SpanTextMatch{{Field: "attributes.payment.provider", Snippet: "payment.provider"}, {Field: "resource.service.name", Snippet: "payments"}}, list.Items[0].Matches)
|
||||
assert.Equal(t, []string{"key `x` not found"}, list.Warnings)
|
||||
})
|
||||
|
||||
t.Run("LastPage_HasNoCursor", func(t *testing.T) {
|
||||
search := &TraceSpanSearch{Hits: []ThreadCursor{{TimeUnixNano: 3, SpanID: "charge"}}, Total: 1, Words: []string{"worldpay"}}
|
||||
|
||||
list := NewTraceSpanList(skeleton, search, []StorableSpan{charge})
|
||||
|
||||
assert.Len(t, list.Items, 1)
|
||||
assert.Empty(t, list.NextCursor)
|
||||
})
|
||||
|
||||
t.Run("FilterWithoutWords_HasNoMatchSnippets", func(t *testing.T) {
|
||||
search := &TraceSpanSearch{Hits: []ThreadCursor{{TimeUnixNano: 3, SpanID: "charge"}}, Total: 1, Words: []string{}}
|
||||
|
||||
list := NewTraceSpanList(skeleton, search, []StorableSpan{charge})
|
||||
|
||||
require.Len(t, list.Items, 1)
|
||||
assert.Empty(t, list.Items[0].Matches)
|
||||
assert.NotNil(t, list.Items[0].Matches)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFitSpan(t *testing.T) {
|
||||
summary := SpanSummary{SpanID: "span", Name: strings.Repeat("operation ", 40)}
|
||||
level := func(n uint32) TraceSpanLevel {
|
||||
spans := make([]SpanSummary, inspectLevelSize)
|
||||
for i := range spans {
|
||||
spans[i] = summary
|
||||
}
|
||||
return TraceSpanLevel{Level: n, Total: inspectLevelSize, Spans: spans}
|
||||
}
|
||||
detail := &TraceSpanDetail{Summary: summary, Descendants: []TraceSpanLevel{level(1), level(2), level(3), level(4), level(5), level(6), level(7), level(8)}}
|
||||
require.Greater(t, jsonChars(detail), inspectMaxResponseChars)
|
||||
|
||||
FitSpan(detail)
|
||||
|
||||
// The deepest levels go first, and no more of them than needed.
|
||||
assert.LessOrEqual(t, jsonChars(detail), inspectMaxResponseChars)
|
||||
assert.Greater(t, jsonChars(detail)+jsonChars(level(9)), inspectMaxResponseChars)
|
||||
require.NotEmpty(t, detail.Descendants)
|
||||
assert.Equal(t, uint32(1), detail.Descendants[0].Level)
|
||||
}
|
||||
|
||||
func TestIdentity(t *testing.T) {
|
||||
inputTokens, outputTokens, cost := 1200.0, 300.0, 0.02
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", name: "POST /chat", service: "web", startMs: 0, durationMs: 100, statusCode: "200"},
|
||||
skeletonSpec{id: "agent", parent: "root", name: "invoke_agent", service: "agent", startMs: 10, durationMs: 80},
|
||||
skeletonSpec{id: "model", parent: "agent", name: "chat gpt", service: "agent", startMs: 20, durationMs: 50, hasError: true},
|
||||
skeletonSpec{id: "tool", parent: "agent", name: "execute_tool lookup", service: "agent", startMs: 75, durationMs: 10},
|
||||
).WithAISpans([]AISpanUsage{
|
||||
{SpanID: "model", IsModelCall: true, InputTokens: &inputTokens, OutputTokens: &outputTokens, Cost: &cost},
|
||||
{SpanID: "tool", IsToolCall: true},
|
||||
{SpanID: "agent"},
|
||||
})
|
||||
|
||||
identity := skeleton.Identity()
|
||||
|
||||
assert.Equal(t, uint64(100_000), identity.StartTimestampMillis)
|
||||
assert.Equal(t, uint64(100*time.Millisecond), identity.DurationNano)
|
||||
require.NotNil(t, identity.Root)
|
||||
assert.Equal(t, "POST /chat", identity.Root.Name)
|
||||
assert.Equal(t, uint64(4), identity.SpanCount)
|
||||
assert.Equal(t, uint64(1), identity.ErrorSpanCount)
|
||||
assert.Equal(t, uint64(2), identity.ServiceCount)
|
||||
assert.Equal(t, []TraceServiceStats{
|
||||
{ServiceName: "agent", SpanCount: 3, ErrorSpanCount: 1, OwnTimeNano: uint64(80 * time.Millisecond)},
|
||||
{ServiceName: "web", SpanCount: 1, ErrorSpanCount: 0, OwnTimeNano: uint64(20 * time.Millisecond)},
|
||||
}, identity.Services)
|
||||
require.NotNil(t, identity.AI)
|
||||
assert.Equal(t, uint64(1), identity.AI.ModelCallCount)
|
||||
assert.Equal(t, uint64(1), identity.AI.ToolCallCount)
|
||||
assert.Equal(t, uint64(1200), identity.AI.Tokens.Input)
|
||||
require.NotNil(t, identity.AI.TotalCost)
|
||||
assert.InDelta(t, 0.02, *identity.AI.TotalCost, 1e-9)
|
||||
|
||||
assert.Nil(t, buildSkeleton(skeletonSpec{id: "plain", startMs: 0, durationMs: 1}).Identity().AI)
|
||||
}
|
||||
|
||||
func TestExpensivePage(t *testing.T) {
|
||||
t.Run("WithCost_RanksByCost", func(t *testing.T) {
|
||||
cheap, costly, manyTokens := 0.01, 0.5, 9000.0
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "cheap", parent: "root", startMs: 1, durationMs: 10},
|
||||
skeletonSpec{id: "costly", parent: "root", startMs: 2, durationMs: 10},
|
||||
).WithAISpans([]AISpanUsage{
|
||||
{SpanID: "cheap", IsModelCall: true, InputTokens: &manyTokens, Cost: &cheap},
|
||||
{SpanID: "costly", IsModelCall: true, Cost: &costly},
|
||||
})
|
||||
|
||||
expensive := skeleton.ExpensivePage(0)
|
||||
|
||||
require.NotNil(t, expensive)
|
||||
assert.Equal(t, uint64(2), expensive.Total)
|
||||
assert.Equal(t, []string{"costly", "cheap"}, []string{expensive.Items[0].Span.SpanID, expensive.Items[1].Span.SpanID})
|
||||
require.NotNil(t, expensive.Items[0].Parent)
|
||||
assert.Equal(t, "root", expensive.Items[0].Parent.SpanID)
|
||||
assert.Equal(t, uint64(9000), expensive.Items[1].InputTokens)
|
||||
})
|
||||
|
||||
t.Run("WithoutCost_RanksByTokens", func(t *testing.T) {
|
||||
few, many := 10.0, 9000.0
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "small", startMs: 0, durationMs: 10},
|
||||
skeletonSpec{id: "large", startMs: 1, durationMs: 10},
|
||||
).WithAISpans([]AISpanUsage{
|
||||
{SpanID: "small", IsModelCall: true, InputTokens: &few},
|
||||
{SpanID: "large", IsModelCall: true, InputTokens: &many},
|
||||
{SpanID: "not-loaded", IsModelCall: true, InputTokens: &many},
|
||||
})
|
||||
|
||||
expensive := skeleton.ExpensivePage(0)
|
||||
|
||||
assert.Equal(t, uint64(2), expensive.Total)
|
||||
assert.Equal(t, []string{"large", "small"}, []string{expensive.Items[0].Span.SpanID, expensive.Items[1].Span.SpanID})
|
||||
assert.Nil(t, expensive.Items[0].Cost)
|
||||
})
|
||||
|
||||
t.Run("MoreThanOnePage_PagesEqualCostsInSpanOrder", func(t *testing.T) {
|
||||
cost := 0.25
|
||||
specs := []skeletonSpec{{id: "root", startMs: 0, durationMs: 1000}}
|
||||
usage := make([]AISpanUsage, 0, 14)
|
||||
for i := range 14 {
|
||||
id := fmt.Sprintf("call-%02d", i)
|
||||
specs = append(specs, skeletonSpec{id: id, parent: "root", startMs: int64(i + 1), durationMs: 1})
|
||||
// Given in reverse so the page order has to come from the spans, not from the input.
|
||||
usage = append([]AISpanUsage{{SpanID: id, IsModelCall: true, Cost: &cost}}, usage...)
|
||||
}
|
||||
skeleton := buildSkeleton(specs...).WithAISpans(usage)
|
||||
|
||||
first := skeleton.ExpensivePage(0)
|
||||
second := skeleton.ExpensivePage(nextOffset(t, OverviewListExpensive, first.NextCursor))
|
||||
|
||||
assert.Equal(t, uint64(14), first.Total)
|
||||
assert.Equal(t, "call-00", first.Items[0].Span.SpanID)
|
||||
assert.Equal(t, "call-09", first.Items[9].Span.SpanID)
|
||||
require.Len(t, second.Items, 4)
|
||||
assert.Equal(t, "call-13", second.Items[3].Span.SpanID)
|
||||
assert.Empty(t, second.NextCursor)
|
||||
})
|
||||
|
||||
t.Run("NoGenAISpans_ReturnsNothing", func(t *testing.T) {
|
||||
assert.Nil(t, buildSkeleton(skeletonSpec{id: "plain", startMs: 0, durationMs: 1}).ExpensivePage(0))
|
||||
})
|
||||
}
|
||||
467
pkg/types/spantypes/trace_skeleton.go
Normal file
467
pkg/types/spantypes/trace_skeleton.go
Normal file
@@ -0,0 +1,467 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
skeletonFlagError uint8 = 1 << iota
|
||||
skeletonFlagExceptionEvent
|
||||
skeletonFlagMissingParent
|
||||
)
|
||||
|
||||
const noParent int32 = -1
|
||||
|
||||
// SkeletonSpan is the ClickHouse scan struct for the columns a TraceSkeleton is built from.
|
||||
type SkeletonSpan struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
ParentSpanID string `ch:"parent_span_id"`
|
||||
StartTime time.Time `ch:"timestamp"`
|
||||
DurationNano uint64 `ch:"duration_nano"`
|
||||
HasError bool `ch:"has_error"`
|
||||
Name string `ch:"name"`
|
||||
ServiceName string `ch:"resource_string_service$$name"`
|
||||
KindString string `ch:"kind_string"`
|
||||
ResponseStatusCode string `ch:"response_status_code"`
|
||||
HasExceptionEvent bool `ch:"has_exception_event"`
|
||||
}
|
||||
|
||||
// AISpanUsage is the token and cost usage of one gen_ai span.
|
||||
type AISpanUsage struct {
|
||||
SpanID string `ch:"span_id"`
|
||||
IsModelCall bool `ch:"is_model_call"`
|
||||
IsToolCall bool `ch:"is_tool_call"`
|
||||
InputTokens *float64 `ch:"input_tokens_value"`
|
||||
OutputTokens *float64 `ch:"output_tokens_value"`
|
||||
CacheReadTokens *float64 `ch:"cache_read_tokens_value"`
|
||||
CacheWriteTokens *float64 `ch:"cache_write_tokens_value"`
|
||||
ReasoningTokens *float64 `ch:"reasoning_tokens_value"`
|
||||
Cost *float64 `ch:"total_cost_value"`
|
||||
}
|
||||
|
||||
// LinkedSpan is a span that links to other spans, in its own trace or another one.
|
||||
type LinkedSpan struct {
|
||||
SpanID string
|
||||
Links []Link
|
||||
}
|
||||
|
||||
// TraceSkeleton holds the tree of a trace as parallel arrays indexed by span position, in start
|
||||
// time order. It is read-only once built.
|
||||
type TraceSkeleton struct {
|
||||
TraceID string
|
||||
|
||||
// SpanIDs holds every span id back to back; span i is SpanIDs[IDOffsets[i]:IDOffsets[i+1]].
|
||||
SpanIDs string
|
||||
IDOffsets []uint32
|
||||
// Parents holds the position of each span's parent, or -1 for a root.
|
||||
Parents []int32
|
||||
StartNano []int64
|
||||
DurationNano []uint64
|
||||
Flags []uint8
|
||||
// Names, Services, Kinds and StatusCodes index into Strings.
|
||||
Names []uint32
|
||||
Services []uint32
|
||||
Kinds []uint32
|
||||
StatusCodes []uint32
|
||||
Strings []string
|
||||
|
||||
AISpans []AISpanUsage
|
||||
// LinkedSpans holds the spans with links, in span order.
|
||||
LinkedSpans []LinkedSpan
|
||||
|
||||
// Children of span i are childList[childStart[i]:childStart[i+1]], in span order.
|
||||
childStart []int32
|
||||
childList []int32
|
||||
roots []int32
|
||||
// preorder lists span positions depth first, roots and siblings in span order.
|
||||
preorder []int32
|
||||
level []uint32
|
||||
// subtree counts the spans below each span, the span itself excluded.
|
||||
subtree []uint32
|
||||
ownTime []uint64
|
||||
// idOrder lists span positions sorted by span id, for lookups by id.
|
||||
idOrder []int32
|
||||
startNs int64
|
||||
endNs int64
|
||||
errCount uint64
|
||||
}
|
||||
|
||||
// TraceSkeletonBuilder collects spans in (start time, name, span id) order.
|
||||
type TraceSkeletonBuilder struct {
|
||||
skeleton *TraceSkeleton
|
||||
ids []byte
|
||||
parentIDs []string
|
||||
interned map[string]uint32
|
||||
}
|
||||
|
||||
// NewLinkedSpan reads the links out of a span's stored references; ok is false when it has none.
|
||||
func NewLinkedSpan(spanID, references string) (LinkedSpan, bool) {
|
||||
refs := []OtelSpanRef{}
|
||||
if err := json.Unmarshal([]byte(references), &refs); err != nil {
|
||||
return LinkedSpan{}, false
|
||||
}
|
||||
linked := LinkedSpan{SpanID: spanID, Links: make([]Link, 0, len(refs))}
|
||||
for _, ref := range refs {
|
||||
// The parent is stored as a CHILD_OF reference beside the links.
|
||||
if ref.RefType != "CHILD_OF" {
|
||||
linked.Links = append(linked.Links, Link{TraceID: ref.TraceId, SpanID: ref.SpanId})
|
||||
}
|
||||
}
|
||||
return linked, len(linked.Links) > 0
|
||||
}
|
||||
|
||||
func NewTraceSkeletonBuilder(traceID string, sizeHint int) *TraceSkeletonBuilder {
|
||||
return &TraceSkeletonBuilder{
|
||||
skeleton: &TraceSkeleton{
|
||||
TraceID: traceID,
|
||||
IDOffsets: make([]uint32, 1, sizeHint+1),
|
||||
Parents: make([]int32, 0, sizeHint),
|
||||
StartNano: make([]int64, 0, sizeHint),
|
||||
DurationNano: make([]uint64, 0, sizeHint),
|
||||
Flags: make([]uint8, 0, sizeHint),
|
||||
Names: make([]uint32, 0, sizeHint),
|
||||
Services: make([]uint32, 0, sizeHint),
|
||||
Kinds: make([]uint32, 0, sizeHint),
|
||||
StatusCodes: make([]uint32, 0, sizeHint),
|
||||
AISpans: []AISpanUsage{},
|
||||
LinkedSpans: []LinkedSpan{},
|
||||
},
|
||||
ids: make([]byte, 0, sizeHint*16),
|
||||
parentIDs: make([]string, 0, sizeHint),
|
||||
interned: make(map[string]uint32),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *TraceSkeletonBuilder) Add(span *SkeletonSpan) {
|
||||
s := b.skeleton
|
||||
b.ids = append(b.ids, span.SpanID...)
|
||||
s.IDOffsets = append(s.IDOffsets, uint32(len(b.ids)))
|
||||
b.parentIDs = append(b.parentIDs, span.ParentSpanID)
|
||||
s.StartNano = append(s.StartNano, span.StartTime.UnixNano())
|
||||
s.DurationNano = append(s.DurationNano, span.DurationNano)
|
||||
var flags uint8
|
||||
if span.HasError {
|
||||
flags |= skeletonFlagError
|
||||
}
|
||||
if span.HasExceptionEvent {
|
||||
flags |= skeletonFlagExceptionEvent
|
||||
}
|
||||
s.Flags = append(s.Flags, flags)
|
||||
s.Names = append(s.Names, b.intern(span.Name))
|
||||
s.Services = append(s.Services, b.intern(span.ServiceName))
|
||||
s.Kinds = append(s.Kinds, b.intern(span.KindString))
|
||||
s.StatusCodes = append(s.StatusCodes, b.intern(span.ResponseStatusCode))
|
||||
}
|
||||
|
||||
func (b *TraceSkeletonBuilder) Len() int {
|
||||
return len(b.parentIDs)
|
||||
}
|
||||
|
||||
// Build links each span to its parent and computes the tree fields. The builder cannot be reused.
|
||||
func (b *TraceSkeletonBuilder) Build() *TraceSkeleton {
|
||||
s := b.skeleton
|
||||
s.SpanIDs = string(b.ids)
|
||||
s.buildIDOrder()
|
||||
|
||||
n := len(b.parentIDs)
|
||||
s.Parents = s.Parents[:0]
|
||||
for i := range n {
|
||||
parent := noParent
|
||||
if parentID := b.parentIDs[i]; parentID != "" {
|
||||
if idx, ok := s.Index(parentID); ok && idx != int32(i) {
|
||||
parent = idx
|
||||
} else {
|
||||
s.Flags[i] |= skeletonFlagMissingParent
|
||||
}
|
||||
}
|
||||
s.Parents = append(s.Parents, parent)
|
||||
}
|
||||
b.parentIDs, b.interned, b.ids = nil, nil, nil
|
||||
s.derive()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) Len() int {
|
||||
return len(s.Parents)
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) SpanID(i int32) string {
|
||||
return s.SpanIDs[s.IDOffsets[i]:s.IDOffsets[i+1]]
|
||||
}
|
||||
|
||||
// Index returns the position of the span with the given id.
|
||||
func (s *TraceSkeleton) Index(spanID string) (int32, bool) {
|
||||
pos, found := sort.Find(len(s.idOrder), func(k int) int {
|
||||
id := s.SpanID(s.idOrder[k])
|
||||
switch {
|
||||
case spanID < id:
|
||||
return -1
|
||||
case spanID > id:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
if !found {
|
||||
return 0, false
|
||||
}
|
||||
return s.idOrder[pos], true
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) Children(i int32) []int32 {
|
||||
return s.childList[s.childStart[i]:s.childStart[i+1]]
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasError(i int32) bool {
|
||||
return s.Flags[i]&skeletonFlagError != 0
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasExceptionEvent(i int32) bool {
|
||||
return s.Flags[i]&skeletonFlagExceptionEvent != 0
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasMissingParent(i int32) bool {
|
||||
return s.Flags[i]&skeletonFlagMissingParent != 0
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) ResponseStatusCode(i int32) string {
|
||||
return s.Strings[s.StatusCodes[i]]
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) HasMissingSpans() bool {
|
||||
for _, root := range s.roots {
|
||||
if s.HasMissingParent(root) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// PrimaryRoot is the earliest span without a parent, or the earliest orphan when every root's
|
||||
// parent is missing.
|
||||
func (s *TraceSkeleton) PrimaryRoot() (int32, bool) {
|
||||
if len(s.roots) == 0 {
|
||||
return 0, false
|
||||
}
|
||||
for _, root := range s.roots {
|
||||
if !s.HasMissingParent(root) {
|
||||
return root, true
|
||||
}
|
||||
}
|
||||
return s.roots[0], true
|
||||
}
|
||||
|
||||
// Ancestors lists the positions above span i, the parent first.
|
||||
func (s *TraceSkeleton) Ancestors(i int32) []int32 {
|
||||
ancestors := make([]int32, 0, s.level[i])
|
||||
for p := s.Parents[i]; p != noParent; p = s.Parents[p] {
|
||||
ancestors = append(ancestors, p)
|
||||
}
|
||||
return ancestors
|
||||
}
|
||||
|
||||
// Summary returns the short form of span i.
|
||||
func (s *TraceSkeleton) Summary(i int32) SpanSummary {
|
||||
name := s.Strings[s.Names[i]]
|
||||
if cut, _, ok := cutString(name, inspectNameChars); ok {
|
||||
name = cut
|
||||
}
|
||||
summary := SpanSummary{
|
||||
SpanID: s.SpanID(i),
|
||||
Name: name,
|
||||
ServiceName: s.Strings[s.Services[i]],
|
||||
Kind: s.Strings[s.Kinds[i]],
|
||||
StartOffsetNano: uint64(s.StartNano[i] - s.startNs),
|
||||
DurationNano: s.DurationNano[i],
|
||||
OwnTimeNano: s.ownTime[i],
|
||||
ResponseStatusCode: s.ResponseStatusCode(i),
|
||||
HasError: s.HasError(i),
|
||||
Level: s.level[i],
|
||||
ChildCount: uint32(s.childStart[i+1] - s.childStart[i]),
|
||||
SubtreeSpanCount: s.subtree[i],
|
||||
}
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
summary.ParentSpanID = s.SpanID(parent)
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
// WithAISpans returns a copy of the skeleton carrying the usage of its gen_ai spans.
|
||||
func (s *TraceSkeleton) WithAISpans(aiSpans []AISpanUsage) *TraceSkeleton {
|
||||
cp := *s
|
||||
cp.AISpans = aiSpans
|
||||
if cp.AISpans == nil {
|
||||
cp.AISpans = []AISpanUsage{}
|
||||
}
|
||||
return &cp
|
||||
}
|
||||
|
||||
// WithLinkedSpans returns a copy of the skeleton carrying its spans that have links, in span order.
|
||||
func (s *TraceSkeleton) WithLinkedSpans(linkedSpans []LinkedSpan) *TraceSkeleton {
|
||||
type positioned struct {
|
||||
index int32
|
||||
linked LinkedSpan
|
||||
}
|
||||
known := make([]positioned, 0, len(linkedSpans))
|
||||
for _, linked := range linkedSpans {
|
||||
if idx, ok := s.Index(linked.SpanID); ok {
|
||||
known = append(known, positioned{index: idx, linked: linked})
|
||||
}
|
||||
}
|
||||
slices.SortFunc(known, func(a, b positioned) int { return int(a.index) - int(b.index) })
|
||||
|
||||
cp := *s
|
||||
cp.LinkedSpans = make([]LinkedSpan, 0, len(known))
|
||||
for _, item := range known {
|
||||
cp.LinkedSpans = append(cp.LinkedSpans, item.linked)
|
||||
}
|
||||
return &cp
|
||||
}
|
||||
|
||||
func (b *TraceSkeletonBuilder) intern(value string) uint32 {
|
||||
if idx, ok := b.interned[value]; ok {
|
||||
return idx
|
||||
}
|
||||
idx := uint32(len(b.skeleton.Strings))
|
||||
b.skeleton.Strings = append(b.skeleton.Strings, value)
|
||||
b.interned[value] = idx
|
||||
return idx
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) buildIDOrder() {
|
||||
n := len(s.IDOffsets) - 1
|
||||
s.idOrder = make([]int32, n)
|
||||
for i := range s.idOrder {
|
||||
s.idOrder[i] = int32(i)
|
||||
}
|
||||
slices.SortFunc(s.idOrder, func(a, b int32) int {
|
||||
idA, idB := s.SpanID(a), s.SpanID(b)
|
||||
switch {
|
||||
case idA < idB:
|
||||
return -1
|
||||
case idA > idB:
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) derive() {
|
||||
n := s.Len()
|
||||
s.startNs, s.endNs, s.errCount = 0, 0, 0
|
||||
for i := range n {
|
||||
if i == 0 || s.StartNano[i] < s.startNs {
|
||||
s.startNs = s.StartNano[i]
|
||||
}
|
||||
s.endNs = max(s.endNs, s.StartNano[i]+int64(s.DurationNano[i]))
|
||||
if s.Flags[i]&skeletonFlagError != 0 {
|
||||
s.errCount++
|
||||
}
|
||||
}
|
||||
|
||||
s.buildChildren()
|
||||
// A span left unvisited sits on a parent cycle; cutting its parent link makes it a root.
|
||||
if s.walk() < n {
|
||||
visited := make([]bool, n)
|
||||
for _, i := range s.preorder {
|
||||
visited[i] = true
|
||||
}
|
||||
for i := range n {
|
||||
if visited[i] {
|
||||
continue
|
||||
}
|
||||
s.Parents[i] = noParent
|
||||
s.Flags[i] |= skeletonFlagMissingParent
|
||||
stack := []int32{int32(i)}
|
||||
for len(stack) > 0 {
|
||||
top := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
if visited[top] {
|
||||
continue
|
||||
}
|
||||
visited[top] = true
|
||||
stack = append(stack, s.Children(top)...)
|
||||
}
|
||||
}
|
||||
s.buildChildren()
|
||||
s.walk()
|
||||
}
|
||||
|
||||
s.subtree = make([]uint32, n)
|
||||
for k := n - 1; k >= 0; k-- {
|
||||
i := s.preorder[k]
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
s.subtree[parent] += s.subtree[i] + 1
|
||||
}
|
||||
}
|
||||
|
||||
s.ownTime = make([]uint64, n)
|
||||
for i := range n {
|
||||
s.ownTime[i] = s.DurationNano[i] - s.childrenCoverNano(int32(i))
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TraceSkeleton) buildChildren() {
|
||||
n := s.Len()
|
||||
s.childStart = make([]int32, n+1)
|
||||
s.roots = s.roots[:0]
|
||||
for i := range n {
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
s.childStart[parent+1]++
|
||||
} else {
|
||||
s.roots = append(s.roots, int32(i))
|
||||
}
|
||||
}
|
||||
for i := range n {
|
||||
s.childStart[i+1] += s.childStart[i]
|
||||
}
|
||||
s.childList = make([]int32, s.childStart[n])
|
||||
next := slices.Clone(s.childStart[:n])
|
||||
for i := range n {
|
||||
if parent := s.Parents[i]; parent != noParent {
|
||||
s.childList[next[parent]] = int32(i)
|
||||
next[parent]++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// walk fills preorder and level from the roots and returns the number of spans reached.
|
||||
func (s *TraceSkeleton) walk() int {
|
||||
n := s.Len()
|
||||
s.preorder = make([]int32, 0, n)
|
||||
s.level = make([]uint32, n)
|
||||
stack := make([]int32, 0, 64)
|
||||
for r := len(s.roots) - 1; r >= 0; r-- {
|
||||
stack = append(stack, s.roots[r])
|
||||
}
|
||||
for len(stack) > 0 {
|
||||
i := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
s.preorder = append(s.preorder, i)
|
||||
children := s.Children(i)
|
||||
for c := len(children) - 1; c >= 0; c-- {
|
||||
s.level[children[c]] = s.level[i] + 1
|
||||
stack = append(stack, children[c])
|
||||
}
|
||||
}
|
||||
return len(s.preorder)
|
||||
}
|
||||
|
||||
// childrenCoverNano is the time of span i its children cover, overlapping children counted once.
|
||||
func (s *TraceSkeleton) childrenCoverNano(i int32) uint64 {
|
||||
start := s.StartNano[i]
|
||||
end := start + int64(s.DurationNano[i])
|
||||
covered, cursor := int64(0), start
|
||||
for _, child := range s.Children(i) {
|
||||
childStart := max(s.StartNano[child], cursor)
|
||||
childEnd := min(s.StartNano[child]+int64(s.DurationNano[child]), end)
|
||||
if childEnd <= childStart {
|
||||
continue
|
||||
}
|
||||
covered += childEnd - childStart
|
||||
cursor = childEnd
|
||||
}
|
||||
return uint64(covered)
|
||||
}
|
||||
227
pkg/types/spantypes/trace_skeleton_test.go
Normal file
227
pkg/types/spantypes/trace_skeleton_test.go
Normal file
@@ -0,0 +1,227 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type skeletonSpec struct {
|
||||
id, parent, name, service string
|
||||
startMs, durationMs int64
|
||||
hasError, hasException bool
|
||||
statusCode string
|
||||
}
|
||||
|
||||
// buildSkeleton adds the spans in the given order, which stands for the store's start time order.
|
||||
func buildSkeleton(specs ...skeletonSpec) *TraceSkeleton {
|
||||
builder := NewTraceSkeletonBuilder("trace", len(specs))
|
||||
for _, spec := range specs {
|
||||
builder.Add(&SkeletonSpan{
|
||||
SpanID: spec.id,
|
||||
ParentSpanID: spec.parent,
|
||||
StartTime: time.Unix(100, 0).Add(time.Duration(spec.startMs) * time.Millisecond),
|
||||
DurationNano: uint64(spec.durationMs) * uint64(time.Millisecond),
|
||||
HasError: spec.hasError,
|
||||
Name: spec.name,
|
||||
ServiceName: spec.service,
|
||||
KindString: "Internal",
|
||||
ResponseStatusCode: spec.statusCode,
|
||||
HasExceptionEvent: spec.hasException,
|
||||
})
|
||||
}
|
||||
return builder.Build()
|
||||
}
|
||||
|
||||
func ownTimeMs(t *testing.T, skeleton *TraceSkeleton, spanID string) int64 {
|
||||
t.Helper()
|
||||
idx, ok := skeleton.Index(spanID)
|
||||
require.True(t, ok, "span %s", spanID)
|
||||
return int64(skeleton.ownTime[idx] / uint64(time.Millisecond))
|
||||
}
|
||||
|
||||
func TestTraceSkeletonOwnTime(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
specs []skeletonSpec
|
||||
expected int64
|
||||
}{
|
||||
{
|
||||
name: "SequentialChildren_SubtractsEach",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 100},
|
||||
{id: "a", parent: "root", startMs: 10, durationMs: 20},
|
||||
{id: "b", parent: "root", startMs: 50, durationMs: 30},
|
||||
},
|
||||
expected: 50,
|
||||
},
|
||||
{
|
||||
name: "ParallelChildren_CountsOverlapOnce",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 100},
|
||||
{id: "a", parent: "root", startMs: 10, durationMs: 40},
|
||||
{id: "b", parent: "root", startMs: 20, durationMs: 40},
|
||||
{id: "c", parent: "root", startMs: 30, durationMs: 10},
|
||||
},
|
||||
expected: 50,
|
||||
},
|
||||
{
|
||||
name: "ChildOutlivesParent_ClipsToParentEnd",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 100},
|
||||
{id: "a", parent: "root", startMs: 80, durationMs: 500},
|
||||
},
|
||||
expected: 80,
|
||||
},
|
||||
{
|
||||
name: "ChildStartsBeforeParent_ClipsToParentStart",
|
||||
specs: []skeletonSpec{
|
||||
{id: "early", parent: "root", startMs: 0, durationMs: 30},
|
||||
{id: "root", startMs: 10, durationMs: 100},
|
||||
},
|
||||
expected: 80,
|
||||
},
|
||||
{
|
||||
name: "ChildCoversParent_LeavesZero",
|
||||
specs: []skeletonSpec{
|
||||
{id: "early", parent: "root", startMs: 0, durationMs: 500},
|
||||
{id: "root", startMs: 10, durationMs: 100},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "NoChildren_EqualsDuration",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 70},
|
||||
},
|
||||
expected: 70,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
skeleton := buildSkeleton(testCase.specs...)
|
||||
assert.Equal(t, testCase.expected, ownTimeMs(t, skeleton, "root"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTraceSkeletonTree(t *testing.T) {
|
||||
skeleton := buildSkeleton(
|
||||
skeletonSpec{id: "root", name: "GET /", service: "web", startMs: 0, durationMs: 100},
|
||||
skeletonSpec{id: "a", parent: "root", name: "a", service: "web", startMs: 10, durationMs: 30},
|
||||
skeletonSpec{id: "a1", parent: "a", name: "a1", service: "db", startMs: 15, durationMs: 5, hasError: true},
|
||||
skeletonSpec{id: "b", parent: "root", name: "b", service: "web", startMs: 50, durationMs: 10},
|
||||
skeletonSpec{id: "orphan", parent: "gone", name: "orphan", service: "worker", startMs: 60, durationMs: 10},
|
||||
)
|
||||
|
||||
order := make([]string, 0, skeleton.Len())
|
||||
for _, i := range skeleton.preorder {
|
||||
order = append(order, skeleton.SpanID(i))
|
||||
}
|
||||
assert.Equal(t, []string{"root", "a", "a1", "b", "orphan"}, order)
|
||||
assert.Len(t, skeleton.roots, 2)
|
||||
assert.True(t, skeleton.HasMissingSpans())
|
||||
assert.Equal(t, uint64(1), skeleton.errCount)
|
||||
|
||||
root, ok := skeleton.PrimaryRoot()
|
||||
require.True(t, ok)
|
||||
rootSummary := skeleton.Summary(root)
|
||||
assert.Equal(t, "root", rootSummary.SpanID)
|
||||
assert.Equal(t, uint32(2), rootSummary.ChildCount)
|
||||
assert.Equal(t, uint32(3), rootSummary.SubtreeSpanCount)
|
||||
|
||||
leaf, ok := skeleton.Index("a1")
|
||||
require.True(t, ok)
|
||||
leafSummary := skeleton.Summary(leaf)
|
||||
assert.Equal(t, uint32(2), leafSummary.Level)
|
||||
assert.Equal(t, "a", leafSummary.ParentSpanID)
|
||||
assert.Equal(t, uint64(15*time.Millisecond), leafSummary.StartOffsetNano)
|
||||
assert.True(t, leafSummary.HasError)
|
||||
assert.Equal(t, "db", leafSummary.ServiceName)
|
||||
|
||||
_, ok = skeleton.Index("gone")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestTraceSkeletonSummary_CutsALongName(t *testing.T) {
|
||||
statement := strings.Repeat("SELECT id FROM orders WHERE region = ? AND ", 20)
|
||||
skeleton := buildSkeleton(skeletonSpec{id: "query", name: statement, service: "db", startMs: 0, durationMs: 5})
|
||||
|
||||
summary := skeleton.Summary(0)
|
||||
|
||||
assert.Equal(t, statement[:inspectNameChars]+truncationMark, summary.Name)
|
||||
}
|
||||
|
||||
func TestTraceSkeletonPrimaryRoot(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
specs []skeletonSpec
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "OrphanStartsFirst_PrefersSpanWithoutParent",
|
||||
specs: []skeletonSpec{
|
||||
{id: "orphan", parent: "gone", startMs: 0, durationMs: 10},
|
||||
{id: "root", startMs: 5, durationMs: 100},
|
||||
},
|
||||
expected: "root",
|
||||
},
|
||||
{
|
||||
name: "OnlyOrphans_TakesEarliest",
|
||||
specs: []skeletonSpec{
|
||||
{id: "first", parent: "gone", startMs: 0, durationMs: 10},
|
||||
{id: "second", parent: "missing", startMs: 5, durationMs: 100},
|
||||
},
|
||||
expected: "first",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
skeleton := buildSkeleton(testCase.specs...)
|
||||
root, ok := skeleton.PrimaryRoot()
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, testCase.expected, skeleton.SpanID(root))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTraceSkeletonParentCycle(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
specs []skeletonSpec
|
||||
}{
|
||||
{
|
||||
name: "TwoSpansPointAtEachOther",
|
||||
specs: []skeletonSpec{
|
||||
{id: "root", startMs: 0, durationMs: 10},
|
||||
{id: "x", parent: "y", startMs: 1, durationMs: 5},
|
||||
{id: "y", parent: "x", startMs: 2, durationMs: 5},
|
||||
{id: "under-y", parent: "y", startMs: 3, durationMs: 1},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "SpanIsItsOwnParent",
|
||||
specs: []skeletonSpec{
|
||||
{id: "self", parent: "self", startMs: 0, durationMs: 10},
|
||||
{id: "child", parent: "self", startMs: 1, durationMs: 5},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
skeleton := buildSkeleton(testCase.specs...)
|
||||
assert.Len(t, skeleton.preorder, len(testCase.specs))
|
||||
total := 0
|
||||
for _, root := range skeleton.roots {
|
||||
total += int(skeleton.Summary(root).SubtreeSpanCount) + 1
|
||||
}
|
||||
assert.Equal(t, len(testCase.specs), total)
|
||||
})
|
||||
}
|
||||
}
|
||||
298
pkg/types/spantypes/trace_span_record.go
Normal file
298
pkg/types/spantypes/trace_span_record.go
Normal file
@@ -0,0 +1,298 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
spanRecordValueChars = 500
|
||||
spanRecordChars = 20_000
|
||||
spanRecordShrunkChars = 100
|
||||
spanRecordMinimumChars = 20
|
||||
spanRecordSelectChars = 40_000
|
||||
// spanRecordSelectMaxChars leaves room in the response for the span's summary and ancestors.
|
||||
spanRecordSelectMaxChars = 44_000
|
||||
spanMatchSnippetRadius = 40
|
||||
spanMatchLimit = 3
|
||||
truncationMark = "…"
|
||||
)
|
||||
|
||||
// TruncatedField names a value of a span record that was cut, with its full length in characters.
|
||||
type TruncatedField struct {
|
||||
Field string `json:"field" required:"true"`
|
||||
Length uint64 `json:"length" required:"true"`
|
||||
}
|
||||
|
||||
// SpanRecord is everything stored on a span. Long values are cut and listed in Truncated.
|
||||
type SpanRecord struct {
|
||||
SpanID string `json:"spanId" required:"true"`
|
||||
StatusCode string `json:"statusCode" required:"true"`
|
||||
StatusMessage string `json:"statusMessage" required:"true"`
|
||||
Attributes map[string]any `json:"attributes" required:"true" nullable:"false"`
|
||||
Resource map[string]string `json:"resource" required:"true" nullable:"false"`
|
||||
Events []EventV2 `json:"events" required:"true" nullable:"false"`
|
||||
// Links holds the spans this span links to, in this or another trace.
|
||||
Links []Link `json:"links" required:"true" nullable:"false"`
|
||||
Truncated []TruncatedField `json:"truncated" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
// NewSpanRecord cuts values over 500 characters, and shorter still while the record is over its size limit.
|
||||
// With selectFields the record holds only those attributes, in full as far as the response allows.
|
||||
func NewSpanRecord(item *StorableSpan, selectFields []string) SpanRecord {
|
||||
record := SpanRecord{
|
||||
SpanID: item.SpanID,
|
||||
StatusCode: item.StatusCodeString,
|
||||
StatusMessage: item.StatusMessage,
|
||||
Attributes: item.Attributes(),
|
||||
Resource: make(map[string]string, len(item.ResourcesString)),
|
||||
Events: make([]EventV2, 0, len(item.Events)),
|
||||
Links: make([]Link, 0),
|
||||
Truncated: []TruncatedField{},
|
||||
}
|
||||
for key, value := range item.ResourcesString {
|
||||
record.Resource[key] = value
|
||||
}
|
||||
for _, event := range item.UnmarshalledEvents() {
|
||||
record.Events = append(record.Events, EventV2{Name: event.Name, TimeUnixNano: event.TimeUnixNano, Attributes: event.AttributeMap, IsError: event.IsError})
|
||||
}
|
||||
for _, ref := range item.UnmarshalledRefs() {
|
||||
if ref.RefType != "CHILD_OF" {
|
||||
record.Links = append(record.Links, Link{TraceID: ref.TraceId, SpanID: ref.SpanId})
|
||||
}
|
||||
}
|
||||
|
||||
if len(selectFields) > 0 {
|
||||
record.selectOnly(selectFields)
|
||||
} else {
|
||||
record.cut(spanRecordValueChars)
|
||||
for _, limit := range []int{spanRecordShrunkChars, spanRecordMinimumChars} {
|
||||
if jsonChars(record) <= spanRecordChars {
|
||||
break
|
||||
}
|
||||
record.cut(limit)
|
||||
}
|
||||
}
|
||||
slices.SortFunc(record.Truncated, func(a, b TruncatedField) int { return strings.Compare(a.Field, b.Field) })
|
||||
return record
|
||||
}
|
||||
|
||||
// SmallTraceRecords returns the full record of every span when the trace is small enough for all
|
||||
// of them to fit in the response; ok is false otherwise.
|
||||
func SmallTraceRecords(spans []StorableSpan) ([]SpanRecord, bool) {
|
||||
records := make([]SpanRecord, 0, len(spans))
|
||||
chars := 0
|
||||
for i := range spans {
|
||||
record := NewSpanRecord(&spans[i], nil)
|
||||
chars += jsonChars(record)
|
||||
if chars > inspectSmallTraceChars {
|
||||
return nil, false
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, true
|
||||
}
|
||||
|
||||
// IsSmallTrace reports whether a trace has few enough spans to try returning their records.
|
||||
func IsSmallTrace(spanCount int) bool {
|
||||
return spanCount <= inspectTreeSpans
|
||||
}
|
||||
|
||||
// selectOnly keeps the named attributes of the span and of its events, and the events that have
|
||||
// one. The status message and the resource follow the usual 500 character cut.
|
||||
func (r *SpanRecord) selectOnly(fields []string) {
|
||||
attributes, events := r.Attributes, r.Events
|
||||
r.Attributes, r.Events = map[string]any{}, []EventV2{}
|
||||
r.cut(spanRecordValueChars)
|
||||
|
||||
lengths := make([]int, 0)
|
||||
keep := func(source map[string]any) map[string]any {
|
||||
kept := make(map[string]any)
|
||||
for _, field := range fields {
|
||||
value, ok := source[field]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
kept[field] = value
|
||||
if str, isString := value.(string); isString {
|
||||
lengths = append(lengths, utf8.RuneCountInString(str))
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
r.Attributes = keep(attributes)
|
||||
for _, event := range events {
|
||||
if kept := keep(event.Attributes); len(kept) > 0 {
|
||||
event.Attributes = kept
|
||||
r.Events = append(r.Events, event)
|
||||
}
|
||||
}
|
||||
|
||||
limit := min(selectLimit(lengths, spanRecordSelectChars), spanRecordSelectChars)
|
||||
r.cut(limit)
|
||||
// The budget counts characters; escaped and multi-byte text takes more room than that in the response.
|
||||
for limit > spanRecordValueChars && jsonChars(r) > spanRecordSelectMaxChars {
|
||||
limit /= 2
|
||||
r.cut(limit)
|
||||
}
|
||||
}
|
||||
|
||||
// selectLimit returns the length each selected value may keep for all of them to fit the budget:
|
||||
// short values keep their length and the long ones share what is left.
|
||||
func selectLimit(lengths []int, budget int) int {
|
||||
slices.Sort(lengths)
|
||||
for i, length := range lengths {
|
||||
share := budget / (len(lengths) - i)
|
||||
if length > share {
|
||||
return share
|
||||
}
|
||||
budget -= length
|
||||
}
|
||||
return math.MaxInt
|
||||
}
|
||||
|
||||
// cut shortens every string value longer than limit and records its full length once.
|
||||
func (r *SpanRecord) cut(limit int) {
|
||||
noted := make(map[string]struct{}, len(r.Truncated))
|
||||
for _, field := range r.Truncated {
|
||||
noted[field.Field] = struct{}{}
|
||||
}
|
||||
note := func(field string, length int) {
|
||||
if _, ok := noted[field]; !ok {
|
||||
noted[field] = struct{}{}
|
||||
r.Truncated = append(r.Truncated, TruncatedField{Field: field, Length: uint64(length)})
|
||||
}
|
||||
}
|
||||
|
||||
if cutValue, length, ok := cutString(r.StatusMessage, limit); ok {
|
||||
r.StatusMessage = cutValue
|
||||
note("statusMessage", length)
|
||||
}
|
||||
for key, value := range r.Attributes {
|
||||
if str, isString := value.(string); isString {
|
||||
if cutValue, length, ok := cutString(str, limit); ok {
|
||||
r.Attributes[key] = cutValue
|
||||
note("attributes."+key, length)
|
||||
}
|
||||
}
|
||||
}
|
||||
for key, value := range r.Resource {
|
||||
if cutValue, length, ok := cutString(value, limit); ok {
|
||||
r.Resource[key] = cutValue
|
||||
note("resource."+key, length)
|
||||
}
|
||||
}
|
||||
for idx := range r.Events {
|
||||
for key, value := range r.Events[idx].Attributes {
|
||||
if str, isString := value.(string); isString {
|
||||
if cutValue, length, ok := cutString(str, limit); ok {
|
||||
r.Events[idx].Attributes[key] = cutValue
|
||||
note(fmt.Sprintf("events[%d].attributes.%s", idx, key), length)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cutString returns value shortened to limit characters, and its length when it was longer or already cut.
|
||||
func cutString(value string, limit int) (string, int, bool) {
|
||||
if len(value) <= limit {
|
||||
return value, 0, false
|
||||
}
|
||||
length := utf8.RuneCountInString(value)
|
||||
if length <= limit {
|
||||
return value, 0, false
|
||||
}
|
||||
cut := 0
|
||||
for idx := range value {
|
||||
if cut == limit {
|
||||
return value[:idx] + truncationMark, length, true
|
||||
}
|
||||
cut++
|
||||
}
|
||||
return value, 0, false
|
||||
}
|
||||
|
||||
// findWords returns up to three places in the span where one of the words occurs, ignoring case.
|
||||
func findWords(item *StorableSpan, words []string) []SpanTextMatch {
|
||||
found := make([]SpanTextMatch, 0)
|
||||
if len(words) == 0 {
|
||||
return found
|
||||
}
|
||||
// check notes the first of texts that holds a word, and reports whether enough places were found.
|
||||
check := func(field string, texts ...string) bool {
|
||||
for _, text := range texts {
|
||||
for _, word := range words {
|
||||
if snippet, ok := snippetAround(text, word); ok {
|
||||
found = append(found, SpanTextMatch{Field: field, Snippet: snippet})
|
||||
return len(found) == spanMatchLimit
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
if check("name", item.Name) || check("statusMessage", item.StatusMessage) {
|
||||
return found
|
||||
}
|
||||
attributes := item.Attributes()
|
||||
keys := make([]string, 0, len(attributes))
|
||||
for key := range attributes {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if check("attributes."+key, fmt.Sprintf("%v", attributes[key]), key) {
|
||||
return found
|
||||
}
|
||||
}
|
||||
keys = keys[:0]
|
||||
for key := range item.ResourcesString {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if check("resource."+key, item.ResourcesString[key], key) {
|
||||
return found
|
||||
}
|
||||
}
|
||||
for idx, event := range item.Events {
|
||||
if check(fmt.Sprintf("events[%d]", idx), event) {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return found
|
||||
}
|
||||
|
||||
// snippetAround returns the text around the first place word occurs in value, ignoring case.
|
||||
func snippetAround(value, word string) (string, bool) {
|
||||
// Lowering can change the byte length of some characters; positions then no longer map back.
|
||||
lowered := strings.ToLower(value)
|
||||
if len(lowered) != len(value) {
|
||||
lowered = value
|
||||
}
|
||||
pos := strings.Index(lowered, strings.ToLower(word))
|
||||
if pos < 0 {
|
||||
return "", false
|
||||
}
|
||||
start := max(pos-spanMatchSnippetRadius, 0)
|
||||
end := min(pos+len(word)+spanMatchSnippetRadius, len(value))
|
||||
for start > 0 && !utf8.RuneStart(value[start]) {
|
||||
start--
|
||||
}
|
||||
for end < len(value) && !utf8.RuneStart(value[end]) {
|
||||
end++
|
||||
}
|
||||
snippet := value[start:end]
|
||||
if start > 0 {
|
||||
snippet = truncationMark + snippet
|
||||
}
|
||||
if end < len(value) {
|
||||
snippet += truncationMark
|
||||
}
|
||||
return snippet, true
|
||||
}
|
||||
127
pkg/types/spantypes/trace_span_record_test.go
Normal file
127
pkg/types/spantypes/trace_span_record_test.go
Normal file
@@ -0,0 +1,127 @@
|
||||
package spantypes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func recordSpan(spanID string, attributes map[string]string) StorableSpan {
|
||||
return StorableSpan{
|
||||
SpanID: spanID,
|
||||
Name: "POST /charge",
|
||||
ServiceName: "payments",
|
||||
SpanKind: "Server",
|
||||
StatusCodeString: "Error",
|
||||
StatusMessage: "worldpay declined the card",
|
||||
AttributesString: attributes,
|
||||
AttributesNumber: map[string]float64{"http.response.status_code": 502},
|
||||
ResourcesString: map[string]string{"service.name": "payments"},
|
||||
Events: []string{`{"name":"exception","timeUnixNano":5,"attributeMap":{"exception.message":"card declined"},"isError":true}`},
|
||||
References: `[{"traceId":"trace","spanId":"parent","refType":"CHILD_OF"},{"traceId":"other-trace","spanId":"producer","refType":"FOLLOWS_FROM"}]`,
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewSpanRecord(t *testing.T) {
|
||||
longStatement := strings.Repeat("SELECT 1; ", 120)
|
||||
|
||||
t.Run("LongValue_IsCutAndListed", func(t *testing.T) {
|
||||
span := recordSpan("span-a", map[string]string{"db.statement": longStatement, "http.route": "/charge"})
|
||||
|
||||
record := NewSpanRecord(&span, nil)
|
||||
|
||||
assert.Equal(t, "Error", record.StatusCode)
|
||||
assert.Equal(t, "/charge", record.Attributes["http.route"])
|
||||
assert.InDelta(t, 502, record.Attributes["http.response.status_code"], 0)
|
||||
assert.Len(t, []rune(record.Attributes["db.statement"].(string)), spanRecordValueChars+1)
|
||||
assert.Equal(t, []TruncatedField{{Field: "attributes.db.statement", Length: uint64(len(longStatement))}}, record.Truncated)
|
||||
require.Len(t, record.Events, 1)
|
||||
assert.True(t, record.Events[0].IsError)
|
||||
assert.Equal(t, []Link{{TraceID: "other-trace", SpanID: "producer"}}, record.Links)
|
||||
})
|
||||
|
||||
t.Run("SelectFields_ReturnsOnlyThoseInFull", func(t *testing.T) {
|
||||
span := recordSpan("span-b", map[string]string{"db.statement": longStatement, "http.route": "/charge"})
|
||||
|
||||
record := NewSpanRecord(&span, []string{"db.statement", "not.present"})
|
||||
|
||||
assert.Equal(t, map[string]any{"db.statement": longStatement}, record.Attributes)
|
||||
assert.Empty(t, record.Truncated)
|
||||
assert.Empty(t, record.Events)
|
||||
})
|
||||
|
||||
t.Run("SelectFields_EventAttribute_KeepsThatEventWithOnlyThatAttribute", func(t *testing.T) {
|
||||
span := recordSpan("span-f", map[string]string{"http.route": "/charge"})
|
||||
|
||||
record := NewSpanRecord(&span, []string{"exception.message"})
|
||||
|
||||
assert.Empty(t, record.Attributes)
|
||||
require.Len(t, record.Events, 1)
|
||||
assert.Equal(t, "exception", record.Events[0].Name)
|
||||
assert.Equal(t, map[string]any{"exception.message": "card declined"}, record.Events[0].Attributes)
|
||||
})
|
||||
|
||||
t.Run("SelectFields_OverBudget_ShortValueStaysWholeAndLongOnesShareTheRest", func(t *testing.T) {
|
||||
long := strings.Repeat("x", 100_000)
|
||||
span := recordSpan("span-g", map[string]string{"prompt": long, "completion": long, "http.route": "/charge"})
|
||||
|
||||
record := NewSpanRecord(&span, []string{"prompt", "completion", "http.route", "not.present"})
|
||||
|
||||
assert.Equal(t, "/charge", record.Attributes["http.route"])
|
||||
share := (spanRecordSelectChars - len("/charge")) / 2
|
||||
assert.Len(t, []rune(record.Attributes["prompt"].(string)), share+1)
|
||||
assert.Len(t, []rune(record.Attributes["completion"].(string)), share+1)
|
||||
assert.Equal(t, []TruncatedField{{Field: "attributes.completion", Length: 100_000}, {Field: "attributes.prompt", Length: 100_000}}, record.Truncated)
|
||||
})
|
||||
|
||||
t.Run("SelectFields_MultiByteValue_FitsTheResponse", func(t *testing.T) {
|
||||
span := recordSpan("span-h", map[string]string{"prompt": strings.Repeat("プロンプト", 20_000)})
|
||||
|
||||
record := NewSpanRecord(&span, []string{"prompt"})
|
||||
|
||||
assert.LessOrEqual(t, jsonChars(record), spanRecordSelectMaxChars)
|
||||
assert.Equal(t, []TruncatedField{{Field: "attributes.prompt", Length: 100_000}}, record.Truncated)
|
||||
})
|
||||
|
||||
t.Run("MultiByteValue_IsCutOnACharacterBoundary", func(t *testing.T) {
|
||||
span := recordSpan("span-c", map[string]string{"note": strings.Repeat("é", 600)})
|
||||
|
||||
record := NewSpanRecord(&span, nil)
|
||||
|
||||
assert.Equal(t, strings.Repeat("é", spanRecordValueChars)+truncationMark, record.Attributes["note"])
|
||||
assert.Equal(t, uint64(600), record.Truncated[0].Length)
|
||||
})
|
||||
|
||||
t.Run("RecordOverBudget_ShrinksValuesFurther", func(t *testing.T) {
|
||||
attributes := make(map[string]string, 80)
|
||||
for i := range 80 {
|
||||
attributes["key."+strings.Repeat("k", i+1)] = strings.Repeat("v", 480)
|
||||
}
|
||||
span := recordSpan("span-d", attributes)
|
||||
|
||||
record := NewSpanRecord(&span, nil)
|
||||
|
||||
assert.LessOrEqual(t, jsonChars(record), spanRecordChars)
|
||||
assert.Len(t, record.Truncated, 80)
|
||||
assert.Equal(t, uint64(480), record.Truncated[0].Length)
|
||||
})
|
||||
}
|
||||
|
||||
func TestSmallTraceRecords(t *testing.T) {
|
||||
small := []StorableSpan{recordSpan("span-a", map[string]string{"k": "v"}), recordSpan("span-b", map[string]string{"k": "v"})}
|
||||
records, ok := SmallTraceRecords(small)
|
||||
require.True(t, ok)
|
||||
assert.Len(t, records, 2)
|
||||
|
||||
large := make([]StorableSpan, 0, 60)
|
||||
for range 60 {
|
||||
large = append(large, recordSpan("span", map[string]string{"payload": strings.Repeat("x", 450)}))
|
||||
}
|
||||
_, ok = SmallTraceRecords(large)
|
||||
assert.False(t, ok)
|
||||
|
||||
assert.True(t, IsSmallTrace(inspectTreeSpans))
|
||||
assert.False(t, IsSmallTrace(inspectTreeSpans+1))
|
||||
}
|
||||
7
tests/fixtures/traces.py
vendored
7
tests/fixtures/traces.py
vendored
@@ -184,16 +184,19 @@ class TracesEvent(ABC):
|
||||
name: str
|
||||
time_unix_nano: np.uint64
|
||||
attribute_map: dict[str, str]
|
||||
is_error: bool
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
timestamp: datetime.datetime,
|
||||
attribute_map: dict[str, str] = {},
|
||||
is_error: bool = False,
|
||||
) -> None:
|
||||
self.name = name
|
||||
self.time_unix_nano = np.uint64(int(timestamp.timestamp() * 1e9))
|
||||
self.attribute_map = attribute_map
|
||||
self.is_error = is_error
|
||||
|
||||
def np_arr(self) -> np.array:
|
||||
return np.array([self.name, self.time_unix_nano, json.dumps(self.attribute_map)])
|
||||
@@ -598,13 +601,15 @@ class Traces(ABC):
|
||||
)
|
||||
# DB shape per event: {"name", "timeUnixNano", "attributeMap"}. Must match
|
||||
# what the consume-layer parser in pkg/types/spantypes expects.
|
||||
# The collector writes isError only when it is set.
|
||||
self._events_db = [
|
||||
json.dumps(
|
||||
{
|
||||
"name": event.name,
|
||||
"timeUnixNano": int(event.time_unix_nano),
|
||||
"attributeMap": dict(event.attribute_map),
|
||||
},
|
||||
}
|
||||
| ({"isError": True} if event.is_error else {}),
|
||||
separators=(",", ":"),
|
||||
)
|
||||
for event in events
|
||||
|
||||
534
tests/integration/tests/tracedetail/05_inspect.py
Normal file
534
tests/integration/tests/tracedetail/05_inspect.py
Normal file
@@ -0,0 +1,534 @@
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.querier import get_attribute_backend_now
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesEvent, TracesKind, TracesLink, TracesRefType, TracesStatusCode
|
||||
|
||||
OVERVIEW_LISTS = ("errors", "unmarkedFailures", "slowest", "expensive", "links")
|
||||
LIST_SIZE = 10
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attribute_backend", ["map", "json"])
|
||||
def test_inspect_overview(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
attribute_backend: str,
|
||||
) -> None:
|
||||
"""Attributes are written to one layout only, so a read from the wrong column loses them."""
|
||||
now = get_attribute_backend_now(signoz, attribute_backend).replace(second=0, microsecond=0)
|
||||
write_mode = "json_only" if attribute_backend == "json" else "legacy_only"
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
other_trace_id = TraceIdGenerator.trace_id()
|
||||
names = ("root", "place_order", "charge_card", "inventory_http", "inventory_grpc", "render", "consume", "model", "tool")
|
||||
ids = {name: TraceIdGenerator.span_id() for name in names}
|
||||
gateway, orders, agent = (f"td-inspect-{service}-{attribute_backend}" for service in ("gateway", "orders", "agent"))
|
||||
|
||||
def span(key: str, parent: str, name: str, service: str, offset_ms: int, duration_ms: int, **kwargs) -> Traces:
|
||||
return Traces(
|
||||
timestamp=now - timedelta(seconds=10) + timedelta(milliseconds=offset_ms),
|
||||
duration=timedelta(milliseconds=duration_ms),
|
||||
trace_id=trace_id,
|
||||
span_id=ids[key],
|
||||
parent_span_id=ids[parent] if parent else "",
|
||||
name=name,
|
||||
resources={"service.name": service},
|
||||
attribute_write_mode=write_mode,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
span("root", "", "POST /checkout", gateway, 0, 1000, kind=TracesKind.SPAN_KIND_SERVER, status_code=TracesStatusCode.STATUS_CODE_ERROR, attributes={"http.response.status_code": 500}),
|
||||
# the error is carried up from charge_card through place_order to the root
|
||||
span("place_order", "root", "place_order", orders, 10, 600, status_code=TracesStatusCode.STATUS_CODE_ERROR),
|
||||
span("charge_card", "place_order", "charge_card", orders, 20, 300, kind=TracesKind.SPAN_KIND_CLIENT, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message="card declined", attributes={"payment.provider": "worldpay"}),
|
||||
# three failures that do not carry the error flag
|
||||
span("inventory_http", "place_order", "GET /inventory", orders, 330, 40, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"http.response.status_code": 404}),
|
||||
span("inventory_grpc", "place_order", "inventory.Check", orders, 380, 40, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"rpc.grpc.status_code": 5}),
|
||||
span("render", "root", "render_receipt", gateway, 700, 50, events=[TracesEvent("exception", now, {"exception.message": "template missing, fell back"}, is_error=True)]),
|
||||
# a link out to another trace
|
||||
span("consume", "root", "consume order.created", orders, 760, 20, kind=TracesKind.SPAN_KIND_CONSUMER, links=[TracesLink(trace_id=other_trace_id, span_id="aaaaaaaaaaaaaaaa", ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM)]),
|
||||
span("model", "root", "chat gpt-4o-mini", agent, 800, 100, kind=TracesKind.SPAN_KIND_CLIENT, attributes={"gen_ai.request.model": "gpt-4o-mini", "gen_ai.usage.input_tokens": 100, "gen_ai.usage.output_tokens": 20, "signoz.gen_ai.usage.tokens.cost": 0.01}),
|
||||
span("tool", "root", "execute_tool", agent, 910, 30, attributes={"gen_ai.tool.name": "get_weather"}),
|
||||
]
|
||||
)
|
||||
|
||||
url = signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect")
|
||||
headers = {"authorization": f"Bearer {get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)}"}
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
overview = response.json()["data"]
|
||||
|
||||
identity = overview["identity"]
|
||||
assert identity["traceId"] == trace_id
|
||||
assert identity["spanCount"] == 9
|
||||
assert identity["errorSpanCount"] == 3
|
||||
assert identity["rootCount"] == 1
|
||||
assert identity["hasMissingSpans"] is False
|
||||
assert identity["durationNano"] == 1_000_000_000
|
||||
assert identity["root"]["spanId"] == ids["root"]
|
||||
assert identity["root"]["name"] == "POST /checkout"
|
||||
assert identity["root"]["responseStatusCode"] == "500"
|
||||
assert identity["serviceCount"] == 3
|
||||
assert {service["serviceName"]: (service["spanCount"], service["errorSpanCount"]) for service in identity["services"]} == {gateway: (2, 1), orders: (5, 2), agent: (2, 0)}
|
||||
assert identity["ai"]["modelCallCount"] == 1
|
||||
assert identity["ai"]["toolCallCount"] == 1
|
||||
assert identity["ai"]["tokens"] == {"input": 100, "output": 20, "cacheRead": 0, "cacheWrite": 0, "reasoning": 0}
|
||||
assert identity["ai"]["totalCost"] == pytest.approx(0.01)
|
||||
|
||||
# one entry for the error, at the span that threw it
|
||||
errors = overview["errors"]
|
||||
assert errors["total"] == 1
|
||||
assert "nextCursor" not in errors
|
||||
assert errors["items"][0]["span"]["spanId"] == ids["charge_card"]
|
||||
assert errors["items"][0]["parent"]["spanId"] == ids["place_order"]
|
||||
assert errors["items"][0]["statusMessage"] == "card declined"
|
||||
assert errors["items"][0]["errorAncestorCount"] == 2
|
||||
|
||||
unmarked = overview["unmarkedFailures"]
|
||||
assert unmarked["total"] == 3
|
||||
assert {item["span"]["spanId"]: item["reason"] for item in unmarked["items"]} == {ids["inventory_http"]: "http_4xx", ids["inventory_grpc"]: "grpc_status", ids["render"]: "exception_event"}
|
||||
|
||||
links = overview["links"]
|
||||
assert links["total"] == 1
|
||||
assert links["items"][0]["span"]["spanId"] == ids["consume"]
|
||||
assert links["items"][0]["parent"]["spanId"] == ids["root"]
|
||||
assert links["items"][0]["linkCount"] == 1
|
||||
assert links["items"][0]["links"] == [{"traceId": other_trace_id, "spanId": "aaaaaaaaaaaaaaaa"}]
|
||||
|
||||
expensive = overview["expensive"]
|
||||
assert expensive["total"] == 2
|
||||
assert expensive["items"][0]["span"]["spanId"] == ids["model"]
|
||||
assert expensive["items"][0]["cost"] == pytest.approx(0.01)
|
||||
assert (expensive["items"][0]["inputTokens"], expensive["items"][0]["outputTokens"]) == (100, 20)
|
||||
|
||||
# own time is the duration minus what the children cover: 300 ms of charge_card is all its own
|
||||
slowest = overview["slowest"]
|
||||
assert slowest["total"] == 9
|
||||
assert slowest["items"][0]["span"]["spanId"] == ids["charge_card"]
|
||||
assert slowest["items"][0]["ownTimeNano"] == 300_000_000
|
||||
assert slowest["items"][0]["spanCount"] == 1
|
||||
|
||||
# every list item carries its direct parent, and the root has none
|
||||
for name in OVERVIEW_LISTS:
|
||||
for item in overview[name]["items"]:
|
||||
assert item.get("parent", {}).get("spanId", "") == item["span"].get("parentSpanId", ""), f"{name}: {item}"
|
||||
|
||||
tree = overview["tree"]
|
||||
assert tree["total"] == 9
|
||||
assert tree["complete"] is True
|
||||
# tree order: a span is followed by the spans below it, siblings by start time
|
||||
assert [line["spanId"] for line in tree["spans"]] == [ids[name] for name in ("root", "place_order", "charge_card", "inventory_http", "inventory_grpc", "render", "consume", "model", "tool")]
|
||||
place_order = tree["spans"][1]
|
||||
assert (place_order["level"], place_order["childCount"], place_order["subtreeSpanCount"]) == (1, 3, 3)
|
||||
assert place_order["ownTimeNano"] == 220_000_000
|
||||
|
||||
# a small trace carries every span's record
|
||||
records = {record["spanId"]: record for record in tree["records"]}
|
||||
assert set(records) == set(ids.values())
|
||||
assert records[ids["charge_card"]]["attributes"]["payment.provider"] == "worldpay"
|
||||
assert records[ids["charge_card"]]["statusMessage"] == "card declined"
|
||||
assert records[ids["charge_card"]]["resource"]["service.name"] == orders
|
||||
assert records[ids["consume"]]["links"] == [{"traceId": other_trace_id, "spanId": "aaaaaaaaaaaaaaaa"}]
|
||||
assert records[ids["render"]]["events"][0]["name"] == "exception"
|
||||
|
||||
|
||||
def test_inspect_pages_every_list(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
root_id = TraceIdGenerator.span_id()
|
||||
resources = {"service.name": "td-inspect-pages"}
|
||||
spans = [Traces(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=20), trace_id=trace_id, span_id=root_id, name="POST /batch", kind=TracesKind.SPAN_KIND_SERVER, resources=resources)]
|
||||
expected: dict[str, list[str]] = {name: [] for name in OVERVIEW_LISTS}
|
||||
|
||||
def child(group: str, i: int, offset_ms: int, **kwargs) -> None:
|
||||
span_id = TraceIdGenerator.span_id()
|
||||
expected[group].append(span_id)
|
||||
# distinct names keep each span its own entry in the slowest list; distinct durations fix its order
|
||||
spans.append(Traces(timestamp=now - timedelta(seconds=29) + timedelta(milliseconds=offset_ms), duration=timedelta(milliseconds=100 + offset_ms), trace_id=trace_id, span_id=span_id, parent_span_id=root_id, name=f"{group}-{i:02d}", resources=resources, **kwargs))
|
||||
|
||||
for i in range(12):
|
||||
child("errors", i, i, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message=f"failure {i:02d}")
|
||||
child("unmarkedFailures", i, 20 + i, attributes={"http.response.status_code": 404})
|
||||
child("expensive", i, 40 + i, attributes={"gen_ai.request.model": "gpt-4o-mini", "signoz.gen_ai.usage.tokens.cost": (i + 1) / 100})
|
||||
child("links", i, 60 + i, links=[TracesLink(trace_id=TraceIdGenerator.trace_id(), span_id=TraceIdGenerator.span_id(), ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM)])
|
||||
insert_traces(spans)
|
||||
# the slowest list ranks every span: the 48 children by duration, the longest first, then the root
|
||||
by_duration = sorted(spans[1:], key=lambda span: int(span.duration_nano), reverse=True)
|
||||
expected["slowest"] = [span.span_id for span in by_duration]
|
||||
expected["expensive"].reverse()
|
||||
|
||||
url = signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect")
|
||||
headers = {"authorization": f"Bearer {get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)}"}
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
overview = response.json()["data"]
|
||||
assert overview["identity"]["spanCount"] == 49
|
||||
|
||||
for name in OVERVIEW_LISTS:
|
||||
first = overview[name]
|
||||
assert len(first["items"]) == LIST_SIZE, name
|
||||
assert first["nextCursor"], name
|
||||
seen = [item["span"]["spanId"] for item in first["items"]]
|
||||
cursor = first["nextCursor"]
|
||||
for _ in range(10):
|
||||
response = requests.get(url, params={"cursor": cursor}, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
page = response.json()["data"]
|
||||
assert list(page) == [name], f"{name}: a page returned {list(page)}"
|
||||
assert page[name]["total"] == first["total"]
|
||||
seen += [item["span"]["spanId"] for item in page[name]["items"]]
|
||||
cursor = page[name].get("nextCursor")
|
||||
if not cursor:
|
||||
break
|
||||
assert len(seen) == first["total"], name
|
||||
assert len(set(seen)) == len(seen), f"{name}: a span was returned twice"
|
||||
if name == "slowest":
|
||||
# the root's own time is what its children leave of it, so only its place is not asserted
|
||||
assert [span_id for span_id in seen if span_id != root_id] == expected[name]
|
||||
assert first["total"] == 49
|
||||
else:
|
||||
assert seen == expected[name], name
|
||||
assert first["total"] == 12
|
||||
|
||||
# the second page of errors carries the status messages of its own spans
|
||||
response = requests.get(url, params={"cursor": overview["errors"]["nextCursor"]}, headers=headers, timeout=10)
|
||||
assert [item["statusMessage"] for item in response.json()["data"]["errors"]["items"]] == ["failure 10", "failure 11"]
|
||||
|
||||
|
||||
def test_inspect_large_trace_shows_the_top_of_the_tree(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
root_id = TraceIdGenerator.span_id()
|
||||
resources = {"service.name": "td-inspect-large"}
|
||||
spans = [Traces(timestamp=now - timedelta(seconds=30), duration=timedelta(seconds=20), trace_id=trace_id, span_id=root_id, name="POST /import", kind=TracesKind.SPAN_KIND_SERVER, resources=resources)]
|
||||
for i in range(150):
|
||||
spans.append(Traces(timestamp=now - timedelta(seconds=29) + timedelta(milliseconds=i * 10), duration=timedelta(milliseconds=5), trace_id=trace_id, span_id=TraceIdGenerator.span_id(), parent_span_id=root_id, name="write_item", resources=resources))
|
||||
insert_traces(spans)
|
||||
in_time_order = [span.span_id for span in spans]
|
||||
|
||||
url = signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect")
|
||||
headers = {"authorization": f"Bearer {get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)}"}
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
overview = response.json()["data"]
|
||||
|
||||
assert overview["identity"]["spanCount"] == 151
|
||||
tree = overview["tree"]
|
||||
assert tree["total"] == 151
|
||||
assert tree["complete"] is False
|
||||
assert "records" not in tree
|
||||
assert [line["spanId"] for line in tree["spans"]] == in_time_order[:50]
|
||||
assert tree["spans"][0]["childCount"] == 150
|
||||
|
||||
slowest = overview["slowest"]
|
||||
assert slowest["total"] == 2
|
||||
group = next(item for item in slowest["items"] if item["span"]["name"] == "write_item")
|
||||
assert group["spanCount"] == 150
|
||||
assert group["ownTimeNano"] == 150 * 5_000_000
|
||||
assert group["parent"]["spanId"] == root_id
|
||||
assert "expensive" not in overview
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attribute_backend", ["map", "json"])
|
||||
def test_inspect_span(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
attribute_backend: str,
|
||||
) -> None:
|
||||
"""Attributes are written to one layout only, so a read from the wrong column loses them."""
|
||||
now = get_attribute_backend_now(signoz, attribute_backend).replace(second=0, microsecond=0)
|
||||
write_mode = "json_only" if attribute_backend == "json" else "legacy_only"
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
other_trace_id = TraceIdGenerator.trace_id()
|
||||
resources = {"service.name": f"td-inspect-span-{attribute_backend}"}
|
||||
root_id, handler_id, middle_id, target_id = (TraceIdGenerator.span_id() for _ in range(4))
|
||||
long_statement = "SELECT * FROM orders WHERE id IN (" + ", ".join(str(i) for i in range(400)) + ")"
|
||||
start = now - timedelta(seconds=10)
|
||||
|
||||
def span(span_id: str, parent_id: str, name: str, offset_ms: int, duration_ms: int, **kwargs) -> Traces:
|
||||
return Traces(timestamp=start + timedelta(milliseconds=offset_ms), duration=timedelta(milliseconds=duration_ms), trace_id=trace_id, span_id=span_id, parent_span_id=parent_id, name=name, resources=resources, attribute_write_mode=write_mode, **kwargs)
|
||||
|
||||
spans = [
|
||||
span(root_id, "", "POST /orders", 0, 2000, kind=TracesKind.SPAN_KIND_SERVER),
|
||||
span(handler_id, root_id, "handle_order", 10, 1900),
|
||||
span(middle_id, handler_id, "load_order", 20, 1800),
|
||||
span(
|
||||
target_id,
|
||||
middle_id,
|
||||
"query_orders",
|
||||
30,
|
||||
1000,
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_ERROR,
|
||||
status_message="deadlock detected",
|
||||
attributes={"db.system": "postgresql", "db.statement": long_statement, "db.rows": 42},
|
||||
events=[TracesEvent("retry", start, {"attempt": "2"})],
|
||||
links=[TracesLink(trace_id=other_trace_id, span_id="bbbbbbbbbbbbbbbb", ref_type=TracesRefType.REF_TYPE_FOLLOWS_FROM)],
|
||||
),
|
||||
]
|
||||
child_ids, grandchild_ids = [], []
|
||||
for i in range(25):
|
||||
child_id, grandchild_id = TraceIdGenerator.span_id(), TraceIdGenerator.span_id()
|
||||
child_ids.append(child_id)
|
||||
grandchild_ids.append(grandchild_id)
|
||||
spans.append(span(child_id, target_id, "fetch_page", 40 + i * 10, 8))
|
||||
spans.append(span(grandchild_id, child_id, "decode_row", 41 + i * 10, 2))
|
||||
# the same operation in nine other traces of the hour before, 100 ms to 900 ms long
|
||||
for i in range(1, 10):
|
||||
spans.append(Traces(timestamp=start - timedelta(minutes=5 * i), duration=timedelta(milliseconds=100 * i), trace_id=TraceIdGenerator.trace_id(), span_id=TraceIdGenerator.span_id(), name="query_orders", kind=TracesKind.SPAN_KIND_CLIENT, resources=resources, attribute_write_mode=write_mode))
|
||||
insert_traces(spans)
|
||||
|
||||
spans_url = signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect/spans")
|
||||
headers = {"authorization": f"Bearer {get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)}"}
|
||||
response = requests.get(f"{spans_url}/{target_id}", headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
detail = response.json()["data"]
|
||||
|
||||
summary = detail["summary"]
|
||||
assert summary["spanId"] == target_id
|
||||
assert summary["parentSpanId"] == middle_id
|
||||
assert summary["name"] == "query_orders"
|
||||
assert (summary["level"], summary["childCount"], summary["subtreeSpanCount"]) == (3, 25, 50)
|
||||
assert summary["hasError"] is True
|
||||
assert summary["durationNano"] == 1_000_000_000
|
||||
|
||||
record = detail["record"]
|
||||
assert record["spanId"] == target_id
|
||||
assert record["statusMessage"] == "deadlock detected"
|
||||
assert record["attributes"]["db.system"] == "postgresql"
|
||||
assert record["attributes"]["db.rows"] == 42
|
||||
assert record["resource"]["service.name"] == resources["service.name"]
|
||||
assert record["events"][0]["name"] == "retry"
|
||||
assert record["links"] == [{"traceId": other_trace_id, "spanId": "bbbbbbbbbbbbbbbb"}]
|
||||
# a value over 500 characters is cut and listed with its full length
|
||||
assert len(record["attributes"]["db.statement"]) == 501
|
||||
assert record["truncated"] == [{"field": "attributes.db.statement", "length": len(long_statement)}]
|
||||
|
||||
# two levels up, the parent first, then the root
|
||||
assert [line["spanId"] for line in detail["ancestors"]] == [middle_id, handler_id, root_id]
|
||||
assert detail["ancestorCount"] == 3
|
||||
# two levels down, each with its total and its first 20 spans in time order
|
||||
levels = detail["descendants"]
|
||||
assert [(level["level"], level["total"], len(level["spans"])) for level in levels] == [(1, 25, 20), (2, 25, 20)]
|
||||
assert [line["spanId"] for line in levels[0]["spans"]] == child_ids[:20]
|
||||
assert [line["spanId"] for line in levels[1]["spans"]] == grandchild_ids[:20]
|
||||
|
||||
# ten spans of this name and service started in the hour up to this one, and this is the longest
|
||||
percentile = detail["percentile"]
|
||||
assert percentile["windowMinutes"] == 60
|
||||
assert percentile["position"] == pytest.approx(100)
|
||||
assert 400_000_000 <= percentile["p50Nano"] <= 700_000_000
|
||||
assert percentile["p50Nano"] <= percentile["p90Nano"] <= percentile["p99Nano"] <= 1_000_000_000
|
||||
|
||||
one_level = requests.get(f"{spans_url}/{target_id}", params={"depth": 1}, headers=headers, timeout=10).json()["data"]
|
||||
assert [line["spanId"] for line in one_level["ancestors"]] == [middle_id, root_id]
|
||||
assert [level["level"] for level in one_level["descendants"]] == [1]
|
||||
|
||||
selected = requests.get(f"{spans_url}/{target_id}", params={"selectFields": "db.statement"}, headers=headers, timeout=10).json()["data"]
|
||||
# a call for named fields reads more of an opened span, so the percentile is not computed again
|
||||
assert "percentile" not in selected
|
||||
selected = selected["record"]
|
||||
assert selected["attributes"] == {"db.statement": long_statement}
|
||||
assert selected["truncated"] == []
|
||||
|
||||
# a selected name also picks the attribute of an event, and keeps only the events that have it
|
||||
event_selected = requests.get(f"{spans_url}/{target_id}", params={"selectFields": ["attempt", "db.system"]}, headers=headers, timeout=10).json()["data"]["record"]
|
||||
assert event_selected["attributes"] == {"db.system": "postgresql"}
|
||||
assert [(event["name"], event["attributes"]) for event in event_selected["events"]] == [("retry", {"attempt": "2"})]
|
||||
|
||||
# a blank name selects nothing, so the usual record comes back
|
||||
blank = requests.get(f"{spans_url}/{target_id}", params={"selectFields": ""}, headers=headers, timeout=10).json()["data"]["record"]
|
||||
assert blank["attributes"]["db.system"] == "postgresql"
|
||||
assert len(blank["attributes"]["db.statement"]) == 501
|
||||
|
||||
root = requests.get(f"{spans_url}/{root_id}", headers=headers, timeout=10).json()["data"]
|
||||
assert root["ancestors"] == []
|
||||
assert [level["total"] for level in root["descendants"]] == [1, 1]
|
||||
|
||||
unknown = requests.get(f"{spans_url}/{TraceIdGenerator.span_id()}", headers=headers, timeout=10)
|
||||
assert unknown.status_code == HTTPStatus.NOT_FOUND, unknown.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attribute_backend", ["map", "json"])
|
||||
def test_inspect_find(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
attribute_backend: str,
|
||||
) -> None:
|
||||
"""Attributes are written to one layout only, so a read from the wrong column finds nothing."""
|
||||
now = get_attribute_backend_now(signoz, attribute_backend).replace(second=0, microsecond=0)
|
||||
write_mode = "json_only" if attribute_backend == "json" else "legacy_only"
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
resources = {"service.name": f"td-inspect-find-{attribute_backend}"}
|
||||
names = ("root", "by_name", "by_status", "by_attribute", "by_event", "unrelated")
|
||||
ids = {name: TraceIdGenerator.span_id() for name in names}
|
||||
start = now - timedelta(seconds=10)
|
||||
|
||||
def span(key: str, parent: str, name: str, offset_ms: int, **kwargs) -> Traces:
|
||||
return Traces(timestamp=start + timedelta(milliseconds=offset_ms), duration=timedelta(milliseconds=5), trace_id=trace_id, span_id=ids[key], parent_span_id=ids[parent] if parent else "", name=name, resources=resources, attribute_write_mode=write_mode, **kwargs)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
span("root", "", "POST /checkout", 0, kind=TracesKind.SPAN_KIND_SERVER),
|
||||
span("by_name", "root", "call worldpay", 10),
|
||||
span("by_status", "root", "charge_card", 20, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message="Worldpay declined the card"),
|
||||
span("by_attribute", "by_status", "POST /v1/payments", 30, attributes={"payment.provider": "worldpay", "http.response.status_code": 402}),
|
||||
span("by_event", "root", "notify", 40, events=[TracesEvent("retry", start, {"reason": "worldpay timeout"})]),
|
||||
span("unrelated", "root", "render_receipt", 50, status_code=TracesStatusCode.STATUS_CODE_ERROR, status_message="template missing"),
|
||||
]
|
||||
)
|
||||
spans_url = signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect/spans")
|
||||
headers = {"authorization": f"Bearer {get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)}"}
|
||||
|
||||
# a plain word, in start time order, wherever it occurs
|
||||
response = requests.get(spans_url, params={"search": "worldpay"}, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
found = response.json()["data"]
|
||||
assert found["total"] == 4
|
||||
assert "nextCursor" not in found
|
||||
assert [item["span"]["spanId"] for item in found["items"]] == [ids[name] for name in ("by_name", "by_status", "by_attribute", "by_event")]
|
||||
where = {item["span"]["spanId"]: [match["field"] for match in item["matches"]] for item in found["items"]}
|
||||
assert where[ids["by_name"]] == ["name"]
|
||||
assert where[ids["by_status"]] == ["statusMessage"]
|
||||
assert where[ids["by_attribute"]] == ["attributes.payment.provider"]
|
||||
assert where[ids["by_event"]] == ["events[0]"]
|
||||
assert found["items"][1]["matches"][0]["snippet"] == "Worldpay declined the card"
|
||||
assert [item["parent"]["spanId"] for item in found["items"]] == [ids["root"], ids["root"], ids["by_status"], ids["root"]]
|
||||
|
||||
# a filter alone has no words to show
|
||||
errors = requests.get(spans_url, params={"search": "has_error = true"}, headers=headers, timeout=10).json()["data"]
|
||||
assert [item["span"]["spanId"] for item in errors["items"]] == [ids["by_status"], ids["unrelated"]]
|
||||
assert all(item["matches"] == [] for item in errors["items"])
|
||||
|
||||
both = requests.get(spans_url, params={"search": "worldpay AND has_error = true"}, headers=headers, timeout=10).json()["data"]
|
||||
assert [item["span"]["spanId"] for item in both["items"]] == [ids["by_status"]]
|
||||
|
||||
by_attribute = requests.get(spans_url, params={"search": "payment.provider = 'worldpay'"}, headers=headers, timeout=10).json()["data"]
|
||||
assert [item["span"]["spanId"] for item in by_attribute["items"]] == [ids["by_attribute"]]
|
||||
|
||||
nothing = requests.get(spans_url, params={"search": "stripe"}, headers=headers, timeout=10).json()["data"]
|
||||
assert (nothing["total"], nothing["items"]) == (0, [])
|
||||
|
||||
# a word is also found in an attribute's name and in a resource attribute
|
||||
by_key = requests.get(spans_url, params={"search": "payment.provider"}, headers=headers, timeout=10).json()["data"]
|
||||
assert [(item["span"]["spanId"], item["matches"][0]["field"]) for item in by_key["items"]] == [(ids["by_attribute"], "attributes.payment.provider")]
|
||||
by_resource = requests.get(spans_url, params={"search": "td-inspect-find"}, headers=headers, timeout=10).json()["data"]
|
||||
assert by_resource["total"] == 6
|
||||
assert {item["matches"][0]["field"] for item in by_resource["items"]} == {"resource.service.name"}
|
||||
|
||||
unknown_key = requests.get(spans_url, params={"search": "no.such.key = 'x'"}, headers=headers, timeout=10).json()["data"]
|
||||
assert unknown_key["total"] == 0
|
||||
assert "no.such.key" in unknown_key["warnings"][0]
|
||||
|
||||
# limit sets the page size; the same search with the cursor returns the next page
|
||||
first = requests.get(spans_url, params={"search": "worldpay", "limit": 3}, headers=headers, timeout=10).json()["data"]
|
||||
assert first["total"] == 4
|
||||
assert [item["span"]["spanId"] for item in first["items"]] == [ids[name] for name in ("by_name", "by_status", "by_attribute")]
|
||||
second = requests.get(spans_url, params={"search": "worldpay", "limit": 3, "cursor": first["nextCursor"]}, headers=headers, timeout=10).json()["data"]
|
||||
assert second["total"] == 4
|
||||
assert [item["span"]["spanId"] for item in second["items"]] == [ids["by_event"]]
|
||||
assert "nextCursor" not in second
|
||||
|
||||
broken = requests.get(spans_url, params={"search": "has_error ="}, headers=headers, timeout=10)
|
||||
assert broken.status_code == HTTPStatus.BAD_REQUEST, broken.text
|
||||
|
||||
|
||||
def test_inspect_all_spans(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[list[Traces]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
resources = {"service.name": "td-inspect-all-spans"}
|
||||
start = now - timedelta(seconds=10)
|
||||
root_id = TraceIdGenerator.span_id()
|
||||
spans = [Traces(timestamp=start, duration=timedelta(seconds=1), trace_id=trace_id, span_id=root_id, name="POST /batch", kind=TracesKind.SPAN_KIND_SERVER, resources=resources)]
|
||||
tree_order = [root_id]
|
||||
branches: list[list[str]] = []
|
||||
for branch in range(3):
|
||||
branch_id = TraceIdGenerator.span_id()
|
||||
spans.append(Traces(timestamp=start + timedelta(milliseconds=100 * branch + 10), duration=timedelta(milliseconds=80), trace_id=trace_id, span_id=branch_id, parent_span_id=root_id, name=f"batch-{branch}", resources=resources))
|
||||
below = [branch_id]
|
||||
for leaf in range(2):
|
||||
leaf_id = TraceIdGenerator.span_id()
|
||||
spans.append(Traces(timestamp=start + timedelta(milliseconds=100 * branch + 20 + 10 * leaf), duration=timedelta(milliseconds=5), trace_id=trace_id, span_id=leaf_id, parent_span_id=branch_id, name="write_item", resources=resources))
|
||||
below.append(leaf_id)
|
||||
branches.append(below)
|
||||
tree_order += below
|
||||
insert_traces(spans)
|
||||
spans_url = signoz.self.host_configs["8080"].get(f"/api/v1/traces/{trace_id}/inspect/spans")
|
||||
headers = {"authorization": f"Bearer {get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)}"}
|
||||
|
||||
response = requests.get(spans_url, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
everything = response.json()["data"]
|
||||
assert everything["total"] == 10
|
||||
assert "nextCursor" not in everything
|
||||
assert [item["span"]["spanId"] for item in everything["items"]] == tree_order
|
||||
assert [item["span"]["level"] for item in everything["items"]] == [0, 1, 2, 2, 1, 2, 2, 1, 2, 2]
|
||||
# a line of this list has no parent of its own: its parent is earlier in the same list
|
||||
assert all("parent" not in item and item["matches"] == [] for item in everything["items"])
|
||||
|
||||
seen, cursor = [], None
|
||||
for _ in range(4):
|
||||
params = {"limit": 4} | ({"cursor": cursor} if cursor else {})
|
||||
page = requests.get(spans_url, params=params, headers=headers, timeout=10).json()["data"]
|
||||
assert page["total"] == 10
|
||||
seen += [item["span"]["spanId"] for item in page["items"]]
|
||||
cursor = page.get("nextCursor")
|
||||
if not cursor:
|
||||
break
|
||||
assert seen == tree_order
|
||||
|
||||
below = requests.get(spans_url, params={"under": branches[1][0]}, headers=headers, timeout=10).json()["data"]
|
||||
assert below["total"] == 3
|
||||
assert [item["span"]["spanId"] for item in below["items"]] == branches[1]
|
||||
|
||||
unknown = requests.get(spans_url, params={"under": TraceIdGenerator.span_id()}, headers=headers, timeout=10)
|
||||
assert unknown.status_code == HTTPStatus.NOT_FOUND, unknown.text
|
||||
|
||||
|
||||
def test_inspect_rejects_invalid_requests(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
url = signoz.self.host_configs["8080"].get(f"/api/v1/traces/{TraceIdGenerator.trace_id()}/inspect")
|
||||
headers = {"authorization": f"Bearer {get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)}"}
|
||||
|
||||
response = requests.get(url, params={"cursor": "not-a-cursor"}, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
|
||||
for depth in (11, "all"):
|
||||
response = requests.get(f"{url}/spans/0000000000000001", params={"depth": depth}, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, f"depth {depth}: {response.text}"
|
||||
for params in ({"search": "worldpay", "under": "0000000000000001"}, {"limit": 1001}, {"cursor": "not-a-cursor"}):
|
||||
response = requests.get(f"{url}/spans", params=params, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST, f"{params}: {response.text}"
|
||||
|
||||
response = requests.get(url, headers=headers, timeout=10)
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND, response.text
|
||||
Reference in New Issue
Block a user