Compare commits

..

72 Commits

Author SHA1 Message Date
Piyush Singariya
19f4cde52e fix: package test fixed 2026-03-17 13:46:58 +05:30
srikanthccv
fcce8b4008 chore: text search tests 2026-03-17 09:08:02 +05:30
srikanthccv
c2c2678125 chore: keep message contained to field mapper 2026-03-17 08:55:37 +05:30
srikanthccv
73dc095b79 Merge branch 'main' into keep-message-contained 2026-03-17 05:11:24 +05:30
Piyush Singariya
b2adbb510b fix: remove the exception checking 2026-03-16 19:46:13 +05:30
Piyush Singariya
4f7ec4c057 revert: remove unused JSON Field datatype 2026-03-16 16:24:38 +05:30
Piyush Singariya
9c5a5488e2 fix: fallback expr switch case 2026-03-16 16:22:14 +05:30
Piyush Singariya
9ee23eae06 Merge branch 'main' into merge-json-col-fields 2026-03-13 23:43:27 +05:30
Piyush Singariya
0f36479787 Merge branch 'main' into merge-json-col-fields 2026-03-13 19:55:05 +05:30
Piyush Singariya
95a9a24875 fix: message field key search in JSON Logs (#10577)
* feat: work in progress

* fix: test run success

* fix: in progress

* fix: excluding message from metadata fetch

* test: cleared

* fix: key name in metadata

* fix: uncomment tests

* chore: change to method for staticfields

* fix: remove confusing comments; remove usage of logical keyword

* chore: shift method above business logic

* chore: changes based on review

* fix: comments in metadata_store.go
2026-03-13 19:16:05 +05:30
Piyush Singariya
7d1e39037c chore: minor changes based on review 2026-03-13 13:06:56 +05:30
Piyush Singariya
54f104db5f fix: cursor comments 2026-03-13 12:15:12 +05:30
Piyush Singariya
ab0852bbfb feat: update message as typehint in JSON Column (#10545) 2026-03-11 14:50:59 +05:30
Piyush Singariya
4c7aba680e fix: lint error 2026-03-10 14:00:12 +05:30
Piyush Singariya
23c247a1ba Merge branch 'main' into merge-json-col-fields 2026-03-10 13:28:30 +05:30
Piyush Singariya
4777b13ddf fix: shift warning attachment to getKeySelectors 2026-03-03 13:27:23 +05:30
Piyush Singariya
2d3060bac4 chore: remove unnecessary change 2026-02-25 12:09:30 +05:30
Piyush Singariya
9101d51920 Merge branch 'main' into merge-json-col-fields 2026-02-23 12:50:11 +05:30
Piyush Singariya
82b82b0208 chore: addressing comments from Nitya 2026-02-23 12:49:48 +05:30
Nityananda Gohain
51bd760d9a Merge branch 'main' into merge-json-col-fields 2026-02-20 15:53:31 +05:30
Piyush Singariya
2a492cc783 fix: change warning to a const to fix tests 2026-02-20 14:54:16 +05:30
Piyush Singariya
24afdad36c fix: append warnings from fieldkeys 2026-02-20 14:51:13 +05:30
Piyush Singariya
5d20019207 fix: body.message not being mapped correctly 2026-02-20 14:26:12 +05:30
Piyush Singariya
1963d5811d Merge branch 'main' into merge-json-col-fields 2026-02-20 10:08:25 +05:30
Piyush Singariya
15cfccad74 revert: change ReadMultiple is needed 2026-02-20 10:08:11 +05:30
Piyush Singariya
a0399560e3 revert: remvoing unused function 2026-02-19 16:33:12 +05:30
Piyush Singariya
265e337d5c Merge branch 'main' into merge-json-col-fields 2026-02-19 16:29:55 +05:30
Piyush Singariya
bb8c874755 fix: go lint 2026-02-17 17:19:24 +05:30
Piyush Singariya
13cbe03d64 fix: tests 2026-02-17 16:58:00 +05:30
Piyush Singariya
93621c29b7 fix: go mod changes 2026-02-17 16:29:00 +05:30
Piyush Singariya
2c691b5a75 fix: test fixed 2026-02-17 16:28:54 +05:30
Piyush Singariya
cd7e1bb114 Merge branch 'main' into merge-json-col-fields 2026-02-17 16:22:58 +05:30
Piyush Singariya
a1d2ec8b8a fix: remove unused function 2026-02-17 16:18:38 +05:30
Piyush Singariya
8bbafb52d5 fix: go.mod required changes 2026-02-17 16:16:17 +05:30
Piyush Singariya
075cfab463 feat: mapping body_v2.message:string map to body 2026-02-17 13:26:34 +05:30
Piyush Singariya
86bccaac0c test: blocked on pr #10153 2026-02-16 15:24:31 +05:30
Piyush Singariya
de1aac63c0 revert: more unrelated change 2026-02-16 13:19:49 +05:30
Piyush Singariya
14fe8745b5 Merge branch 'main' into merge-json-col-fields 2026-02-16 13:14:39 +05:30
Piyush Singariya
4013c7ee03 revert: few unrelated changes 2026-02-16 13:13:07 +05:30
Piyush Singariya
0d34360e0b fix: handle datatype collision 2026-01-30 12:17:28 +05:30
srikanthccv
d204c89dec Merge branch 'main' into merge-json-col-fields 2026-01-30 02:12:14 +05:30
Piyush Singariya
8dd33c1ab7 Merge branch 'main' into merge-json-col-fields 2026-01-29 19:57:22 +05:30
Piyush Singariya
8e5c3d5ae1 chore: merge json fields 2026-01-29 16:46:00 +05:30
Piyush Singariya
d45bb52f33 Merge branch 'has-jsonqb' into merge-json-col-fields 2026-01-29 13:21:13 +05:30
Piyush Singariya
e71818292d fix: go test flakiness 2026-01-29 10:17:53 +05:30
Piyush Singariya
37557f7f24 Merge branch 'main' into has-jsonqb 2026-01-29 09:12:22 +05:30
Piyush Singariya
27ff102660 Merge branch 'main' into has-jsonqb 2026-01-28 17:44:48 +05:30
Piyush Singariya
cb2aa4cffd fix: tests 2026-01-28 17:42:17 +05:30
Piyush Singariya
58d1d84ec7 test: fix 2026-01-28 15:34:22 +05:30
Piyush Singariya
d8e116a7bc fix: merge conflict 2026-01-28 15:15:50 +05:30
Piyush Singariya
6a48bdc37e Merge branch 'main' into has-jsonqb 2026-01-28 15:15:17 +05:30
Piyush Singariya
ffb62432f8 chore: var renamed 2026-01-28 14:42:51 +05:30
Piyush Singariya
57c51f070c fix: merge json body columns together 2026-01-28 14:36:15 +05:30
Piyush Singariya
36becfc7a2 fix: removed comment 2026-01-27 13:20:52 +05:30
Piyush Singariya
8e71de09f3 fix: remove unnecessary bool checking 2026-01-27 13:16:30 +05:30
Piyush Singariya
56de92de73 fix: changes based on review from Srikanth 2026-01-27 13:12:32 +05:30
Piyush Singariya
62b10f8e77 Merge branch 'main' into has-jsonqb 2026-01-27 10:04:32 +05:30
Piyush Singariya
20b53d7856 fix: review based on tushar 2026-01-27 10:04:15 +05:30
Piyush Singariya
8f2c506304 fix: json qb test fix 2026-01-22 22:00:06 +05:30
Srikanth Chekuri
7b5b9027dd Merge branch 'main' into has-jsonqb 2026-01-22 20:19:39 +05:30
Piyush Singariya
b77f97fcb7 fix: tests 2026-01-22 17:24:26 +05:30
Piyush Singariya
62942a4162 fix: tests 2026-01-22 15:47:20 +05:30
Piyush Singariya
349bbbbf1d Merge branch 'main' into has-jsonqb 2026-01-22 12:36:45 +05:30
Piyush Singariya
1966a7a5f6 fix: empty filteredArrays condition 2026-01-22 12:36:29 +05:30
Piyush Singariya
a4eed9ff13 fix: build json plans in metadata 2026-01-22 12:33:51 +05:30
Piyush Singariya
24d1ee33b5 revert: gitignore change 2026-01-22 10:39:02 +05:30
Srikanth Chekuri
3402203021 Merge branch 'main' into has-jsonqb 2026-01-21 13:47:39 +05:30
Piyush Singariya
e8e4897cc8 fix: tests GroupBy 2026-01-20 12:10:44 +05:30
Piyush Singariya
96fb88aaee fix: ignored .vscode in gitignore 2026-01-20 11:47:34 +05:30
Piyush Singariya
5a00e6c2cd Merge branch 'main' into has-jsonqb 2026-01-20 11:44:32 +05:30
Piyush Singariya
e2500cff7d fix: tests expected queries and values 2026-01-20 11:40:56 +05:30
Piyush Singariya
4864c3bc37 feat: has JSON QB 2026-01-20 11:23:50 +05:30
48 changed files with 1515 additions and 3486 deletions

View File

@@ -17,7 +17,5 @@
},
"[html]": {
"editor.defaultFormatter": "vscode.html-language-features"
},
"python-envs.defaultEnvManager": "ms-python.python:system",
"python-envs.pythonProjects": []
}
}

View File

@@ -3001,68 +3001,6 @@ paths:
summary: Update auth domain
tags:
- authdomains
/api/v1/export_raw_data:
post:
deprecated: false
description: This endpoints allows complex query exporting raw data for traces
and logs
operationId: HandleExportRawDataPOST
parameters:
- description: The output format for the export.
in: query
name: format
schema:
default: csv
description: The output format for the export.
enum:
- csv
- jsonl
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/Querybuildertypesv5QueryRangeRequest'
responses:
"200":
content:
application/json:
schema:
type: string
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Export raw data
tags:
- logs
- traces
/api/v1/fields/keys:
get:
deprecated: false

View File

@@ -123,7 +123,6 @@ if err := router.Handle("/api/v1/things", handler.New(
Description: "This endpoint creates a thing",
Request: new(types.PostableThing),
RequestContentType: "application/json",
RequestQuery: new(types.QueryableThing),
Response: new(types.GettableThing),
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusCreated,
@@ -156,8 +155,6 @@ The `handler.New` function ties the HTTP handler to OpenAPI metadata via `OpenAP
- **Request / RequestContentType**:
- `Request` is a Go type that describes the request body or form.
- `RequestContentType` is usually `"application/json"` or `"application/x-www-form-urlencoded"` (for callbacks like SAML).
- **RequestQuery**:
- `RequestQuery` is a Go type that descirbes query url params.
- **RequestExamples**: An array of `handler.OpenAPIExample` that provide concrete request payloads in the generated spec. See [Adding request examples](#adding-request-examples) below.
- **Response / ResponseContentType**:
- `Response` is the Go type for the successful response payload.

View File

@@ -20,113 +20,11 @@ import { useMutation, useQuery } from 'react-query';
import type { BodyType, ErrorType } from '../../../generatedAPIInstance';
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
import type {
HandleExportRawDataPOSTParams,
ListPromotedAndIndexedPaths200,
PromotetypesPromotePathDTO,
Querybuildertypesv5QueryRangeRequestDTO,
RenderErrorResponseDTO,
} from '../sigNoz.schemas';
/**
* This endpoints allows complex query exporting raw data for traces and logs
* @summary Export raw data
*/
export const handleExportRawDataPOST = (
querybuildertypesv5QueryRangeRequestDTO: BodyType<Querybuildertypesv5QueryRangeRequestDTO>,
params?: HandleExportRawDataPOSTParams,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<string>({
url: `/api/v1/export_raw_data`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: querybuildertypesv5QueryRangeRequestDTO,
params,
signal,
});
};
export const getHandleExportRawDataPOSTMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
> => {
const mutationKey = ['handleExportRawDataPOST'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
}
> = (props) => {
const { data, params } = props ?? {};
return handleExportRawDataPOST(data, params);
};
return { mutationFn, ...mutationOptions };
};
export type HandleExportRawDataPOSTMutationResult = NonNullable<
Awaited<ReturnType<typeof handleExportRawDataPOST>>
>;
export type HandleExportRawDataPOSTMutationBody = BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
export type HandleExportRawDataPOSTMutationError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Export raw data
*/
export const useHandleExportRawDataPOST = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
> => {
const mutationOptions = getHandleExportRawDataPOSTMutationOptions(options);
return useMutation(mutationOptions);
};
/**
* This endpoints promotes and indexes paths
* @summary Promote and index paths

View File

@@ -2896,19 +2896,6 @@ export type DeleteAuthDomainPathParameters = {
export type UpdateAuthDomainPathParameters = {
id: string;
};
export type HandleExportRawDataPOSTParams = {
/**
* @enum csv,jsonl
* @type string
* @description The output format for the export.
*/
format?: HandleExportRawDataPOSTFormat;
};
export enum HandleExportRawDataPOSTFormat {
csv = 'csv',
jsonl = 'jsonl',
}
export type GetFieldsKeysParams = {
/**
* @description undefined

2
go.mod
View File

@@ -11,7 +11,6 @@ require (
github.com/SigNoz/signoz-otel-collector v0.144.2
github.com/antlr4-go/antlr/v4 v4.13.1
github.com/antonmedv/expr v1.15.3
github.com/bytedance/sonic v1.14.1
github.com/cespare/xxhash/v2 v2.3.0
github.com/coreos/go-oidc/v3 v3.17.0
github.com/dgraph-io/ristretto/v2 v2.3.0
@@ -106,6 +105,7 @@ require (
github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
github.com/aws/smithy-go v1.24.0 // indirect
github.com/bytedance/gopkg v0.1.3 // indirect
github.com/bytedance/sonic v1.14.1 // indirect
github.com/bytedance/sonic/loader v0.3.0 // indirect
github.com/cloudwego/base64x v0.1.6 // indirect
github.com/gabriel-vasile/mimetype v1.4.8 // indirect

View File

@@ -18,7 +18,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/preference"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/session"
"github.com/SigNoz/signoz/pkg/modules/user"
@@ -48,7 +47,6 @@ type provider struct {
gatewayHandler gateway.Handler
fieldsHandler fields.Handler
authzHandler authz.Handler
rawDataExportHandler rawdataexport.Handler
zeusHandler zeus.Handler
querierHandler querier.Handler
serviceAccountHandler serviceaccount.Handler
@@ -71,7 +69,6 @@ func NewFactory(
gatewayHandler gateway.Handler,
fieldsHandler fields.Handler,
authzHandler authz.Handler,
rawDataExportHandler rawdataexport.Handler,
zeusHandler zeus.Handler,
querierHandler querier.Handler,
serviceAccountHandler serviceaccount.Handler,
@@ -97,7 +94,6 @@ func NewFactory(
gatewayHandler,
fieldsHandler,
authzHandler,
rawDataExportHandler,
zeusHandler,
querierHandler,
serviceAccountHandler,
@@ -125,7 +121,6 @@ func newProvider(
gatewayHandler gateway.Handler,
fieldsHandler fields.Handler,
authzHandler authz.Handler,
rawDataExportHandler rawdataexport.Handler,
zeusHandler zeus.Handler,
querierHandler querier.Handler,
serviceAccountHandler serviceaccount.Handler,
@@ -151,7 +146,6 @@ func newProvider(
gatewayHandler: gatewayHandler,
fieldsHandler: fieldsHandler,
authzHandler: authzHandler,
rawDataExportHandler: rawDataExportHandler,
zeusHandler: zeusHandler,
querierHandler: querierHandler,
serviceAccountHandler: serviceAccountHandler,
@@ -227,10 +221,6 @@ func (provider *provider) AddToRouter(router *mux.Router) error {
return err
}
if err := provider.addRawDataExportRoutes(router); err != nil {
return err
}
if err := provider.addZeusRoutes(router); err != nil {
return err
}

View File

@@ -1,33 +0,0 @@
package signozapiserver
import (
"net/http"
"github.com/SigNoz/signoz/pkg/http/handler"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/types/exporttypes"
v5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/gorilla/mux"
)
func (provider *provider) addRawDataExportRoutes(router *mux.Router) error {
if err := router.Handle("/api/v1/export_raw_data", handler.New(provider.authZ.ViewAccess(provider.rawDataExportHandler.ExportRawData), handler.OpenAPIDef{
ID: "HandleExportRawDataPOST",
Tags: []string{"logs", "traces"},
Summary: "Export raw data",
Description: "This endpoints allows complex query exporting raw data for traces and logs",
Request: new(v5.QueryRangeRequest),
RequestQuery: new(exporttypes.ExportRawDataFormatQueryParam),
RequestContentType: "application/json",
Response: nil,
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest},
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
})).Methods(http.MethodPost).GetError(); err != nil {
return err
}
return nil
}

View File

@@ -78,7 +78,7 @@ func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetype
// add the paths that are not promoted but have indexes
for path, indexes := range aggr {
path := strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix)
path := strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
path = telemetrytypes.BodyJSONStringSearchPrefix + path
response = append(response, promotetypes.PromotePath{
Path: path,
@@ -163,7 +163,7 @@ func (m *module) PromoteAndIndexPaths(
}
}
if len(it.Indexes) > 0 {
parentColumn := telemetrylogs.LogsV2BodyJSONColumn
parentColumn := telemetrylogs.LogsV2BodyV2Column
// if the path is already promoted or is being promoted, add it to the promoted column
if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
parentColumn = telemetrylogs.LogsV2BodyPromotedColumn

View File

@@ -6,19 +6,20 @@ import (
"fmt"
"io"
"net/http"
"slices"
"net/url"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/http/binding"
"github.com/SigNoz/signoz/pkg/http/render"
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/exporttypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
@@ -30,31 +31,129 @@ func NewHandler(module rawdataexport.Module) rawdataexport.Handler {
return &handler{module: module}
}
// ExportRawData handles data export requests.
//
// API Documentation:
// Endpoint: GET /api/v1/export_raw_data
//
// Query Parameters:
//
// - source (optional): Type of data to export ["logs" (default), "metrics", "traces"]
// Note: Currently only "logs" is fully supported
//
// - format (optional): Output format ["csv" (default), "jsonl"]
//
// - start (required): Start time for query (Unix timestamp in nanoseconds)
//
// - end (required): End time for query (Unix timestamp in nanoseconds)
//
// - limit (optional): Maximum number of rows to export
// Constraints: Must be positive and cannot exceed MAX_EXPORT_ROW_COUNT_LIMIT
//
// - filter (optional): Filter expression to apply to the query
//
// - columns (optional): Specific columns to include in export
// Default: all columns are returned
// Format: ["context.field:type", "context.field", "field"]
//
// - order_by (optional): Sorting specification ["column:direction" or "context.field:type:direction"]
// Direction: "asc" or "desc"
// Default: ["timestamp:desc", "id:desc"]
//
// Response Headers:
// - Content-Type: "text/csv" or "application/x-ndjson"
// - Content-Encoding: "gzip" (handled by HTTP middleware)
// - Content-Disposition: "attachment; filename=\"data_exported.[format]\""
// - Cache-Control: "no-cache"
// - Vary: "Accept-Encoding"
// - Transfer-Encoding: "chunked"
// - Trailers: X-Response-Complete
//
// Response Format:
//
// CSV: Headers in first row, data in subsequent rows
// JSONL: One JSON object per line
//
// Example Usage:
//
// Basic CSV export:
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000
//
// Export with columns and format:
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000&format=jsonl
// &columns=timestamp&columns=severity&columns=message
//
// Export with filter and ordering:
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000
// &filter=severity="error"&order_by=timestamp:desc&limit=1000
func (handler *handler) ExportRawData(rw http.ResponseWriter, r *http.Request) {
var queryRangeRequest qbtypes.QueryRangeRequest
var formatParam exporttypes.ExportRawDataFormatQueryParam
if err := binding.Query.BindQuery(r.URL.Query(), &formatParam); err != nil {
render.Error(rw, err)
return
}
format := formatParam.Format
if err := binding.JSON.BindBody(r.Body, &queryRangeRequest); err != nil {
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid request body: %v", err))
return
}
if err := validateSpecForExport(&queryRangeRequest); err != nil {
source, err := getExportQuerySource(r.URL.Query())
if err != nil {
render.Error(rw, err)
return
}
if err := validateAndApplyDefaultExportLimits(queryRangeRequest.CompositeQuery.Queries); err != nil {
switch source {
case "logs":
handler.exportLogs(rw, r)
case "traces":
handler.exportTraces(rw, r)
case "metrics":
handler.exportMetrics(rw, r)
default:
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid source: must be logs"))
}
}
func (handler *handler) exportMetrics(rw http.ResponseWriter, r *http.Request) {
render.Error(rw, errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported, "metrics export is not yet supported"))
}
func (handler *handler) exportTraces(rw http.ResponseWriter, r *http.Request) {
render.Error(rw, errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported, "traces export is not yet supported"))
}
func (handler *handler) exportLogs(rw http.ResponseWriter, r *http.Request) {
// Set up response headers
rw.Header().Set("Cache-Control", "no-cache")
rw.Header().Set("Vary", "Accept-Encoding") // Indicate that response varies based on Accept-Encoding
rw.Header().Set("Access-Control-Expose-Headers", "Content-Disposition, X-Response-Complete")
rw.Header().Set("Trailer", "X-Response-Complete")
rw.Header().Set("Transfer-Encoding", "chunked")
queryParams := r.URL.Query()
startTime, endTime, err := getExportQueryTimeRange(queryParams)
if err != nil {
render.Error(rw, err)
return
}
queryRangeRequest.UseDefaultOrderBy()
limit, err := getExportQueryLimit(queryParams)
if err != nil {
render.Error(rw, err)
return
}
format, err := getExportQueryFormat(queryParams)
if err != nil {
render.Error(rw, err)
return
}
// Set appropriate content type and filename
filename := fmt.Sprintf("data_exported_%s.%s", time.Now().Format("2006-01-02_150405"), format)
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
filterExpression := queryParams.Get("filter")
orderByExpression, err := getExportQueryOrderBy(queryParams)
if err != nil {
render.Error(rw, err)
return
}
columns := getExportQueryColumns(queryParams)
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
@@ -62,98 +161,76 @@ func (handler *handler) ExportRawData(rw http.ResponseWriter, r *http.Request) {
return
}
orgID := valuer.MustNewUUID(claims.OrgID)
orgID, err := valuer.NewUUID(claims.OrgID)
if err != nil {
render.Error(rw, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "orgID is invalid"))
return
}
setExportResponseHeaders(rw, format)
queryRangeRequest := qbtypes.QueryRangeRequest{
Start: startTime,
End: endTime,
RequestType: qbtypes.RequestTypeRaw,
CompositeQuery: qbtypes.CompositeQuery{
Queries: []qbtypes.QueryEnvelope{
{
Type: qbtypes.QueryTypeBuilder,
Spec: nil,
},
},
},
}
spec := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Name: "raw",
Filter: &qbtypes.Filter{
Expression: filterExpression,
},
Limit: limit,
Order: orderByExpression,
}
spec.SelectFields = columns
queryRangeRequest.CompositeQuery.Queries[0].Spec = spec
// This will signal Export module to stop sending data
doneChan := make(chan any)
defer close(doneChan)
rowChan, errChan := handler.module.ExportRawData(r.Context(), orgID, &queryRangeRequest, doneChan)
isComplete, err := handler.executeExport(rowChan, errChan, format, rw)
if err != nil {
render.Error(rw, err)
return
}
rw.Header().Set("X-Response-Complete", strconv.FormatBool(isComplete))
}
var isComplete bool
// validateSpecForExport validates query specs
func validateSpecForExport(req *qbtypes.QueryRangeRequest) error {
queries := req.CompositeQuery.Queries
// If the trace operator query is not present, and there are multiple queries, return an error
if req.TraceOperatorQueryIndex() == -1 && len(queries) > 1 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "multiple queries not allowed without a trace operator query")
}
for idx := range queries {
switch spec := queries[idx].Spec.(type) {
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
qbtypes.QueryBuilderTraceOperator:
// Supported spec types
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported query at index %d type: %T", idx, spec)
}
}
opts := append(qbtypes.GetValidationOptions(req.RequestType), qbtypes.WithSkipLimitOffsetValidation())
return req.Validate(opts...)
}
func validateAndApplyDefaultExportLimits(queries []qbtypes.QueryEnvelope) error {
for idx := range queries {
limit := queries[idx].GetLimit()
if limit == 0 {
limit = DefaultExportRowCountLimit
} else if limit < 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "limit must be positive")
} else if limit > MaxExportRowCountLimit {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "limit cannot be more than %d", MaxExportRowCountLimit)
}
queries[idx].SetLimit(limit)
}
return nil
}
// setExportResponseHeaders sets common HTTP headers for export responses.
func setExportResponseHeaders(rw http.ResponseWriter, format string) {
rw.Header().Set("Cache-Control", "no-cache")
rw.Header().Set("Vary", "Accept-Encoding")
rw.Header().Set("Access-Control-Expose-Headers", "Content-Disposition, X-Response-Complete")
rw.Header().Set("Trailer", "X-Response-Complete")
rw.Header().Set("Transfer-Encoding", "chunked")
filename := fmt.Sprintf("data_exported_%s.%s", time.Now().Format("2006-01-02_150405"), format)
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
}
// executeExport streams data from rowChan to the response writer in the specified format.
func (handler *handler) executeExport(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, format string, rw http.ResponseWriter) (bool, error) {
switch format {
case "csv", "":
rw.Header().Set("Content-Type", "text/csv")
csvWriter := csv.NewWriter(rw)
isComplete, err := handler.exportRawDataCSV(rowChan, errChan, csvWriter)
isComplete, err = handler.exportLogsCSV(rowChan, errChan, csvWriter)
if err != nil {
return false, err
render.Error(rw, err)
return
}
csvWriter.Flush()
return isComplete, nil
case "jsonl":
rw.Header().Set("Content-Type", "application/x-ndjson")
return handler.exportRawDataJSONL(rowChan, errChan, rw)
isComplete, err = handler.exportLogsJSONL(rowChan, errChan, rw)
if err != nil {
render.Error(rw, err)
return
}
default:
return false, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl")
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl"))
return
}
rw.Header().Set("X-Response-Complete", strconv.FormatBool(isComplete))
}
// exportRawDataCSV is a generic CSV export function that works with any raw data (logs, traces, etc.)
func (handler *handler) exportRawDataCSV(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, csvWriter *csv.Writer) (bool, error) {
func (handler *handler) exportLogsCSV(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, csvWriter *csv.Writer) (bool, error) {
var header []string
headerToIndexMapping := make(map[string]int)
headerToIndexMapping := make(map[string]int, len(header))
totalBytes := uint64(0)
for {
@@ -191,8 +268,8 @@ func (handler *handler) exportRawDataCSV(rowChan <-chan *qbtypes.RawRow, errChan
}
}
// exportRawDataJSONL is a generic JSONL export function that works with any raw data (logs, traces, etc.)
func (handler *handler) exportRawDataJSONL(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, writer io.Writer) (bool, error) {
func (handler *handler) exportLogsJSONL(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, writer io.Writer) (bool, error) {
totalBytes := uint64(0)
for {
select {
@@ -200,11 +277,9 @@ func (handler *handler) exportRawDataJSONL(rowChan <-chan *qbtypes.RawRow, errCh
if !ok {
return true, nil
}
jsonBytes, err := json.Marshal(row.Data)
if err != nil {
return false, errors.NewUnexpectedf(errors.CodeInternal, "error marshaling JSON: %s", err)
}
totalBytes += uint64(len(jsonBytes)) + 1
// Handle JSON format (JSONL - one object per line)
jsonBytes, _ := json.Marshal(row.Data)
totalBytes += uint64(len(jsonBytes)) + 1 // +1 for newline
if _, err := writer.Write(jsonBytes); err != nil {
return false, errors.NewUnexpectedf(errors.CodeInternal, "error writing JSON: %s", err)
@@ -224,33 +299,74 @@ func (handler *handler) exportRawDataJSONL(rowChan <-chan *qbtypes.RawRow, errCh
}
}
// priorityColumns defines the columns that should appear first in the CSV output, in order.
var priorityColumns = []string{"timestamp", "id"}
func getExportQuerySource(queryParams url.Values) (string, error) {
switch queryParams.Get("source") {
case "logs", "":
return "logs", nil
case "metrics":
return "metrics", errors.NewInvalidInputf(errors.CodeInvalidInput, "metrics export not yet supported")
case "traces":
return "traces", errors.NewInvalidInputf(errors.CodeInvalidInput, "traces export not yet supported")
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid source: must be logs, metrics or traces")
}
}
func getExportQueryFormat(queryParams url.Values) (string, error) {
switch queryParams.Get("format") {
case "csv", "":
return "csv", nil
case "jsonl":
return "jsonl", nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl")
}
}
func getExportQueryLimit(queryParams url.Values) (int, error) {
limitStr := queryParams.Get("limit")
if limitStr == "" {
return DefaultExportRowCountLimit, nil
} else {
limit, err := strconv.Atoi(limitStr)
if err != nil {
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid limit format: %s", err.Error())
}
if limit <= 0 {
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "limit must be positive")
}
if limit > MaxExportRowCountLimit {
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "limit cannot be more than %d", MaxExportRowCountLimit)
}
return limit, nil
}
}
func getExportQueryTimeRange(queryParams url.Values) (uint64, uint64, error) {
startTimeStr := queryParams.Get("start")
endTimeStr := queryParams.Get("end")
if startTimeStr == "" || endTimeStr == "" {
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "start and end time are required")
}
startTime, err := strconv.ParseUint(startTimeStr, 10, 64)
if err != nil {
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid start time format: %s", err.Error())
}
endTime, err := strconv.ParseUint(endTimeStr, 10, 64)
if err != nil {
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid end time format: %s", err.Error())
}
return startTime, endTime, nil
}
func constructCSVHeaderFromQueryResponse(data map[string]any) []string {
header := make([]string, 0, len(data))
for key := range data {
header = append(header, key)
}
// This is to ensure CSV output is consistent across multiple queries
slices.SortFunc(header, func(a, b string) int {
ai, bi := slices.Index(priorityColumns, a), slices.Index(priorityColumns, b)
switch {
case ai != -1 && bi != -1:
return ai - bi
case ai != -1:
return -1
case bi != -1:
return 1
default:
if a < b {
return -1
} else if a > b {
return 1
}
return 0
}
})
return header
}
@@ -311,12 +427,9 @@ func constructCSVRecordFromQueryResponse(data map[string]any, headerToIndexMappi
valueStr = v.String()
default:
jsonBytes, err := json.Marshal(v)
if err != nil {
valueStr = fmt.Sprintf("%v", v)
} else {
valueStr = string(jsonBytes)
}
// For all other complex types (maps, structs, etc.)
jsonBytes, _ := json.Marshal(v)
valueStr = string(jsonBytes)
}
record[index] = sanitizeForCSV(valueStr)
@@ -325,6 +438,26 @@ func constructCSVRecordFromQueryResponse(data map[string]any, headerToIndexMappi
return record
}
// getExportQueryColumns parses the "columns" query parameters and returns a slice of TelemetryFieldKey structs.
// Each column should be a valid telemetry field key in the format "context.field:type" or "context.field" or "field"
func getExportQueryColumns(queryParams url.Values) []telemetrytypes.TelemetryFieldKey {
columnParams := queryParams["columns"]
columns := make([]telemetrytypes.TelemetryFieldKey, 0, len(columnParams))
for _, columnStr := range columnParams {
// Skip empty strings
columnStr = strings.TrimSpace(columnStr)
if columnStr == "" {
continue
}
columns = append(columns, telemetrytypes.GetFieldKeyFromKeyText(columnStr))
}
return columns
}
func getsizeOfStringSlice(slice []string) uint64 {
var totalBytes uint64
for _, str := range slice {
@@ -332,3 +465,52 @@ func getsizeOfStringSlice(slice []string) uint64 {
}
return totalBytes
}
// getExportQueryOrderBy parses the "order_by" query parameters and returns a slice of OrderBy structs.
// Each "order_by" parameter should be in the format "column:direction"
// Each "column" should be a valid telemetry field key in the format "context.field:type" or "context.field" or "field"
func getExportQueryOrderBy(queryParams url.Values) ([]qbtypes.OrderBy, error) {
orderByParam := queryParams.Get("order_by")
orderByParam = strings.TrimSpace(orderByParam)
if orderByParam == "" {
return telemetrylogs.DefaultLogsV2SortingOrder, nil
}
parts := strings.Split(orderByParam, ":")
if len(parts) != 2 && len(parts) != 3 {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order_by format: %s, should be <column>:<direction>", orderByParam)
}
column := strings.Join(parts[:len(parts)-1], ":")
direction := parts[len(parts)-1]
orderDirection, ok := qbtypes.OrderDirectionMap[direction]
if !ok {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order_by direction: %s, should be one of %s, %s", direction, qbtypes.OrderDirectionAsc, qbtypes.OrderDirectionDesc)
}
orderByKey := telemetrytypes.GetFieldKeyFromKeyText(column)
orderBy := []qbtypes.OrderBy{
{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: orderByKey,
},
Direction: orderDirection,
},
}
// If we are ordering by the timestamp column, also order by the ID column
if orderByKey.Name == telemetrylogs.LogsV2TimestampColumn {
orderBy = append(orderBy, qbtypes.OrderBy{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
Direction: orderDirection,
})
}
return orderBy, nil
}

View File

@@ -2,84 +2,58 @@ package implrawdataexport
import (
"net/url"
"strconv"
"testing"
"github.com/SigNoz/signoz/pkg/http/binding"
"github.com/SigNoz/signoz/pkg/types/exporttypes"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
)
func TestExportRawDataFormatQueryParam_BindingDefaults(t *testing.T) {
var params exporttypes.ExportRawDataFormatQueryParam
err := binding.Query.BindQuery(url.Values{}, &params)
assert.NoError(t, err)
assert.Equal(t, "csv", params.Format)
}
func logQuery(limit int) qbtypes.QueryEnvelope {
return qbtypes.QueryEnvelope{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{Limit: limit},
}
}
func traceQuery(limit int) qbtypes.QueryEnvelope {
return qbtypes.QueryEnvelope{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{Limit: limit},
}
}
func traceOperatorQuery(limit int) qbtypes.QueryEnvelope {
return qbtypes.QueryEnvelope{
Type: qbtypes.QueryTypeTraceOperator,
Spec: qbtypes.QueryBuilderTraceOperator{Limit: limit, Expression: "A"},
}
}
func makeRequest(queries ...qbtypes.QueryEnvelope) qbtypes.QueryRangeRequest {
return qbtypes.QueryRangeRequest{
Start: 1000000000000,
End: 1000003600000,
RequestType: qbtypes.RequestTypeRaw,
CompositeQuery: qbtypes.CompositeQuery{Queries: queries},
}
}
func TestValidateSpecForExport(t *testing.T) {
func TestGetExportQuerySource(t *testing.T) {
tests := []struct {
name string
req qbtypes.QueryRangeRequest
expectedError bool
name string
queryParams url.Values
expectedSource string
expectedError bool
}{
{
name: "single log query",
req: makeRequest(logQuery(0)),
name: "default logs source",
queryParams: url.Values{},
expectedSource: "logs",
expectedError: false,
},
{
name: "single trace query",
req: makeRequest(traceQuery(0)),
name: "explicit logs source",
queryParams: url.Values{"source": {"logs"}},
expectedSource: "logs",
expectedError: false,
},
{
name: "trace operator alone",
req: makeRequest(traceOperatorQuery(0)),
name: "metrics source - not supported",
queryParams: url.Values{"source": {"metrics"}},
expectedSource: "metrics",
expectedError: true,
},
{
name: "multiple queries without trace operator",
req: makeRequest(logQuery(0), traceQuery(0)),
expectedError: true,
name: "traces source - not supported",
queryParams: url.Values{"source": {"traces"}},
expectedSource: "traces",
expectedError: true,
},
{
name: "unsupported query type",
req: makeRequest(qbtypes.QueryEnvelope{Type: qbtypes.QueryTypeBuilder, Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{}}),
expectedError: true,
name: "invalid source",
queryParams: url.Values{"source": {"invalid"}},
expectedSource: "",
expectedError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateSpecForExport(&tt.req)
source, err := getExportQuerySource(tt.queryParams)
assert.Equal(t, tt.expectedSource, source)
if tt.expectedError {
assert.Error(t, err)
} else {
@@ -89,69 +63,456 @@ func TestValidateSpecForExport(t *testing.T) {
}
}
func TestValidateAndApplyDefaultExportLimits(t *testing.T) {
func TestGetExportQueryFormat(t *testing.T) {
tests := []struct {
name string
queries []qbtypes.QueryEnvelope
expectedError bool
checkQueries func(t *testing.T, queries []qbtypes.QueryEnvelope)
name string
queryParams url.Values
expectedFormat string
expectedError bool
}{
{
name: "single log query, zero limit gets default",
queries: makeRequest(logQuery(0)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
},
name: "default csv format",
queryParams: url.Values{},
expectedFormat: "csv",
expectedError: false,
},
{
name: "single log query, valid limit kept",
queries: makeRequest(logQuery(1000)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, 1000, q[0].GetLimit())
},
name: "explicit csv format",
queryParams: url.Values{"format": {"csv"}},
expectedFormat: "csv",
expectedError: false,
},
{
name: "single log query, max limit kept",
queries: makeRequest(logQuery(MaxExportRowCountLimit)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, MaxExportRowCountLimit, q[0].GetLimit())
},
name: "jsonl format",
queryParams: url.Values{"format": {"jsonl"}},
expectedFormat: "jsonl",
expectedError: false,
},
{
name: "single log query, limit exceeds max",
queries: makeRequest(logQuery(MaxExportRowCountLimit + 1)).CompositeQuery.Queries,
name: "invalid format",
queryParams: url.Values{"format": {"xml"}},
expectedFormat: "",
expectedError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
format, err := getExportQueryFormat(tt.queryParams)
assert.Equal(t, tt.expectedFormat, format)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
func TestGetExportQueryLimit(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedLimit int
expectedError bool
}{
{
name: "default limit",
queryParams: url.Values{},
expectedLimit: DefaultExportRowCountLimit,
expectedError: false,
},
{
name: "valid limit",
queryParams: url.Values{"limit": {"5000"}},
expectedLimit: 5000,
expectedError: false,
},
{
name: "maximum limit",
queryParams: url.Values{"limit": {strconv.Itoa(MaxExportRowCountLimit)}},
expectedLimit: MaxExportRowCountLimit,
expectedError: false,
},
{
name: "limit exceeds maximum",
queryParams: url.Values{"limit": {"100000"}},
expectedLimit: 0,
expectedError: true,
},
{
name: "single log query, negative limit",
queries: makeRequest(logQuery(-1)).CompositeQuery.Queries,
name: "invalid limit format",
queryParams: url.Values{"limit": {"invalid"}},
expectedLimit: 0,
expectedError: true,
},
{
name: "single trace query, zero limit gets default",
queries: makeRequest(traceQuery(0)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
name: "negative limit",
queryParams: url.Values{"limit": {"-100"}},
expectedLimit: 0,
expectedError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
limit, err := getExportQueryLimit(tt.queryParams)
assert.Equal(t, tt.expectedLimit, limit)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}
func TestGetExportQueryTimeRange(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedStartTime uint64
expectedEndTime uint64
expectedError bool
}{
{
name: "valid time range",
queryParams: url.Values{
"start": {"1640995200"},
"end": {"1641081600"},
},
expectedStartTime: 1640995200,
expectedEndTime: 1641081600,
expectedError: false,
},
{
name: "missing start time",
queryParams: url.Values{"end": {"1641081600"}},
expectedError: true,
},
{
name: "missing end time",
queryParams: url.Values{"start": {"1640995200"}},
expectedError: true,
},
{
name: "missing both times",
queryParams: url.Values{},
expectedError: true,
},
{
name: "invalid start time format",
queryParams: url.Values{
"start": {"invalid"},
"end": {"1641081600"},
},
expectedError: true,
},
{
name: "invalid end time format",
queryParams: url.Values{
"start": {"1640995200"},
"end": {"invalid"},
},
expectedError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
startTime, endTime, err := getExportQueryTimeRange(tt.queryParams)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.expectedStartTime, startTime)
assert.Equal(t, tt.expectedEndTime, endTime)
}
})
}
}
func TestGetExportQueryColumns(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedColumns []telemetrytypes.TelemetryFieldKey
}{
{
name: "no columns specified",
queryParams: url.Values{},
expectedColumns: []telemetrytypes.TelemetryFieldKey{},
},
{
name: "single column",
queryParams: url.Values{
"columns": {"timestamp"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
},
},
{
name: "trace operator alone, zero limit gets default",
queries: makeRequest(traceOperatorQuery(0)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
name: "multiple columns",
queryParams: url.Values{
"columns": {"timestamp", "message", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "message"},
{Name: "level"},
},
},
{
name: "empty column name (should be skipped)",
queryParams: url.Values{
"columns": {"timestamp", "", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "level"},
},
},
{
name: "whitespace column name (should be skipped)",
queryParams: url.Values{
"columns": {"timestamp", " ", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "level"},
},
},
{
name: "valid column name with data type",
queryParams: url.Values{
"columns": {"timestamp", "attribute.user:string", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "user", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "level"},
},
},
{
name: "valid column name with dot notation",
queryParams: url.Values{
"columns": {"timestamp", "attribute.user.string", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "user.string", FieldContext: telemetrytypes.FieldContextAttribute},
{Name: "level"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateAndApplyDefaultExportLimits(tt.queries)
columns := getExportQueryColumns(tt.queryParams)
assert.Equal(t, len(tt.expectedColumns), len(columns))
for i, expectedCol := range tt.expectedColumns {
assert.Equal(t, expectedCol, columns[i])
}
})
}
}
func TestGetExportQueryOrderBy(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedOrder []qbtypes.OrderBy
expectedError bool
}{
{
name: "no order specified",
queryParams: url.Values{},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "single order error, direction not specified",
queryParams: url.Values{
"order_by": {"timestamp"},
},
expectedOrder: nil,
expectedError: true,
},
{
name: "single order no error",
queryParams: url.Values{
"order_by": {"timestamp:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "multiple orders",
queryParams: url.Values{
"order_by": {"timestamp:asc", "body:desc", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "empty order name (should be skipped)",
queryParams: url.Values{
"order_by": {"timestamp:asc", "", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "whitespace order name (should be skipped)",
queryParams: url.Values{
"order_by": {"timestamp:asc", " ", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "invalid order name (should error out)",
queryParams: url.Values{
"order_by": {"attributes.user:", "id:asc"},
},
expectedOrder: nil,
expectedError: true,
},
{
name: "valid order name (should be included)",
queryParams: url.Values{
"order_by": {"attribute.user:string:desc", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "user",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
},
expectedError: false,
},
{
name: "valid order name (should be included)",
queryParams: url.Values{
"order_by": {"attribute.user.string:desc", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "user.string",
FieldContext: telemetrytypes.FieldContextAttribute,
},
},
},
},
expectedError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
order, err := getExportQueryOrderBy(tt.queryParams)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
if tt.checkQueries != nil {
tt.checkQueries(t, tt.queries)
assert.Equal(t, len(tt.expectedOrder), len(order))
for i, expectedOrd := range tt.expectedOrder {
assert.Equal(t, expectedOrd, order[i])
}
}
})
@@ -168,8 +529,13 @@ func TestConstructCSVHeaderFromQueryResponse(t *testing.T) {
header := constructCSVHeaderFromQueryResponse(data)
// Priority columns come first in order, then the rest alphabetically.
assert.Equal(t, []string{"timestamp", "id", "level", "message"}, header)
// Since map iteration order is not guaranteed, check that all expected keys are present
expectedKeys := []string{"timestamp", "message", "level", "id"}
assert.Equal(t, len(expectedKeys), len(header))
for _, key := range expectedKeys {
assert.Contains(t, header, key)
}
}
func TestConstructCSVRecordFromQueryResponse(t *testing.T) {

View File

@@ -28,18 +28,8 @@ func (m *Module) ExportRawData(ctx context.Context, orgID valuer.UUID, rangeRequ
instrumentationtypes.CodeFunctionName: "ExportRawData",
})
traceOperatorQueryIndex := rangeRequest.TraceOperatorQueryIndex()
queries := rangeRequest.CompositeQuery.Queries
// If the trace operator query is present, mark the queries other than trace operator as disabled
if traceOperatorQueryIndex > -1 {
for idx := range len(queries) {
if idx != traceOperatorQueryIndex {
queries[idx].SetDisabled(true)
}
}
}
spec := rangeRequest.CompositeQuery.Queries[0].Spec.(qbtypes.QueryBuilderQuery[qbtypes.LogAggregation])
rowCountLimit := spec.Limit
rowChan := make(chan *qbtypes.RawRow, 1)
errChan := make(chan error, 1)
@@ -53,62 +43,52 @@ func (m *Module) ExportRawData(ctx context.Context, orgID valuer.UUID, rangeRequ
defer close(errChan)
defer close(rowChan)
if traceOperatorQueryIndex > -1 {
// If the trace operator query is present, we need to export the data for the trace operator query only
exportRawDataForSingleQuery(m.querier, contextWithTimeout, orgID, rangeRequest, rowChan, errChan, doneChan, traceOperatorQueryIndex)
} else {
// If the trace operator query is not present, we need to export the data for the first query only
exportRawDataForSingleQuery(m.querier, contextWithTimeout, orgID, rangeRequest, rowChan, errChan, doneChan, 0)
rowCount := 0
for rowCount < rowCountLimit {
spec.Limit = min(ChunkSize, rowCountLimit-rowCount)
spec.Offset = rowCount
rangeRequest.CompositeQuery.Queries[0].Spec = spec
response, err := m.querier.QueryRange(contextWithTimeout, orgID, rangeRequest)
if err != nil {
errChan <- err
return
}
newRowsCount := 0
for _, result := range response.Data.Results {
resultData, ok := result.(*qbtypes.RawData)
if !ok {
errChan <- errors.NewInternalf(errors.CodeInternal, "expected RawData, got %T", result)
return
}
newRowsCount += len(resultData.Rows)
for _, row := range resultData.Rows {
select {
case rowChan <- row:
case <-doneChan:
return
case <-ctx.Done():
errChan <- ctx.Err()
return
}
}
}
// Break if we did not receive any new rows
if newRowsCount == 0 {
return
}
rowCount += newRowsCount
}
}()
return rowChan, errChan
}
func exportRawDataForSingleQuery(querier querier.Querier, ctx context.Context, orgID valuer.UUID, rangeRequest *qbtypes.QueryRangeRequest, rowChan chan *qbtypes.RawRow, errChan chan error, doneChan chan any, queryIndex int) {
queries := rangeRequest.CompositeQuery.Queries
rowCountLimit := queries[queryIndex].GetLimit()
rowCount := 0
for rowCount < rowCountLimit {
chunkSize := min(ChunkSize, rowCountLimit-rowCount)
queries[queryIndex].SetLimit(chunkSize)
queries[queryIndex].SetOffset(rowCount)
response, err := querier.QueryRange(ctx, orgID, rangeRequest)
if err != nil {
errChan <- err
return
}
newRowsCount := 0
for _, result := range response.Data.Results {
resultData, ok := result.(*qbtypes.RawData)
if !ok {
errChan <- errors.NewInternalf(errors.CodeInternal, "expected RawData, got %T", result)
return
}
newRowsCount += len(resultData.Rows)
for _, row := range resultData.Rows {
select {
case rowChan <- row:
case <-doneChan:
return
case <-ctx.Done():
errChan <- ctx.Err()
return
}
}
}
rowCount += newRowsCount
// Stop if we received fewer rows than requested — no more data available
if newRowsCount < chunkSize {
return
}
}
}

View File

@@ -10,13 +10,11 @@ import (
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrystore"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/bytedance/sonic"
)
type builderQuery[T any] struct {
@@ -262,40 +260,6 @@ func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string,
return nil, err
}
// merge body_json and promoted into body
if q.spec.Signal == telemetrytypes.SignalLogs {
switch typedPayload := payload.(type) {
case *qbtypes.RawData:
for _, rr := range typedPayload.Rows {
seeder := func() error {
body, ok := rr.Data[telemetrylogs.LogsV2BodyJSONColumn].(map[string]any)
if !ok {
return nil
}
promoted, ok := rr.Data[telemetrylogs.LogsV2BodyPromotedColumn].(map[string]any)
if !ok {
return nil
}
seed(promoted, body)
str, err := sonic.MarshalString(body)
if err != nil {
return errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to marshal body")
}
rr.Data["body"] = str
return nil
}
err := seeder()
if err != nil {
return nil, err
}
delete(rr.Data, telemetrylogs.LogsV2BodyJSONColumn)
delete(rr.Data, telemetrylogs.LogsV2BodyPromotedColumn)
}
payload = typedPayload
}
}
return &qbtypes.Result{
Type: q.kind,
Value: payload,
@@ -423,18 +387,3 @@ func decodeCursor(cur string) (int64, error) {
}
return strconv.ParseInt(string(b), 10, 64)
}
func seed(promoted map[string]any, body map[string]any) {
for key, fromValue := range promoted {
if toValue, ok := body[key]; !ok {
body[key] = fromValue
} else {
if fromValue, ok := fromValue.(map[string]any); ok {
if toValue, ok := toValue.(map[string]any); ok {
seed(fromValue, toValue)
body[key] = toValue
}
}
}
}
}

View File

@@ -14,7 +14,6 @@ import (
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/bytedance/sonic"
)
var (
@@ -394,17 +393,11 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
// de-reference the typed pointer to any
val := reflect.ValueOf(cellPtr).Elem().Interface()
// Post-process JSON columns: normalize into structured values
// Post-process JSON columns: normalize into String value
if strings.HasPrefix(strings.ToUpper(colTypes[i].DatabaseTypeName()), "JSON") {
switch x := val.(type) {
case []byte:
if len(x) > 0 {
var v any
if err := sonic.Unmarshal(x, &v); err == nil {
val = v
}
}
val = string(x)
default:
// already a structured type (map[string]any, []any, etc.)
}

View File

@@ -574,6 +574,9 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *middleware.AuthZ) {
aH.LicensingAPI.Activate(rw, req)
})).Methods(http.MethodGet)
// Export
router.HandleFunc("/api/v1/export_raw_data", am.ViewAccess(aH.Signoz.Handlers.RawDataExport.ExportRawData)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/span_percentile", am.ViewAccess(aH.Signoz.Handlers.SpanPercentile.GetSpanPercentileDetails)).Methods(http.MethodPost)
// Query Filter Analyzer api used to extract metric names and grouping columns from a query

View File

@@ -219,7 +219,6 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
// we don't have a toBoolOrNull in ClickHouse, so we need to convert the bool to a string
value = fmt.Sprintf("%t", v)
}
case telemetrytypes.FieldDataTypeInt64,
telemetrytypes.FieldDataTypeArrayInt64,
telemetrytypes.FieldDataTypeNumber,

View File

@@ -313,37 +313,30 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
return ""
}
child := ctx.GetChild(0)
var searchText string
if keyCtx, ok := child.(*grammar.KeyContext); ok {
// create a full text search condition on the body field
keyText := keyCtx.GetText()
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(keyText), v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
return cond
searchText = keyCtx.GetText()
} else if valCtx, ok := child.(*grammar.ValueContext); ok {
var text string
if valCtx.QUOTED_TEXT() != nil {
text = trimQuotes(valCtx.QUOTED_TEXT().GetText())
searchText = trimQuotes(valCtx.QUOTED_TEXT().GetText())
} else if valCtx.NUMBER() != nil {
text = valCtx.NUMBER().GetText()
searchText = valCtx.NUMBER().GetText()
} else if valCtx.BOOL() != nil {
text = valCtx.BOOL().GetText()
searchText = valCtx.BOOL().GetText()
} else if valCtx.KEY() != nil {
text = valCtx.KEY().GetText()
searchText = valCtx.KEY().GetText()
} else {
v.errors = append(v.errors, fmt.Sprintf("unsupported value type: %s", valCtx.GetText()))
return ""
}
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
return cond
}
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText), v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
return cond
}
return "" // Should not happen with valid input
@@ -383,6 +376,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
for _, key := range keys {
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, nil, v.builder, v.startNs, v.endNs)
if err != nil {
v.errors = append(v.errors, fmt.Sprintf("failed to build condition: %s", err.Error()))
return ""
}
conds = append(conds, condition)
@@ -648,7 +642,6 @@ func (v *filterExpressionVisitor) VisitValueList(ctx *grammar.ValueListContext)
// VisitFullText handles standalone quoted strings for full-text search
func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) any {
if v.skipFullTextFilter {
return ""
}
@@ -670,6 +663,7 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
return ""
}
return cond
}

View File

@@ -22,7 +22,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/preference"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/session"
"github.com/SigNoz/signoz/pkg/modules/user"
@@ -59,7 +58,6 @@ func NewOpenAPI(ctx context.Context, instrumentation instrumentation.Instrumenta
struct{ gateway.Handler }{},
struct{ fields.Handler }{},
struct{ authz.Handler }{},
struct{ rawdataexport.Handler }{},
struct{ zeus.Handler }{},
struct{ querier.Handler }{},
struct{ serviceaccount.Handler }{},

View File

@@ -256,7 +256,6 @@ func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.Au
handlers.GatewayHandler,
handlers.Fields,
handlers.AuthzHandler,
handlers.RawDataExport,
handlers.ZeusHandler,
handlers.QuerierHandler,
handlers.ServiceAccountHandler,

View File

@@ -3,14 +3,12 @@ package telemetrylogs
import (
"context"
"fmt"
"slices"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"golang.org/x/exp/maps"
"github.com/huandu/go-sqlbuilder"
)
@@ -35,7 +33,7 @@ func (c *conditionBuilder) conditionFor(
return "", err
}
if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled {
if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled && key.Name != messageSubField {
valueType, value := InferDataType(value, operator, key)
cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
if err != nil {
@@ -54,14 +52,14 @@ func (c *conditionBuilder) conditionFor(
}
// Check if this is a body JSON search - either by FieldContext
if key.FieldContext == telemetrytypes.FieldContextBody {
if key.FieldContext == telemetrytypes.FieldContextBody && !querybuilder.BodyJSONQueryEnabled {
tblFieldName, value = GetBodyJSONKey(ctx, key, operator, value)
}
tblFieldName, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, tblFieldName, operator)
// make use of case insensitive index for body
if tblFieldName == "body" {
if tblFieldName == "body" || tblFieldName == messageSubColumn {
switch operator {
case qbtypes.FilterOperatorLike:
return sb.ILike(tblFieldName, value), nil
@@ -108,7 +106,6 @@ func (c *conditionBuilder) conditionFor(
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
case qbtypes.FilterOperatorNotContains:
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
case qbtypes.FilterOperatorRegexp:
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
@@ -178,9 +175,8 @@ func (c *conditionBuilder) conditionFor(
case schema.ColumnTypeEnumJSON:
if operator == qbtypes.FilterOperatorExists {
return sb.IsNotNull(tblFieldName), nil
} else {
return sb.IsNull(tblFieldName), nil
}
return sb.IsNull(tblFieldName), nil
case schema.ColumnTypeEnumLowCardinality:
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
case schema.ColumnTypeEnumString:
@@ -247,19 +243,30 @@ func (c *conditionBuilder) ConditionFor(
return "", err
}
if !(key.FieldContext == telemetrytypes.FieldContextBody && querybuilder.BodyJSONQueryEnabled) && operator.AddDefaultExistsFilter() {
// skip adding exists filter for intrinsic fields
// with an exception for body json search
field, _ := c.fm.FieldFor(ctx, key)
if slices.Contains(maps.Keys(IntrinsicFields), field) && key.FieldContext != telemetrytypes.FieldContextBody {
// Skip adding exists filter for intrinsic fields i.e. Table level log context fields
buildExistCondition := operator.AddDefaultExistsFilter()
switch key.FieldContext {
case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextScope:
// pass; No need to build exist condition for top level columns
// immidiately return
return condition, nil
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextAttribute:
// build exist condition for resource and attribute fields based on filter operator
case telemetrytypes.FieldContextBody:
// Querying JSON fields already account for Nullability of fields
// so additional exists checks are not needed
if querybuilder.BodyJSONQueryEnabled {
return condition, nil
}
}
if buildExistCondition {
existsCondition, err := c.conditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb)
if err != nil {
return "", err
}
return sb.And(condition, existsCondition), nil
}
return condition, nil
}

View File

@@ -127,7 +127,8 @@ func TestConditionFor(t *testing.T) {
{
name: "Contains operator - body",
key: telemetrytypes.TelemetryFieldKey{
Name: "body",
Name: "body",
FieldContext: telemetrytypes.FieldContextLog,
},
operator: qbtypes.FilterOperatorContains,
value: 521509198310,

View File

@@ -1,7 +1,10 @@
package telemetrylogs
import (
"fmt"
"github.com/SigNoz/signoz-otel-collector/constants"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
@@ -17,7 +20,7 @@ const (
LogsV2TimestampColumn = "timestamp"
LogsV2ObservedTimestampColumn = "observed_timestamp"
LogsV2BodyColumn = "body"
LogsV2BodyJSONColumn = constants.BodyV2Column
LogsV2BodyV2Column = constants.BodyV2Column
LogsV2BodyPromotedColumn = constants.BodyPromotedColumn
LogsV2TraceIDColumn = "trace_id"
LogsV2SpanIDColumn = "span_id"
@@ -34,8 +37,14 @@ const (
LogsV2ResourcesStringColumn = "resources_string"
LogsV2ScopeStringColumn = "scope_string"
BodyJSONColumnPrefix = constants.BodyV2ColumnPrefix
BodyV2ColumnPrefix = constants.BodyV2ColumnPrefix
BodyPromotedColumnPrefix = constants.BodyPromotedColumnPrefix
// messageSubColumn is the ClickHouse sub-column that body searches map to
// when BodyJSONQueryEnabled is true.
messageSubField = "message"
messageSubColumn = "body_v2.message"
bodySearchDefaultWarning = "body searches default to `body.message:string`. Use `body.<key>` to search a different field inside body"
)
var (
@@ -118,3 +127,11 @@ var (
},
}
)
func bodyAliasExpression() string {
if !querybuilder.BodyJSONQueryEnabled {
return LogsV2BodyColumn
}
return fmt.Sprintf("%s as body", LogsV2BodyV2Column)
}

View File

@@ -30,7 +30,8 @@ var (
"severity_text": {Name: "severity_text", Type: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}},
"severity_number": {Name: "severity_number", Type: schema.ColumnTypeUInt8},
"body": {Name: "body", Type: schema.ColumnTypeString},
LogsV2BodyJSONColumn: {Name: LogsV2BodyJSONColumn, Type: schema.JSONColumnType{
messageSubColumn: {Name: messageSubColumn, Type: schema.ColumnTypeString},
LogsV2BodyV2Column: {Name: LogsV2BodyV2Column, Type: schema.JSONColumnType{
MaxDynamicTypes: utils.ToPointer(uint(32)),
MaxDynamicPaths: utils.ToPointer(uint(0)),
}},
@@ -88,21 +89,26 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry
return logsV2Columns["attributes_bool"], nil
}
case telemetrytypes.FieldContextBody:
// Body context is for JSON body fields
// Use body_json if feature flag is enabled
// Body context is for JSON body fields. Use body_v2 if feature flag is enabled.
if querybuilder.BodyJSONQueryEnabled {
return logsV2Columns[LogsV2BodyJSONColumn], nil
if key.Name == messageSubField {
return logsV2Columns[messageSubColumn], nil
}
return logsV2Columns[LogsV2BodyV2Column], nil
}
// Fall back to legacy body column
return logsV2Columns["body"], nil
case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextUnspecified:
if key.Name == LogsV2BodyColumn && querybuilder.BodyJSONQueryEnabled {
return logsV2Columns[messageSubColumn], nil
}
col, ok := logsV2Columns[key.Name]
if !ok {
// check if the key has body JSON search
if strings.HasPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix) {
// Use body_json if feature flag is enabled and we have a body condition builder
// Use body_v2 if feature flag is enabled and we have a body condition builder
if querybuilder.BodyJSONQueryEnabled {
return logsV2Columns[LogsV2BodyJSONColumn], nil
return logsV2Columns[LogsV2BodyV2Column], nil
}
// Fall back to legacy body column
return logsV2Columns["body"], nil
@@ -138,6 +144,10 @@ func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.Telemetr
}
return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
case telemetrytypes.FieldContextBody:
if key.Name == messageSubField {
return messageSubColumn, nil
}
if key.JSONDataType == nil {
return "", qbtypes.ErrColumnNotFound
}
@@ -246,34 +256,37 @@ func (m *fieldMapper) buildFieldForJSON(key *telemetrytypes.TelemetryFieldKey) (
node := plan[0]
expr := fmt.Sprintf("dynamicElement(%s, '%s')", node.FieldPath(), node.TerminalConfig.ElemType.StringValue())
if key.Materialized {
if len(plan) < 2 {
return "", errors.Newf(errors.TypeUnexpected, CodePromotedPlanMissing,
"plan length is less than 2 for promoted path: %s", key.Name)
}
// TODO(Piyush): Promoted path logic commented out. Materialized now means type hint
// promotion will be extracted from key field evolution
// (direct sub-column access), not a promoted body_promoted.* column.
// if key.Materialized {
// if len(plan) < 2 {
// return "", errors.Newf(errors.TypeUnexpected, CodePromotedPlanMissing,
// "plan length is less than 2 for promoted path: %s", key.Name)
// }
node := plan[1]
promotedExpr := fmt.Sprintf(
"dynamicElement(%s, '%s')",
node.FieldPath(),
node.TerminalConfig.ElemType.StringValue(),
)
// node := plan[1]
// promotedExpr := fmt.Sprintf(
// "dynamicElement(%s, '%s')",
// node.FieldPath(),
// node.TerminalConfig.ElemType.StringValue(),
// )
// dynamicElement returns NULL for scalar types or an empty array for array types.
if node.TerminalConfig.ElemType.IsArray {
expr = fmt.Sprintf(
"if(length(%s) > 0, %s, %s)",
promotedExpr,
promotedExpr,
expr,
)
} else {
// promoted column first then body_json column
// TODO(Piyush): Change this in future for better performance
expr = fmt.Sprintf("coalesce(%s, %s)", promotedExpr, expr)
}
// // dynamicElement returns NULL for scalar types or an empty array for array types.
// if node.TerminalConfig.ElemType.IsArray {
// expr = fmt.Sprintf(
// "if(length(%s) > 0, %s, %s)",
// promotedExpr,
// promotedExpr,
// expr,
// )
// } else {
// // promoted column first then body_json column
// // TODO(Piyush): Change this in future for better performance
// expr = fmt.Sprintf("coalesce(%s, %s)", promotedExpr, expr)
// }
}
// }
return expr, nil
}

View File

@@ -30,7 +30,7 @@ func NewJSONConditionBuilder(key *telemetrytypes.TelemetryFieldKey, valueType te
return &jsonConditionBuilder{key: key, valueType: telemetrytypes.MappingFieldDataTypeToJSONDataType[valueType]}
}
// BuildCondition builds the full WHERE condition for body_json JSON paths
// BuildCondition builds the full WHERE condition for body_v2 JSON paths
func (c *jsonConditionBuilder) buildJSONCondition(operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
conditions := []string{}
for _, node := range c.key.JSONPlan {
@@ -40,6 +40,7 @@ func (c *jsonConditionBuilder) buildJSONCondition(operator qbtypes.FilterOperato
}
conditions = append(conditions, condition)
}
return sb.Or(conditions...), nil
}
@@ -288,9 +289,9 @@ func (c *jsonConditionBuilder) applyOperator(sb *sqlbuilder.SelectBuilder, field
}
return sb.NotIn(fieldExpr, values...), nil
case qbtypes.FilterOperatorExists:
return fmt.Sprintf("%s IS NOT NULL", fieldExpr), nil
return sb.IsNotNull(fieldExpr), nil
case qbtypes.FilterOperatorNotExists:
return fmt.Sprintf("%s IS NULL", fieldExpr), nil
return sb.IsNull(fieldExpr), nil
// between and not between
case qbtypes.FilterOperatorBetween, qbtypes.FilterOperatorNotBetween:
values, ok := value.([]any)

File diff suppressed because one or more lines are too long

View File

@@ -65,7 +65,7 @@ func (b *logQueryStatementBuilder) Build(
start = querybuilder.ToNanoSecs(start)
end = querybuilder.ToNanoSecs(end)
keySelectors := getKeySelectors(query)
keySelectors, warnings := getKeySelectors(query)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
@@ -76,20 +76,29 @@ func (b *logQueryStatementBuilder) Build(
// Create SQL builder
q := sqlbuilder.NewSelectBuilder()
var stmt *qbtypes.Statement
switch requestType {
case qbtypes.RequestTypeRaw, qbtypes.RequestTypeRawStream:
return b.buildListQuery(ctx, q, query, start, end, keys, variables)
stmt, err = b.buildListQuery(ctx, q, query, start, end, keys, variables)
case qbtypes.RequestTypeTimeSeries:
return b.buildTimeSeriesQuery(ctx, q, query, start, end, keys, variables)
stmt, err = b.buildTimeSeriesQuery(ctx, q, query, start, end, keys, variables)
case qbtypes.RequestTypeScalar:
return b.buildScalarQuery(ctx, q, query, start, end, keys, false, variables)
stmt, err = b.buildScalarQuery(ctx, q, query, start, end, keys, false, variables)
default:
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type: %s", requestType)
}
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type: %s", requestType)
if err != nil {
return nil, err
}
stmt.Warnings = append(stmt.Warnings, warnings...)
return stmt, nil
}
func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) []*telemetrytypes.FieldKeySelector {
func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) ([]*telemetrytypes.FieldKeySelector, []string) {
var keySelectors []*telemetrytypes.FieldKeySelector
var warnings []string
for idx := range query.Aggregations {
aggExpr := query.Aggregations[idx]
@@ -136,7 +145,19 @@ func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) []
keySelectors[idx].SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact
}
return keySelectors
// When the new JSON body experience is enabled, warn the user if they use the bare
// "body" key in the filter — queries on plain "body" default to body.message:string.
// TODO(Piyush): Setup better for coming FTS support.
if querybuilder.BodyJSONQueryEnabled {
for _, sel := range keySelectors {
if sel.Name == LogsV2BodyColumn {
warnings = append(warnings, bodySearchDefaultWarning)
break
}
}
}
return keySelectors, warnings
}
func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[string][]*telemetrytypes.TelemetryFieldKey, query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation], requestType qbtypes.RequestType) qbtypes.QueryBuilderQuery[qbtypes.LogAggregation] {
@@ -203,7 +224,6 @@ func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[stri
}
func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
// First check if it matches with any intrinsic fields
var intrinsicOrCalculatedField telemetrytypes.TelemetryFieldKey
if _, ok := IntrinsicFields[key.Name]; ok {
@@ -212,7 +232,6 @@ func (b *logQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldK
}
return querybuilder.AdjustKey(key, keys, nil)
}
// buildListQuery builds a query for list panel type
@@ -249,11 +268,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
sb.SelectMore(LogsV2SeverityNumberColumn)
sb.SelectMore(LogsV2ScopeNameColumn)
sb.SelectMore(LogsV2ScopeVersionColumn)
sb.SelectMore(LogsV2BodyColumn)
if querybuilder.BodyJSONQueryEnabled {
sb.SelectMore(LogsV2BodyJSONColumn)
sb.SelectMore(LogsV2BodyPromotedColumn)
}
sb.SelectMore(bodyAliasExpression())
sb.SelectMore(LogsV2AttributesStringColumn)
sb.SelectMore(LogsV2AttributesNumberColumn)
sb.SelectMore(LogsV2AttributesBoolColumn)

View File

@@ -5,6 +5,7 @@ import (
"testing"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter"
@@ -886,3 +887,246 @@ func TestAdjustKey(t *testing.T) {
})
}
}
func TestStmtBuilderBodyField(t *testing.T) {
cases := []struct {
name string
requestType qbtypes.RequestType
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
enableBodyJSONQuery bool
expected qbtypes.Statement
expectedErr error
}{
{
name: "body_exists",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body Exists"},
Limit: 10,
},
enableBodyJSONQuery: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body_v2.message <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
expectedErr: nil,
},
{
name: "body_exists_disabled",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body Exists"},
Limit: 10,
},
enableBodyJSONQuery: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body <> ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
{
name: "body_empty",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body == ''"},
Limit: 10,
},
enableBodyJSONQuery: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body_v2.message = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
expectedErr: nil,
},
{
name: "body_empty_disabled",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body == ''"},
Limit: 10,
},
enableBodyJSONQuery: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND body = ? AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
{
name: "body_contains",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"},
Limit: 10,
},
enableBodyJSONQuery: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body_v2.message) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
Warnings: []string{bodySearchDefaultWarning},
},
expectedErr: nil,
},
{
name: "body_contains_disabled",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "body CONTAINS 'error'"},
Limit: 10,
},
enableBodyJSONQuery: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND LOWER(body) LIKE LOWER(?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
}
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
enable, disable := jsonQueryTestUtil(t)
defer disable()
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if c.enableBodyJSONQuery {
enable()
} else {
disable()
}
// build the key map after enabling/disabling body JSON query
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
for _, field := range IntrinsicFields {
f := field
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
}
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
resourceFilterStmtBuilder := resourceFilterStmtBuilder()
statementBuilder := NewLogQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
resourceFilterStmtBuilder,
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
)
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
if c.expectedErr != nil {
require.Error(t, err)
require.Contains(t, err.Error(), c.expectedErr.Error())
} else {
if err != nil {
_, _, _, _, _, add := errors.Unwrapb(err)
t.Logf("error additionals: %v", add)
}
require.NoError(t, err)
require.Equal(t, c.expected.Query, q.Query)
require.Equal(t, c.expected.Args, q.Args)
require.Equal(t, c.expected.Warnings, q.Warnings)
}
})
}
}
func TestStmtBuilderBodyFullTextSearch(t *testing.T) {
cases := []struct {
name string
requestType qbtypes.RequestType
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
enableBodyJSONQuery bool
expected qbtypes.Statement
expectedErr error
}{
{
name: "body_contains",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "'error'"},
Limit: 10,
},
enableBodyJSONQuery: true,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body_v2 as body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND match(LOWER(body_v2.message), LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "error", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
{
name: "body_contains_disabled",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Filter: &qbtypes.Filter{Expression: "'error'"},
Limit: 10,
},
enableBodyJSONQuery: false,
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND match(LOWER(body), LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "error", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
},
expectedErr: nil,
},
}
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
enable, disable := jsonQueryTestUtil(t)
defer disable()
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if c.enableBodyJSONQuery {
enable()
} else {
disable()
}
// build the key map after enabling/disabling body JSON query
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
for _, field := range IntrinsicFields {
f := field
mockMetadataStore.KeysMap[field.Name] = append(mockMetadataStore.KeysMap[field.Name], &f)
}
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
resourceFilterStmtBuilder := resourceFilterStmtBuilder()
statementBuilder := NewLogQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
resourceFilterStmtBuilder,
aggExprRewriter,
DefaultFullTextColumn,
GetBodyJSONKey,
)
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
if c.expectedErr != nil {
require.Error(t, err)
require.Contains(t, err.Error(), c.expectedErr.Error())
} else {
if err != nil {
_, _, _, _, _, add := errors.Unwrapb(err)
t.Logf("error additionals: %v", add)
}
require.NoError(t, err)
require.Equal(t, c.expected.Query, q.Query)
require.Equal(t, c.expected.Args, q.Args)
require.Equal(t, c.expected.Warnings, q.Warnings)
}
})
}
}

View File

@@ -27,13 +27,6 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"body": {
{
Name: "body",
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
"http.status_code": {
{
Name: "http.status_code",
@@ -938,6 +931,13 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
Materialized: true,
},
},
"body": {
{
Name: "body",
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
}
for _, keys := range keysMap {
@@ -945,6 +945,7 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
key.Signal = telemetrytypes.SignalLogs
}
}
return keysMap
}

View File

@@ -54,6 +54,7 @@ func (t *telemetryMetaStore) fetchBodyJSONPaths(ctx context.Context,
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "fetchBodyJSONPaths",
})
query, args, limit := buildGetBodyJSONPathsQuery(fieldKeySelectors)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
@@ -184,7 +185,6 @@ func buildGetBodyJSONPathsQuery(fieldKeySelectors []*telemetrytypes.FieldKeySele
limit += fieldKeySelector.Limit
}
sb.Where(sb.Or(orClauses...))
// Group by path to get unique paths with aggregated types
sb.GroupBy("path")
@@ -319,7 +319,7 @@ func (t *telemetryMetaStore) ListJSONValues(ctx context.Context, path string, li
if promoted {
path = telemetrylogs.BodyPromotedColumnPrefix + path
} else {
path = telemetrylogs.BodyJSONColumnPrefix + path
path = telemetrylogs.BodyV2ColumnPrefix + path
}
from := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
@@ -522,7 +522,7 @@ func (t *telemetryMetaStore) GetPromotedPaths(ctx context.Context, paths ...stri
// TODO(Piyush): Remove this function
func CleanPathPrefixes(path string) string {
path = strings.TrimPrefix(path, telemetrytypes.BodyJSONStringSearchPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyV2ColumnPrefix)
path = strings.TrimPrefix(path, telemetrylogs.BodyPromotedColumnPrefix)
return path
}

View File

@@ -102,7 +102,7 @@ func NewTelemetryMetaStore(
jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{
telemetrytypes.SignalLogs: {
telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{
BaseColumn: telemetrylogs.LogsV2BodyJSONColumn,
BaseColumn: telemetrylogs.LogsV2BodyV2Column,
PromotedColumn: telemetrylogs.LogsV2BodyPromotedColumn,
},
},

View File

@@ -1,6 +0,0 @@
package exporttypes
type ExportRawDataFormatQueryParam struct {
// Format specifies the output format: "csv" or "jsonl"
Format string `query:"format,default=csv" default:"csv" enum:"csv,jsonl" description:"The output format for the export."`
}

View File

@@ -393,77 +393,6 @@ func (r *QueryRangeRequest) HasOrderSpecified() bool {
return false
}
// UseDefaultOrderBy applies UseDefaultOrderByForListQuery to every query in the
// composite query when the request type is a list query (raw, raw_stream, trace).
func (r *QueryRangeRequest) UseDefaultOrderBy() {
// Based on the request type, handle default order-bys
switch r.RequestType {
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace:
for idx := range r.CompositeQuery.Queries {
r.CompositeQuery.Queries[idx].UseDefaultOrderByForListQuery()
}
}
}
// UseDefaultOrderByForListQuery applies a default timestamp-descending order
// for list/raw queries when no explicit order is specified. This is intended
// for raw data listing endpoints (e.g. export, list views) where a sensible
// default sort is needed, not for aggregation or timeseries queries.
func (q *QueryEnvelope) UseDefaultOrderByForListQuery() {
if len(q.GetOrder()) > 0 {
return
}
switch q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation],
QueryBuilderTraceOperator:
q.SetOrder(
[]OrderBy{
{
Key: OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "timestamp",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
},
},
Direction: OrderDirectionDesc,
},
},
)
case QueryBuilderQuery[LogAggregation]:
q.SetOrder(
[]OrderBy{
{
Key: OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "timestamp",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
},
},
Direction: OrderDirectionDesc,
},
{
Key: OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "id",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
Direction: OrderDirectionDesc,
},
},
)
}
}
func (r *QueryRangeRequest) FuncsForQuery(name string) []Function {
funcs := []Function{}
for _, query := range r.CompositeQuery.Queries {
@@ -508,16 +437,6 @@ func (r *QueryRangeRequest) IsAnomalyRequest() (*QueryBuilderQuery[MetricAggrega
return &q, hasAnomaly
}
func (r *QueryRangeRequest) TraceOperatorQueryIndex() int {
for idx, query := range r.CompositeQuery.Queries {
switch query.Spec.(type) {
case QueryBuilderTraceOperator:
return idx
}
}
return -1
}
// We do not support fill gaps for these queries. Maybe support in future?
func (r *QueryRangeRequest) SkipFillGaps(name string) bool {
for _, query := range r.CompositeQuery.Queries {

View File

@@ -1,379 +0,0 @@
package querybuildertypesv5
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
// GetExpression returns the expression string.
func (q *QueryEnvelope) GetExpression() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Expression
case QueryBuilderFormula:
return spec.Expression
}
return ""
}
// GetReturnSpansFrom returns the return-spans-from value.
func (q *QueryEnvelope) GetReturnSpansFrom() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.ReturnSpansFrom
}
return ""
}
// GetSignal returns the signal.
func (q *QueryEnvelope) GetSignal() telemetrytypes.Signal {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Signal
case QueryBuilderQuery[LogAggregation]:
return spec.Signal
case QueryBuilderQuery[MetricAggregation]:
return spec.Signal
}
return telemetrytypes.SignalUnspecified
}
// GetSource returns the source.
func (q *QueryEnvelope) GetSource() telemetrytypes.Source {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Source
case QueryBuilderQuery[LogAggregation]:
return spec.Source
case QueryBuilderQuery[MetricAggregation]:
return spec.Source
}
return telemetrytypes.SourceUnspecified
}
// GetQuery returns the raw query string.
func (q *QueryEnvelope) GetQuery() string {
switch spec := q.Spec.(type) {
case PromQuery:
return spec.Query
case ClickHouseQuery:
return spec.Query
}
return ""
}
// GetStats returns the PromQL stats flag.
func (q *QueryEnvelope) GetStats() bool {
switch spec := q.Spec.(type) {
case PromQuery:
return spec.Stats
}
return false
}
// GetLeft returns the left query reference of a join.
func (q *QueryEnvelope) GetLeft() QueryRef {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.Left
}
return QueryRef{}
}
// GetRight returns the right query reference of a join.
func (q *QueryEnvelope) GetRight() QueryRef {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.Right
}
return QueryRef{}
}
// GetJoinType returns the join type.
func (q *QueryEnvelope) GetJoinType() JoinType {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.Type
}
return JoinType{}
}
// GetOn returns the join ON condition.
func (q *QueryEnvelope) GetOn() string {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.On
}
return ""
}
// GetQueryName returns the name of the spec.
func (q *QueryEnvelope) GetQueryName() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Name
case QueryBuilderQuery[TraceAggregation]:
return spec.Name
case QueryBuilderQuery[LogAggregation]:
return spec.Name
case QueryBuilderQuery[MetricAggregation]:
return spec.Name
case QueryBuilderFormula:
return spec.Name
case QueryBuilderJoin:
return spec.Name
case PromQuery:
return spec.Name
case ClickHouseQuery:
return spec.Name
}
return ""
}
// IsDisabled returns whether the spec is disabled.
func (q *QueryEnvelope) IsDisabled() bool {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Disabled
case QueryBuilderQuery[TraceAggregation]:
return spec.Disabled
case QueryBuilderQuery[LogAggregation]:
return spec.Disabled
case QueryBuilderQuery[MetricAggregation]:
return spec.Disabled
case QueryBuilderFormula:
return spec.Disabled
case QueryBuilderJoin:
return spec.Disabled
case PromQuery:
return spec.Disabled
case ClickHouseQuery:
return spec.Disabled
}
return false
}
// GetLimit returns the row limit.
func (q *QueryEnvelope) GetLimit() int {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Limit
case QueryBuilderQuery[TraceAggregation]:
return spec.Limit
case QueryBuilderQuery[LogAggregation]:
return spec.Limit
case QueryBuilderQuery[MetricAggregation]:
return spec.Limit
case QueryBuilderFormula:
return spec.Limit
case QueryBuilderJoin:
return spec.Limit
}
return 0
}
// GetOffset returns the row offset.
func (q *QueryEnvelope) GetOffset() int {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Offset
case QueryBuilderQuery[TraceAggregation]:
return spec.Offset
case QueryBuilderQuery[LogAggregation]:
return spec.Offset
case QueryBuilderQuery[MetricAggregation]:
return spec.Offset
}
return 0
}
// GetType returns the QueryType of the envelope.
func (q *QueryEnvelope) GetType() QueryType {
return q.Type
}
// GetOrder returns the order-by clauses.
func (q *QueryEnvelope) GetOrder() []OrderBy {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Order
case QueryBuilderQuery[TraceAggregation]:
return spec.Order
case QueryBuilderQuery[LogAggregation]:
return spec.Order
case QueryBuilderQuery[MetricAggregation]:
return spec.Order
case QueryBuilderFormula:
return spec.Order
case QueryBuilderJoin:
return spec.Order
}
return nil
}
// GetGroupBy returns the group-by keys.
func (q *QueryEnvelope) GetGroupBy() []GroupByKey {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.GroupBy
case QueryBuilderQuery[TraceAggregation]:
return spec.GroupBy
case QueryBuilderQuery[LogAggregation]:
return spec.GroupBy
case QueryBuilderQuery[MetricAggregation]:
return spec.GroupBy
case QueryBuilderJoin:
return spec.GroupBy
}
return nil
}
// GetFilter returns the filter.
func (q *QueryEnvelope) GetFilter() *Filter {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Filter
case QueryBuilderQuery[TraceAggregation]:
return spec.Filter
case QueryBuilderQuery[LogAggregation]:
return spec.Filter
case QueryBuilderQuery[MetricAggregation]:
return spec.Filter
case QueryBuilderJoin:
return spec.Filter
}
return nil
}
// GetHaving returns the having clause.
func (q *QueryEnvelope) GetHaving() *Having {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Having
case QueryBuilderQuery[TraceAggregation]:
return spec.Having
case QueryBuilderQuery[LogAggregation]:
return spec.Having
case QueryBuilderQuery[MetricAggregation]:
return spec.Having
case QueryBuilderFormula:
return spec.Having
case QueryBuilderJoin:
return spec.Having
}
return nil
}
// GetFunctions returns the post-processing functions.
func (q *QueryEnvelope) GetFunctions() []Function {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Functions
case QueryBuilderQuery[TraceAggregation]:
return spec.Functions
case QueryBuilderQuery[LogAggregation]:
return spec.Functions
case QueryBuilderQuery[MetricAggregation]:
return spec.Functions
case QueryBuilderFormula:
return spec.Functions
case QueryBuilderJoin:
return spec.Functions
}
return nil
}
// GetSelectFields returns the selected fields.
func (q *QueryEnvelope) GetSelectFields() []telemetrytypes.TelemetryFieldKey {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.SelectFields
case QueryBuilderQuery[TraceAggregation]:
return spec.SelectFields
case QueryBuilderQuery[LogAggregation]:
return spec.SelectFields
case QueryBuilderQuery[MetricAggregation]:
return spec.SelectFields
case QueryBuilderJoin:
return spec.SelectFields
}
return nil
}
// GetLegend returns the legend label.
func (q *QueryEnvelope) GetLegend() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Legend
case QueryBuilderQuery[TraceAggregation]:
return spec.Legend
case QueryBuilderQuery[LogAggregation]:
return spec.Legend
case QueryBuilderQuery[MetricAggregation]:
return spec.Legend
case QueryBuilderFormula:
return spec.Legend
case PromQuery:
return spec.Legend
case ClickHouseQuery:
return spec.Legend
}
return ""
}
// GetCursor returns the pagination cursor.
func (q *QueryEnvelope) GetCursor() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Cursor
case QueryBuilderQuery[TraceAggregation]:
return spec.Cursor
case QueryBuilderQuery[LogAggregation]:
return spec.Cursor
case QueryBuilderQuery[MetricAggregation]:
return spec.Cursor
}
return ""
}
// GetStepInterval returns the step interval.
func (q *QueryEnvelope) GetStepInterval() Step {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.StepInterval
case QueryBuilderQuery[TraceAggregation]:
return spec.StepInterval
case QueryBuilderQuery[LogAggregation]:
return spec.StepInterval
case QueryBuilderQuery[MetricAggregation]:
return spec.StepInterval
case PromQuery:
return spec.Step
}
return Step{}
}
// GetSecondaryAggregations returns the secondary aggregations.
func (q *QueryEnvelope) GetSecondaryAggregations() []SecondaryAggregation {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.SecondaryAggregations
case QueryBuilderQuery[LogAggregation]:
return spec.SecondaryAggregations
case QueryBuilderQuery[MetricAggregation]:
return spec.SecondaryAggregations
case QueryBuilderJoin:
return spec.SecondaryAggregations
}
return nil
}
// GetLimitBy returns the limit-by configuration.
func (q *QueryEnvelope) GetLimitBy() *LimitBy {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.LimitBy
case QueryBuilderQuery[LogAggregation]:
return spec.LimitBy
case QueryBuilderQuery[MetricAggregation]:
return spec.LimitBy
}
return nil
}

View File

@@ -1,452 +0,0 @@
package querybuildertypesv5
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
// SetExpression sets the expression string of the spec, if applicable.
func (q *QueryEnvelope) SetExpression(expression string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Expression = expression
q.Spec = spec
case QueryBuilderFormula:
spec.Expression = expression
q.Spec = spec
}
}
// SetReturnSpansFrom sets the return-spans-from value, if applicable.
func (q *QueryEnvelope) SetReturnSpansFrom(returnSpansFrom string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.ReturnSpansFrom = returnSpansFrom
q.Spec = spec
}
}
// SetSignal sets the signal of the spec, if applicable.
func (q *QueryEnvelope) SetSignal(signal telemetrytypes.Signal) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.Signal = signal
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Signal = signal
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Signal = signal
q.Spec = spec
}
}
// SetSource sets the source of the spec, if applicable.
func (q *QueryEnvelope) SetSource(source telemetrytypes.Source) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.Source = source
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Source = source
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Source = source
q.Spec = spec
}
}
// SetQuery sets the raw query string of the spec, if applicable.
func (q *QueryEnvelope) SetQuery(query string) {
switch spec := q.Spec.(type) {
case PromQuery:
spec.Query = query
q.Spec = spec
case ClickHouseQuery:
spec.Query = query
q.Spec = spec
}
}
// SetStats sets the PromQL stats flag, if applicable.
func (q *QueryEnvelope) SetStats(stats bool) {
switch spec := q.Spec.(type) {
case PromQuery:
spec.Stats = stats
q.Spec = spec
}
}
// SetLeft sets the left query reference of a join, if applicable.
func (q *QueryEnvelope) SetLeft(left QueryRef) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.Left = left
q.Spec = spec
}
}
// SetRight sets the right query reference of a join, if applicable.
func (q *QueryEnvelope) SetRight(right QueryRef) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.Right = right
q.Spec = spec
}
}
// SetJoinType sets the join type, if applicable.
func (q *QueryEnvelope) SetJoinType(joinType JoinType) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.Type = joinType
q.Spec = spec
}
}
// SetOn sets the join ON condition, if applicable.
func (q *QueryEnvelope) SetOn(on string) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.On = on
q.Spec = spec
}
}
// SetQueryName sets the name of the spec, if applicable.
func (q *QueryEnvelope) SetQueryName(name string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Name = name
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Name = name
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Name = name
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Name = name
q.Spec = spec
case QueryBuilderFormula:
spec.Name = name
q.Spec = spec
case QueryBuilderJoin:
spec.Name = name
q.Spec = spec
case PromQuery:
spec.Name = name
q.Spec = spec
case ClickHouseQuery:
spec.Name = name
q.Spec = spec
}
}
// SetDisabled sets the disabled flag of the spec, if applicable.
func (q *QueryEnvelope) SetDisabled(disabled bool) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderFormula:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderJoin:
spec.Disabled = disabled
q.Spec = spec
case PromQuery:
spec.Disabled = disabled
q.Spec = spec
case ClickHouseQuery:
spec.Disabled = disabled
q.Spec = spec
}
}
// SetLimit sets the row limit of the spec, if applicable.
func (q *QueryEnvelope) SetLimit(limit int) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Limit = limit
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Limit = limit
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Limit = limit
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Limit = limit
q.Spec = spec
case QueryBuilderFormula:
spec.Limit = limit
q.Spec = spec
case QueryBuilderJoin:
spec.Limit = limit
q.Spec = spec
}
}
// SetOffset sets the row offset of the spec, if applicable.
func (q *QueryEnvelope) SetOffset(offset int) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Offset = offset
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Offset = offset
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Offset = offset
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Offset = offset
q.Spec = spec
}
}
// SetType sets the QueryType of the envelope.
func (q *QueryEnvelope) SetType(t QueryType) {
q.Type = t
}
// SetOrder sets the order-by clauses of the spec, if applicable.
func (q *QueryEnvelope) SetOrder(order []OrderBy) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Order = order
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Order = order
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Order = order
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Order = order
q.Spec = spec
case QueryBuilderFormula:
spec.Order = order
q.Spec = spec
case QueryBuilderJoin:
spec.Order = order
q.Spec = spec
}
}
// SetGroupBy sets the group-by keys of the spec, if applicable.
func (q *QueryEnvelope) SetGroupBy(groupBy []GroupByKey) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderJoin:
spec.GroupBy = groupBy
q.Spec = spec
}
}
// SetFilter sets the filter of the spec, if applicable.
func (q *QueryEnvelope) SetFilter(filter *Filter) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Filter = filter
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Filter = filter
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Filter = filter
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Filter = filter
q.Spec = spec
case QueryBuilderJoin:
spec.Filter = filter
q.Spec = spec
}
}
// SetHaving sets the having clause of the spec, if applicable.
func (q *QueryEnvelope) SetHaving(having *Having) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Having = having
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Having = having
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Having = having
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Having = having
q.Spec = spec
case QueryBuilderFormula:
spec.Having = having
q.Spec = spec
case QueryBuilderJoin:
spec.Having = having
q.Spec = spec
}
}
// SetFunctions sets the post-processing functions of the spec, if applicable.
func (q *QueryEnvelope) SetFunctions(functions []Function) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Functions = functions
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Functions = functions
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Functions = functions
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Functions = functions
q.Spec = spec
case QueryBuilderFormula:
spec.Functions = functions
q.Spec = spec
case QueryBuilderJoin:
spec.Functions = functions
q.Spec = spec
}
}
// SetSelectFields sets the selected fields of the spec, if applicable.
func (q *QueryEnvelope) SetSelectFields(fields []telemetrytypes.TelemetryFieldKey) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderJoin:
spec.SelectFields = fields
q.Spec = spec
}
}
// SetLegend sets the legend label of the spec, if applicable.
func (q *QueryEnvelope) SetLegend(legend string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Legend = legend
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Legend = legend
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Legend = legend
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Legend = legend
q.Spec = spec
case QueryBuilderFormula:
spec.Legend = legend
q.Spec = spec
case PromQuery:
spec.Legend = legend
q.Spec = spec
case ClickHouseQuery:
spec.Legend = legend
q.Spec = spec
}
}
// SetCursor sets the pagination cursor of the spec, if applicable.
func (q *QueryEnvelope) SetCursor(cursor string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Cursor = cursor
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Cursor = cursor
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Cursor = cursor
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Cursor = cursor
q.Spec = spec
}
}
// SetStepInterval sets the step interval of the spec, if applicable.
func (q *QueryEnvelope) SetStepInterval(step Step) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.StepInterval = step
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.StepInterval = step
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.StepInterval = step
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.StepInterval = step
q.Spec = spec
case PromQuery:
spec.Step = step
q.Spec = spec
}
}
// SetSecondaryAggregations sets the secondary aggregations of the spec, if applicable.
func (q *QueryEnvelope) SetSecondaryAggregations(secondaryAggregations []SecondaryAggregation) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
case QueryBuilderJoin:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
}
}
// SetLimitBy sets the limit-by configuration of the spec, if applicable.
func (q *QueryEnvelope) SetLimitBy(limitBy *LimitBy) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.LimitBy = limitBy
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.LimitBy = limitBy
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.LimitBy = limitBy
q.Spec = spec
}
}

View File

@@ -10,9 +10,55 @@ import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// queryName returns the name from any query envelope spec type.
func (e QueryEnvelope) queryName() string {
switch spec := e.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Name
case QueryBuilderQuery[LogAggregation]:
return spec.Name
case QueryBuilderQuery[MetricAggregation]:
return spec.Name
case QueryBuilderFormula:
return spec.Name
case QueryBuilderTraceOperator:
return spec.Name
case QueryBuilderJoin:
return spec.Name
case PromQuery:
return spec.Name
case ClickHouseQuery:
return spec.Name
}
return ""
}
// isDisabled returns the disabled status from any query envelope spec type.
func (e QueryEnvelope) isDisabled() bool {
switch spec := e.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Disabled
case QueryBuilderQuery[LogAggregation]:
return spec.Disabled
case QueryBuilderQuery[MetricAggregation]:
return spec.Disabled
case QueryBuilderFormula:
return spec.Disabled
case QueryBuilderTraceOperator:
return spec.Disabled
case QueryBuilderJoin:
return spec.Disabled
case PromQuery:
return spec.Disabled
case ClickHouseQuery:
return spec.Disabled
}
return false
}
// getQueryIdentifier returns a friendly identifier for a query based on its type and name/content
func getQueryIdentifier(envelope QueryEnvelope, index int) string {
name := envelope.GetQueryName()
name := envelope.queryName()
var typeLabel string
switch envelope.Type {
@@ -43,115 +89,50 @@ const (
MaxQueryLimit = 10000
)
// ValidationOption is a functional option for configuring validation behaviour.
type ValidationOption func(*validationConfig)
type validationConfig struct {
skipLimitOffsetValidation bool
skipAggregationValidation bool
skipHavingValidation bool
skipAggregationOrderBy bool
skipSelectFieldValidation bool
skipGroupByValidation bool
}
func applyValidationOptions(opts []ValidationOption) validationConfig {
cfg := validationConfig{}
for _, opt := range opts {
opt(&cfg)
}
return cfg
}
// SkipLimitOffsetValidation returns a ValidationOption that skips the limit and offset range checks.
// Use this when the caller has already validated limits and offsets with different constraints.
func WithSkipLimitOffsetValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipLimitOffsetValidation = true
}
}
// SkipAggregationValidation skips aggregation validation.
// Used for raw/trace request types where aggregations are not required.
func WithSkipAggregationValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipAggregationValidation = true
}
}
// SkipHavingValidation skips having-clause validation.
// Used for raw/trace request types where having clauses do not apply.
func WithSkipHavingValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipHavingValidation = true
}
}
// SkipAggregationOrderBy skips the aggregation-specific order-by key validation.
// Used for raw/trace request types where order-by keys are not restricted to group-by or aggregation keys.
func WithSkipAggregationOrderBy() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipAggregationOrderBy = true
}
}
// SkipSelectFieldValidation skips select-field validation.
// Used for aggregation request types where select fields do not apply.
func WithSkipSelectFieldValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipSelectFieldValidation = true
}
}
// SkipGroupByValidation skips group-by validation.
// Used for raw/trace request types where group-by does not apply.
func WithSkipGroupByValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipGroupByValidation = true
}
}
// Validate performs preliminary validation on QueryBuilderQuery.
func (q *QueryBuilderQuery[T]) Validate(opts ...ValidationOption) error {
cfg := applyValidationOptions(opts)
// Validate performs preliminary validation on QueryBuilderQuery
func (q *QueryBuilderQuery[T]) Validate(requestType RequestType) error {
// Validate signal
if err := q.validateSignal(); err != nil {
return err
}
if err := q.validateAggregations(cfg); err != nil {
if err := q.validateAggregations(requestType); err != nil {
return err
}
if err := q.validateGroupBy(cfg); err != nil {
if err := q.validateGroupBy(requestType); err != nil {
return err
}
if err := q.validateLimitAndPagination(cfg); err != nil {
// Validate limit and pagination
if err := q.validateLimitAndPagination(); err != nil {
return err
}
// Validate functions
if err := q.validateFunctions(); err != nil {
return err
}
// Validate secondary aggregations
if err := q.validateSecondaryAggregations(); err != nil {
return err
}
if err := q.validateOrderBy(cfg); err != nil {
if err := q.validateOrderBy(requestType); err != nil {
return err
}
if err := q.validateSelectFields(cfg); err != nil {
if err := q.validateSelectFields(requestType); err != nil {
return err
}
return nil
}
func (q *QueryBuilderQuery[T]) validateSelectFields(cfg validationConfig) error {
if cfg.skipSelectFieldValidation {
func (q *QueryBuilderQuery[T]) validateSelectFields(requestType RequestType) error {
// selectFields don't apply to aggregation queries, skip validation
if requestType.IsAggregation() {
return nil
}
@@ -167,8 +148,9 @@ func (q *QueryBuilderQuery[T]) validateSelectFields(cfg validationConfig) error
return nil
}
func (q *QueryBuilderQuery[T]) validateGroupBy(cfg validationConfig) error {
if cfg.skipGroupByValidation {
func (q *QueryBuilderQuery[T]) validateGroupBy(requestType RequestType) error {
// groupBy doesn't apply to non-aggregation queries, skip validation
if !requestType.IsAggregation() {
return nil
}
for idx, item := range q.GroupBy {
@@ -201,8 +183,9 @@ func (q *QueryBuilderQuery[T]) validateSignal() error {
}
}
func (q *QueryBuilderQuery[T]) validateAggregations(cfg validationConfig) error {
if cfg.skipAggregationValidation {
func (q *QueryBuilderQuery[T]) validateAggregations(requestType RequestType) error {
// aggregations don't apply to non-aggregation queries, skip validation
if !requestType.IsAggregation() {
return nil
}
@@ -289,11 +272,8 @@ func (q *QueryBuilderQuery[T]) validateAggregations(cfg validationConfig) error
return nil
}
func (q *QueryBuilderQuery[T]) validateLimitAndPagination(cfg validationConfig) error {
if cfg.skipLimitOffsetValidation {
return nil
}
func (q *QueryBuilderQuery[T]) validateLimitAndPagination() error {
// Validate limit
if q.Limit < 0 {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
@@ -356,7 +336,7 @@ func (q *QueryBuilderQuery[T]) validateSecondaryAggregations() error {
return nil
}
func (q *QueryBuilderQuery[T]) validateOrderBy(cfg validationConfig) error {
func (q *QueryBuilderQuery[T]) validateOrderBy(requestType RequestType) error {
for i, order := range q.Order {
// Direction validation is handled by the OrderDirection type
if order.Direction != OrderDirectionAsc && order.Direction != OrderDirectionDesc {
@@ -375,7 +355,8 @@ func (q *QueryBuilderQuery[T]) validateOrderBy(cfg validationConfig) error {
}
}
if !cfg.skipAggregationOrderBy {
// aggregation-specific order key validation only applies to aggregation queries
if requestType.IsAggregation() {
return q.validateOrderByForAggregation()
}
@@ -457,8 +438,8 @@ func (q *QueryBuilderQuery[T]) validateOrderByForAggregation() error {
return nil
}
// Validate validates the entire query range request.
func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
// ValidateQueryRangeRequest validates the entire query range request
func (r *QueryRangeRequest) Validate() error {
// Validate time range
if r.RequestType != RequestTypeRawStream && r.Start >= r.End {
return errors.NewInvalidInputf(
@@ -469,8 +450,8 @@ func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
// Validate request type
switch r.RequestType {
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace, RequestTypeTimeSeries, RequestTypeScalar:
opts = append(opts, GetValidationOptions(r.RequestType)...)
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTimeSeries, RequestTypeScalar, RequestTypeTrace:
// Valid request types
default:
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
@@ -482,7 +463,7 @@ func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
}
// Validate composite query
if err := r.CompositeQuery.Validate(opts...); err != nil {
if err := r.validateCompositeQuery(); err != nil {
return err
}
@@ -497,7 +478,7 @@ func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
// validateAllQueriesNotDisabled validates that at least one query in the composite query is enabled
func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error {
for _, envelope := range r.CompositeQuery.Queries {
if !envelope.IsDisabled() {
if !envelope.isDisabled() {
return nil
}
}
@@ -508,8 +489,12 @@ func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error {
)
}
func (r *QueryRangeRequest) validateCompositeQuery() error {
return r.CompositeQuery.Validate(r.RequestType)
}
// Validate performs validation on CompositeQuery
func (c *CompositeQuery) Validate(opts ...ValidationOption) error {
func (c *CompositeQuery) Validate(requestType RequestType) error {
if len(c.Queries) == 0 {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
@@ -521,14 +506,14 @@ func (c *CompositeQuery) Validate(opts ...ValidationOption) error {
queryNames := make(map[string]bool)
for i, envelope := range c.Queries {
if err := validateQueryEnvelope(envelope, opts...); err != nil {
if err := validateQueryEnvelope(envelope, requestType); err != nil {
queryId := getQueryIdentifier(envelope, i)
return wrapValidationError(err, queryId, "invalid %s: %s")
}
// Check name uniqueness for builder queries
if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery {
name := envelope.GetQueryName()
name := envelope.queryName()
if name != "" {
if queryNames[name] {
return errors.NewInvalidInputf(
@@ -545,16 +530,16 @@ func (c *CompositeQuery) Validate(opts ...ValidationOption) error {
return nil
}
func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) error {
func validateQueryEnvelope(envelope QueryEnvelope, requestType RequestType) error {
switch envelope.Type {
case QueryTypeBuilder, QueryTypeSubQuery:
switch spec := envelope.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Validate(opts...)
return spec.Validate(requestType)
case QueryBuilderQuery[LogAggregation]:
return spec.Validate(opts...)
return spec.Validate(requestType)
case QueryBuilderQuery[MetricAggregation]:
return spec.Validate(opts...)
return spec.Validate(requestType)
default:
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
@@ -640,14 +625,3 @@ func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) err
)
}
}
func GetValidationOptions(requestType RequestType) []ValidationOption {
switch requestType {
case RequestTypeTimeSeries, RequestTypeScalar:
return []ValidationOption{WithSkipSelectFieldValidation()}
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace:
return []ValidationOption{WithSkipAggregationValidation(), WithSkipHavingValidation(), WithSkipAggregationOrderBy(), WithSkipGroupByValidation()}
default:
return []ValidationOption{}
}
}

View File

@@ -743,7 +743,7 @@ func TestValidateQueryEnvelope(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateQueryEnvelope(tt.envelope)
err := validateQueryEnvelope(tt.envelope, tt.requestType)
if tt.wantErr {
if err == nil {
t.Errorf("validateQueryEnvelope() expected error but got none")
@@ -816,7 +816,7 @@ func TestQueryEnvelope_Helpers(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.envelope.GetQueryName()
got := tt.envelope.queryName()
if got != tt.want {
t.Errorf("queryName() = %q, want %q", got, tt.want)
}
@@ -868,7 +868,7 @@ func TestQueryEnvelope_Helpers(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.envelope.IsDisabled()
got := tt.envelope.isDisabled()
if got != tt.want {
t.Errorf("isDisabled() = %v, want %v", got, tt.want)
}
@@ -1107,7 +1107,7 @@ func TestQueryRangeRequest_ValidateOrderByForAggregation(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.query.Validate(GetValidationOptions(RequestTypeTimeSeries)...)
err := tt.query.Validate(RequestTypeTimeSeries)
if tt.wantErr {
if err == nil {
t.Errorf("validateOrderByForAggregation() expected error but got none")
@@ -1161,7 +1161,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}},
},
}
err := query.Validate(GetValidationOptions(RequestTypeRaw)...)
err := query.Validate(RequestTypeRaw)
if err != nil {
t.Errorf("expected no error for groupBy with raw request type, got: %v", err)
}
@@ -1178,7 +1178,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: ""}},
},
}
err := query.Validate(GetValidationOptions(RequestTypeTimeSeries)...)
err := query.Validate(RequestTypeTimeSeries)
if err == nil {
t.Errorf("expected error for empty groupBy key with timeseries request type")
}
@@ -1190,7 +1190,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
Signal: telemetrytypes.SignalLogs,
Having: &Having{Expression: "count() > 10"},
}
err := query.Validate(GetValidationOptions(RequestTypeRaw)...)
err := query.Validate(RequestTypeRaw)
if err != nil {
t.Errorf("expected no error for having with raw request type, got: %v", err)
}
@@ -1202,7 +1202,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
Signal: telemetrytypes.SignalTraces,
Having: &Having{Expression: "count() > 10"},
}
err := query.Validate(GetValidationOptions(RequestTypeTrace)...)
err := query.Validate(RequestTypeTrace)
if err != nil {
t.Errorf("expected no error for having with trace request type, got: %v", err)
}
@@ -1216,7 +1216,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{Expression: "count()"},
},
}
err := query.Validate(GetValidationOptions(RequestTypeRaw)...)
err := query.Validate(RequestTypeRaw)
if err != nil {
t.Errorf("expected no error for aggregations with raw request type, got: %v", err)
}
@@ -1230,7 +1230,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{Expression: "count()"},
},
}
err := query.Validate(GetValidationOptions(RequestTypeRawStream)...)
err := query.Validate(RequestTypeRawStream)
if err != nil {
t.Errorf("expected no error for aggregations with raw_stream request type, got: %v", err)
}
@@ -1248,12 +1248,12 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
},
}
// Should error for raw (selectFields are validated)
err := query.Validate(GetValidationOptions(RequestTypeRaw)...)
err := query.Validate(RequestTypeRaw)
if err == nil {
t.Errorf("expected error for isRoot in selectFields with raw request type")
}
// Should pass for timeseries (selectFields skipped)
err = query.Validate(GetValidationOptions(RequestTypeTimeSeries)...)
err = query.Validate(RequestTypeTimeSeries)
if err != nil {
t.Errorf("expected no error for isRoot in selectFields with timeseries request type, got: %v", err)
}

View File

@@ -40,7 +40,7 @@ type JSONAccessNode struct {
// Node information
Name string
IsTerminal bool
isRoot bool // marked true for only body_json and body_json_promoted
isRoot bool // marked true for only body_v2 and body_promoted
// Precomputed type information (single source of truth)
AvailableTypes []JSONDataType

View File

@@ -1,12 +1,19 @@
package telemetrytypes
import (
"fmt"
"testing"
otelconstants "github.com/SigNoz/signoz-otel-collector/constants"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
)
const (
bodyV2Column = otelconstants.BodyV2Column
bodyPromotedColumn = otelconstants.BodyPromotedColumn
)
// ============================================================================
// Helper Functions for Test Data Creation
// ============================================================================
@@ -109,8 +116,8 @@ func TestNode_Alias(t *testing.T) {
}{
{
name: "Root node returns name as-is",
node: NewRootJSONAccessNode("body_json", 32, 0),
expected: "body_json",
node: NewRootJSONAccessNode(bodyV2Column, 32, 0),
expected: bodyV2Column,
},
{
name: "Node without parent returns backticked name",
@@ -124,9 +131,9 @@ func TestNode_Alias(t *testing.T) {
name: "Node with root parent uses dot separator",
node: &JSONAccessNode{
Name: "age",
Parent: NewRootJSONAccessNode("body_json", 32, 0),
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
},
expected: "`" + "body_json" + ".age`",
expected: "`" + bodyV2Column + ".age`",
},
{
name: "Node with non-root parent uses array separator",
@@ -134,10 +141,10 @@ func TestNode_Alias(t *testing.T) {
Name: "name",
Parent: &JSONAccessNode{
Name: "education",
Parent: NewRootJSONAccessNode("body_json", 32, 0),
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
},
},
expected: "`" + "body_json" + ".education[].name`",
expected: "`" + bodyV2Column + ".education[].name`",
},
{
name: "Nested array path with multiple levels",
@@ -147,11 +154,11 @@ func TestNode_Alias(t *testing.T) {
Name: "awards",
Parent: &JSONAccessNode{
Name: "education",
Parent: NewRootJSONAccessNode("body_json", 32, 0),
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
},
},
},
expected: "`" + "body_json" + ".education[].awards[].type`",
expected: "`" + bodyV2Column + ".education[].awards[].type`",
},
}
@@ -173,18 +180,18 @@ func TestNode_FieldPath(t *testing.T) {
name: "Simple field path from root",
node: &JSONAccessNode{
Name: "user",
Parent: NewRootJSONAccessNode("body_json", 32, 0),
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
},
// FieldPath() always wraps the field name in backticks
expected: "body_json" + ".`user`",
expected: bodyV2Column + ".`user`",
},
{
name: "Field path with backtick-required key",
node: &JSONAccessNode{
Name: "user-name", // requires backtick
Parent: NewRootJSONAccessNode("body_json", 32, 0),
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
},
expected: "body_json" + ".`user-name`",
expected: bodyV2Column + ".`user-name`",
},
{
name: "Nested field path",
@@ -192,11 +199,11 @@ func TestNode_FieldPath(t *testing.T) {
Name: "age",
Parent: &JSONAccessNode{
Name: "user",
Parent: NewRootJSONAccessNode("body_json", 32, 0),
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
},
},
// FieldPath() always wraps the field name in backticks
expected: "`" + "body_json" + ".user`.`age`",
expected: "`" + bodyV2Column + ".user`.`age`",
},
{
name: "Array element field path",
@@ -204,11 +211,11 @@ func TestNode_FieldPath(t *testing.T) {
Name: "name",
Parent: &JSONAccessNode{
Name: "education",
Parent: NewRootJSONAccessNode("body_json", 32, 0),
Parent: NewRootJSONAccessNode(bodyV2Column, 32, 0),
},
},
// FieldPath() always wraps the field name in backticks
expected: "`" + "body_json" + ".education`.`name`",
expected: "`" + bodyV2Column + ".education`.`name`",
},
}
@@ -236,36 +243,36 @@ func TestPlanJSON_BasicStructure(t *testing.T) {
{
name: "Simple path not promoted",
key: makeKey("user.name", String, false),
expectedYAML: `
expectedYAML: fmt.Sprintf(`
- name: user.name
column: body_json
column: %s
availableTypes:
- String
maxDynamicTypes: 16
isTerminal: true
elemType: String
`,
`, bodyV2Column),
},
{
name: "Simple path promoted",
key: makeKey("user.name", String, true),
expectedYAML: `
expectedYAML: fmt.Sprintf(`
- name: user.name
column: body_json
column: %s
availableTypes:
- String
maxDynamicTypes: 16
isTerminal: true
elemType: String
- name: user.name
column: body_json_promoted
column: %s
availableTypes:
- String
maxDynamicTypes: 16
maxDynamicPaths: 256
isTerminal: true
elemType: String
`,
`, bodyV2Column, bodyPromotedColumn),
},
{
name: "Empty path returns error",
@@ -278,8 +285,8 @@ func TestPlanJSON_BasicStructure(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
}, types)
if tt.expectErr {
require.Error(t, err)
@@ -304,9 +311,9 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
{
name: "Single array level - JSON branch only",
path: "education[].name",
expectedYAML: `
expectedYAML: fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -318,14 +325,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
maxDynamicTypes: 8
isTerminal: true
elemType: String
`,
`, bodyV2Column),
},
{
name: "Single array level - both JSON and Dynamic branches",
path: "education[].awards[].type",
expectedYAML: `
expectedYAML: fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -352,14 +359,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
maxDynamicPaths: 256
isTerminal: true
elemType: String
`,
`, bodyV2Column),
},
{
name: "Deeply nested array path",
path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name",
expectedYAML: `
expectedYAML: fmt.Sprintf(`
- name: interests
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -399,14 +406,14 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
- String
isTerminal: true
elemType: String
`,
`, bodyV2Column),
},
{
name: "ArrayAnyIndex replacement [*] to []",
path: "education[*].name",
expectedYAML: `
expectedYAML: fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -418,7 +425,7 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
maxDynamicTypes: 8
isTerminal: true
elemType: String
`,
`, bodyV2Column),
},
}
@@ -426,8 +433,8 @@ func TestPlanJSON_ArrayPaths(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
key := makeKey(tt.path, String, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
}, types)
require.NoError(t, err)
require.NotNil(t, key.JSONPlan)
@@ -445,15 +452,15 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
t.Run("Non-promoted plan", func(t *testing.T) {
key := makeKey(path, String, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
}, types)
require.NoError(t, err)
require.Len(t, key.JSONPlan, 1)
expectedYAML := `
expectedYAML := fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -480,7 +487,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
maxDynamicPaths: 256
isTerminal: true
elemType: String
`
`, bodyV2Column)
got := plansToYAML(t, key.JSONPlan)
require.YAMLEq(t, expectedYAML, got)
})
@@ -488,15 +495,15 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
t.Run("Promoted plan", func(t *testing.T) {
key := makeKey(path, String, true)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
}, types)
require.NoError(t, err)
require.Len(t, key.JSONPlan, 2)
expectedYAML := `
expectedYAML := fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -524,7 +531,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
isTerminal: true
elemType: String
- name: education
column: body_json_promoted
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -554,7 +561,7 @@ func TestPlanJSON_PromotedVsNonPromoted(t *testing.T) {
maxDynamicPaths: 256
isTerminal: true
elemType: String
`
`, bodyV2Column, bodyPromotedColumn)
got := plansToYAML(t, key.JSONPlan)
require.YAMLEq(t, expectedYAML, got)
})
@@ -575,11 +582,11 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
expectErr: true,
},
{
name: "Very deep nesting - validates progression doesn't go negative",
path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name",
expectedYAML: `
name: "Very deep nesting - validates progression doesn't go negative",
path: "interests[].entities[].reviews[].entries[].metadata[].positions[].name",
expectedYAML: fmt.Sprintf(`
- name: interests
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -619,14 +626,14 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
- String
isTerminal: true
elemType: String
`,
`, bodyV2Column),
},
{
name: "Path with mixed scalar and array types",
path: "education[].type",
expectedYAML: `
name: "Path with mixed scalar and array types",
path: "education[].type",
expectedYAML: fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -639,20 +646,20 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
maxDynamicTypes: 8
isTerminal: true
elemType: String
`,
`, bodyV2Column),
},
{
name: "Exists with only array types available",
path: "education",
expectedYAML: `
name: "Exists with only array types available",
path: "education",
expectedYAML: fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
isTerminal: true
elemType: Array(JSON)
`,
`, bodyV2Column),
},
}
@@ -668,8 +675,8 @@ func TestPlanJSON_EdgeCases(t *testing.T) {
}
key := makeKey(tt.path, keyType, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
}, types)
if tt.expectErr {
require.Error(t, err)
@@ -687,15 +694,15 @@ func TestPlanJSON_TreeStructure(t *testing.T) {
path := "education[].awards[].participated[].team[].branch"
key := makeKey(path, String, false)
err := key.SetJSONAccessPlan(JSONColumnMetadata{
BaseColumn: "body_json",
PromotedColumn: "body_json_promoted",
BaseColumn: bodyV2Column,
PromotedColumn: bodyPromotedColumn,
}, types)
require.NoError(t, err)
require.Len(t, key.JSONPlan, 1)
expectedYAML := `
expectedYAML := fmt.Sprintf(`
- name: education
column: body_json
column: %s
availableTypes:
- Array(JSON)
maxDynamicTypes: 16
@@ -780,7 +787,7 @@ func TestPlanJSON_TreeStructure(t *testing.T) {
maxDynamicPaths: 64
isTerminal: true
elemType: String
`
`, bodyV2Column)
got := plansToYAML(t, key.JSONPlan)
require.YAMLEq(t, expectedYAML, got)

View File

@@ -172,7 +172,7 @@ def clickhouse(
(
'version="v0.0.1" && '
'node_os=$(uname -s | tr "[:upper:]" "[:lower:]") && '
"node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && "
'node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && '
"cd /tmp && "
'wget -O histogram-quantile.tar.gz "https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2F${version}/histogram-quantile_${node_os}_${node_arch}.tar.gz" && '
"tar -xzf histogram-quantile.tar.gz && "

View File

@@ -1,6 +1,5 @@
from dataclasses import dataclass
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, List, Optional
import requests
@@ -11,95 +10,6 @@ DEFAULT_TOLERANCE = 1e-9
QUERY_TIMEOUT = 30 # seconds
@dataclass
class TelemetryFieldKey:
name: str
field_data_type: str
field_context: str
def to_dict(self) -> Dict:
return {
"name": self.name,
"fieldDataType": self.field_data_type,
"fieldContext": self.field_context,
}
@dataclass
class OrderBy:
key: TelemetryFieldKey
direction: str = "asc"
def to_dict(self) -> Dict:
return {"key": self.key.to_dict(), "direction": self.direction}
@dataclass
class BuilderQuery:
signal: str
name: str = "A"
limit: Optional[int] = None
filter_expression: Optional[str] = None
select_fields: Optional[List[TelemetryFieldKey]] = None
order: Optional[List[OrderBy]] = None
def to_dict(self) -> Dict:
spec: Dict[str, Any] = {
"signal": self.signal,
"name": self.name,
}
if self.limit is not None:
spec["limit"] = self.limit
if self.filter_expression:
spec["filter"] = {"expression": self.filter_expression}
if self.select_fields:
spec["selectFields"] = [f.to_dict() for f in self.select_fields]
if self.order:
spec["order"] = [o.to_dict() for o in self.order]
return {"type": "builder_query", "spec": spec}
@dataclass
class TraceOperatorQuery:
name: str
expression: str
return_spans_from: str
limit: Optional[int] = None
order: Optional[List[OrderBy]] = None
def to_dict(self) -> Dict:
spec: Dict[str, Any] = {
"name": self.name,
"expression": self.expression,
"returnSpansFrom": self.return_spans_from,
}
if self.limit is not None:
spec["limit"] = self.limit
if self.order:
spec["order"] = [o.to_dict() for o in self.order]
return {"type": "builder_trace_operator", "spec": spec}
@dataclass
class QueryRangeRequest:
start: int # nanoseconds
end: int # nanoseconds
queries: List[Union[BuilderQuery, TraceOperatorQuery]]
request_type: Optional[str] = "raw"
def to_dict(self) -> Dict:
body: Dict[str, Any] = {
"start": self.start,
"end": self.end,
"compositeQuery": {
"queries": [q.to_dict() for q in self.queries],
},
}
if self.request_type is not None:
body["requestType"] = self.request_type
return body
def make_query_request(
signoz: types.SigNoz,
token: str,

View File

@@ -664,9 +664,7 @@ def test_saml_sso_deleted_user_gets_new_user_on_login(
# --- Step 3: SSO login should be blocked for deleted user ---
create_user_idp(email, "password", True, "SAML", "Lifecycle")
perform_saml_login(
signoz, driver, get_session_context, idp_login, email, "password"
)
perform_saml_login(signoz, driver, get_session_context, idp_login, email, "password")
# Verify user is NOT reactivated — check via DB since API may filter deleted users
with signoz.sqlstore.conn.connect() as conn:
@@ -685,11 +683,7 @@ def test_saml_sso_deleted_user_gets_new_user_on_login(
headers={"Authorization": f"Bearer {admin_token}"},
)
found_user = next(
(
user
for user in response.json()["data"]
if user["email"] == email and user["id"] != user_id
),
(user for user in response.json()["data"] if user["email"] == email and user["id"] != user_id),
None,
)
assert found_user is not None

View File

@@ -4,6 +4,7 @@ from urllib.parse import urlparse
import requests
from selenium import webdriver
from sqlalchemy import sql
from wiremock.resources.mappings import Mapping
from fixtures.auth import (

View File

@@ -110,7 +110,9 @@ def test_invite_and_register(
signoz.self.host_configs["8080"].get("/api/v1/invite"),
json={"email": "editor@integration.test", "role": "EDITOR", "name": "editor"},
timeout=2,
headers={"Authorization": f"Bearer {admin_token}"},
headers={
"Authorization": f"Bearer {admin_token}"
},
)
assert response.status_code == HTTPStatus.CREATED
@@ -137,7 +139,9 @@ def test_invite_and_register(
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/user"),
timeout=2,
headers={"Authorization": f"Bearer {editor_token}"},
headers={
"Authorization": f"Bearer {editor_token}"
},
)
assert response.status_code == HTTPStatus.FORBIDDEN
@@ -190,6 +194,7 @@ def test_revoke_invite_and_register(
)
assert response.status_code == HTTPStatus.NO_CONTENT
# Try to use the reset token — should fail (user deleted)
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/resetPassword"),
@@ -234,11 +239,7 @@ def test_old_invite_flow(signoz: types.SigNoz, get_token: Callable[[str, str], s
# invite a new user
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/invite"),
json={
"email": "oldinviteflow@integration.test",
"role": "VIEWER",
"name": "old invite flow",
},
json={"email": "oldinviteflow@integration.test", "role": "VIEWER", "name": "old invite flow"},
timeout=2,
headers={"Authorization": f"Bearer {admin_token}"},
)
@@ -248,7 +249,9 @@ def test_old_invite_flow(signoz: types.SigNoz, get_token: Callable[[str, str], s
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/invite"),
timeout=2,
headers={"Authorization": f"Bearer {admin_token}"},
headers={
"Authorization": f"Bearer {admin_token}"
},
)
invite_response = response.json()["data"]
@@ -294,17 +297,15 @@ def test_old_invite_flow(signoz: types.SigNoz, get_token: Callable[[str, str], s
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/user"),
timeout=2,
headers={"Authorization": f"Bearer {admin_token}"},
headers={
"Authorization": f"Bearer {admin_token}"
},
)
assert response.status_code == HTTPStatus.OK
user_response = response.json()["data"]
found_user = next(
(
user
for user in user_response
if user["email"] == "oldinviteflow@integration.test"
),
(user for user in user_response if user["email"] == "oldinviteflow@integration.test"),
None,
)

View File

@@ -6,6 +6,8 @@ import requests
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.types import SigNoz
from sqlalchemy import sql
def test_reinvite_deleted_user(
signoz: SigNoz,
@@ -29,11 +31,7 @@ def test_reinvite_deleted_user(
# invite the user
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/invite"),
json={
"email": reinvite_user_email,
"role": reinvite_user_role,
"name": reinvite_user_name,
},
json={"email": reinvite_user_email, "role": reinvite_user_role, "name": reinvite_user_name},
headers={"Authorization": f"Bearer {admin_token}"},
timeout=2,
)
@@ -60,27 +58,21 @@ def test_reinvite_deleted_user(
# Re-invite the same email — should succeed
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/invite"),
json={
"email": reinvite_user_email,
"role": "VIEWER",
"name": "reinvite user v2",
},
json={"email": reinvite_user_email, "role": "VIEWER", "name": "reinvite user v2"},
headers={"Authorization": f"Bearer {admin_token}"},
timeout=2,
)
assert response.status_code == HTTPStatus.CREATED
reinvited_user = response.json()["data"]
assert reinvited_user["role"] == "VIEWER"
assert reinvited_user["id"] != invited_user["id"] # confirms a new user was created
assert reinvited_user["role"] == "VIEWER"
assert reinvited_user["id"] != invited_user["id"] # confirms a new user was created
reinvited_user_reset_password_token = reinvited_user["token"]
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/resetPassword"),
json={
"password": "newPassword123Z$",
"token": reinvited_user_reset_password_token,
},
json={"password": "newPassword123Z$", "token": reinvited_user_reset_password_token},
timeout=2,
)
assert response.status_code == HTTPStatus.NO_CONTENT
@@ -103,16 +95,8 @@ def test_bulk_invite(
signoz.self.host_configs["8080"].get("/api/v1/invite/bulk"),
json={
"invites": [
{
"email": "bulk1@integration.test",
"role": "EDITOR",
"name": "bulk user 1",
},
{
"email": "bulk2@integration.test",
"role": "VIEWER",
"name": "bulk user 2",
},
{"email": "bulk1@integration.test", "role": "EDITOR", "name": "bulk user 1"},
{"email": "bulk2@integration.test", "role": "VIEWER", "name": "bulk user 2"},
]
},
headers={"Authorization": f"Bearer {admin_token}"},

View File

@@ -42,11 +42,7 @@ def test_unique_index_allows_multiple_deleted_rows(
# Step 1: invite and delete the first user
resp = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/invite"),
json={
"email": UNIQUE_INDEX_USER_EMAIL,
"role": "EDITOR",
"name": "unique index user v1",
},
json={"email": UNIQUE_INDEX_USER_EMAIL, "role": "EDITOR", "name": "unique index user v1"},
headers={"Authorization": f"Bearer {admin_token}"},
timeout=2,
)
@@ -63,11 +59,7 @@ def test_unique_index_allows_multiple_deleted_rows(
# Step 2: re-invite and delete the same email (second deleted row)
resp = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/invite"),
json={
"email": UNIQUE_INDEX_USER_EMAIL,
"role": "EDITOR",
"name": "unique index user v2",
},
json={"email": UNIQUE_INDEX_USER_EMAIL, "role": "EDITOR", "name": "unique index user v2"},
headers={"Authorization": f"Bearer {admin_token}"},
timeout=2,
)
@@ -93,9 +85,9 @@ def test_unique_index_allows_multiple_deleted_rows(
)
deleted_rows = result.fetchall()
assert (
len(deleted_rows) == 2
), f"expected 2 deleted rows for {UNIQUE_INDEX_USER_EMAIL}, got {len(deleted_rows)}"
assert len(deleted_rows) == 2, (
f"expected 2 deleted rows for {UNIQUE_INDEX_USER_EMAIL}, got {len(deleted_rows)}"
)
deleted_ids = {row[0] for row in deleted_rows}
assert first_user_id in deleted_ids
assert second_user_id in deleted_ids

View File

@@ -373,7 +373,6 @@ def test_histogram_count_no_param(
) ## to keep parallel to the cumulative test cases, first_value refers to the value at 10:02
assert values[-1]["value"] == last_values[le]
@pytest.mark.parametrize(
"space_agg, zeroth_value, first_value, last_value",
[
@@ -424,7 +423,6 @@ def test_histogram_percentile_for_all_services(
assert result_values[1]["value"] == first_value
assert result_values[-1]["value"] == last_value
@pytest.mark.parametrize(
"space_agg, first_value, last_value",
[
@@ -474,7 +472,6 @@ def test_histogram_percentile_for_cumulative_service(
assert result_values[0]["value"] == first_value
assert result_values[-1]["value"] == last_value
@pytest.mark.parametrize(
"space_agg, zeroth_value, first_value, last_value",
[
@@ -524,4 +521,4 @@ def test_histogram_percentile_for_delta_service(
assert len(result_values) == 60
assert result_values[0]["value"] == zeroth_value
assert result_values[1]["value"] == first_value
assert result_values[-1]["value"] == last_value
assert result_values[-1]["value"] == last_value

View File

@@ -1,670 +0,0 @@
import csv
import io
import json
from datetime import datetime, timedelta, timezone
from http import HTTPStatus
from typing import Callable, List
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
from fixtures.querier import BuilderQuery, OrderBy, QueryRangeRequest, TelemetryFieldKey
def test_export_raw_data_get_not_allowed(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
1. GET request to export_raw_data is rejected with 405 Method Not Allowed
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data"),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.METHOD_NOT_ALLOWED
def test_export_logs_csv(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert 3 logs with different severity levels and attributes.
Tests:
1. Export logs as CSV format
2. Verify CSV structure and content
3. Validate headers are present
4. Check log data is correctly formatted
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="Application started successfully",
severity_text="INFO",
resources={
"service.name": "api-service",
"deployment.environment": "production",
"host.name": "server-01",
},
attributes={
"http.method": "GET",
"http.status_code": 200,
"user.id": "user123",
},
),
Logs(
timestamp=now - timedelta(seconds=8),
body="Connection to database failed",
severity_text="ERROR",
resources={
"service.name": "api-service",
"deployment.environment": "production",
"host.name": "server-01",
},
attributes={
"error.type": "ConnectionError",
"db.name": "production_db",
},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Request processed",
severity_text="DEBUG",
resources={
"service.name": "worker-service",
"deployment.environment": "production",
"host.name": "server-02",
},
attributes={
"request.id": "req-456",
"duration_ms": 150.5,
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[BuilderQuery(signal="logs", name="A")],
).to_dict()
# Export logs as CSV (default format)
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data"),
json=body,
timeout=30,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows, got {len(rows)}"
# Verify log bodies are present in the exported data
bodies = [row.get("body") for row in rows]
assert "Application started successfully" in bodies
assert "Connection to database failed" in bodies
assert "Request processed" in bodies
# Verify severity levels
severities = [row.get("severity_text") for row in rows]
assert "INFO" in severities
assert "ERROR" in severities
assert "DEBUG" in severities
def test_export_logs_jsonl(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert 2 logs with different attributes.
Tests:
1. Export logs as JSONL format
2. Verify JSONL structure and content
3. Check each line is valid JSON
4. Validate log data is correctly formatted
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="User logged in",
severity_text="INFO",
resources={
"service.name": "auth-service",
"deployment.environment": "staging",
},
attributes={
"user.email": "test@example.com",
"session.id": "sess-789",
},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Payment processed successfully",
severity_text="INFO",
resources={
"service.name": "payment-service",
"deployment.environment": "staging",
},
attributes={
"transaction.id": "txn-123",
"amount": 99.99,
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[BuilderQuery(signal="logs", name="A")],
).to_dict()
# Export logs as JSONL
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 2, f"Expected 2 lines, got {len(jsonl_lines)}"
# Verify each line is valid JSON
json_objects = []
for line in jsonl_lines:
obj = json.loads(line)
json_objects.append(obj)
assert "id" in obj
assert "timestamp" in obj
assert "body" in obj
assert "severity_text" in obj
# Verify log bodies
bodies = [obj.get("body") for obj in json_objects]
assert "User logged in" in bodies
assert "Payment processed successfully" in bodies
def test_export_logs_with_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs with different severity levels.
Tests:
1. Export logs with filter applied
2. Verify only filtered logs are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="Info message",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=8),
body="Error message",
severity_text="ERROR",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Another error message",
severity_text="ERROR",
resources={
"service.name": "test-service",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[
BuilderQuery(
signal="logs", name="A", filter_expression="severity_text = 'ERROR'"
)
],
).to_dict()
# Export logs with filter
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 2, f"Expected 2 lines (filtered), got {len(jsonl_lines)}"
# Verify only ERROR logs are returned
for line in jsonl_lines:
obj = json.loads(line)
assert obj["severity_text"] == "ERROR"
assert "error message" in obj["body"].lower()
def test_export_logs_with_limit(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert 5 logs.
Tests:
1. Export logs with limit applied
2. Verify only limited number of logs are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
logs = []
for i in range(5):
logs.append(
Logs(
timestamp=now - timedelta(seconds=i),
body=f"Log message {i}",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={
"index": i,
},
)
)
insert_logs(logs)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[BuilderQuery(signal="logs", name="A", limit=3)],
).to_dict()
# Export logs with limit
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=csv"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows (limited), got {len(rows)}"
def test_export_logs_with_columns(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs with various attributes.
Tests:
1. Export logs with specific columns
2. Verify only specified columns are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="Test log message",
severity_text="INFO",
resources={
"service.name": "test-service",
"deployment.environment": "production",
},
attributes={
"http.method": "GET",
"http.status_code": 200,
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[
BuilderQuery(
signal="logs",
name="A",
select_fields=[
TelemetryFieldKey("timestamp", "string", "log"),
TelemetryFieldKey("severity_text", "string", "log"),
TelemetryFieldKey("body", "string", "log"),
],
)
],
).to_dict()
# Export logs with specific columns
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=csv"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 1
# Verify the specified columns are present
row = rows[0]
assert "timestamp" in row
assert "severity_text" in row
assert "body" in row
assert row["severity_text"] == "INFO"
assert row["body"] == "Test log message"
def test_export_logs_with_order_by(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs at different timestamps.
Tests:
1. Export logs with ascending timestamp order
2. Verify logs are returned in correct order
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="First log",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Second log",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=1),
body="Third log",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[
BuilderQuery(
signal="logs",
name="A",
order=[OrderBy(TelemetryFieldKey("timestamp", "string", "log"), "asc")],
)
],
).to_dict()
# Export logs with ascending order
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 3
# Verify order - first log should be "First log" (oldest)
json_objects = [json.loads(line) for line in jsonl_lines]
assert json_objects[0]["body"] == "First log"
assert json_objects[1]["body"] == "Second log"
assert json_objects[2]["body"] == "Third log"
def test_export_logs_with_complex_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs with various service names and severity levels.
Tests:
1. Export logs with complex filter (multiple conditions)
2. Verify only logs matching all conditions are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="API error occurred",
severity_text="ERROR",
resources={
"service.name": "api-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=8),
body="Worker info message",
severity_text="INFO",
resources={
"service.name": "worker-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="API info message",
severity_text="INFO",
resources={
"service.name": "api-service",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[
BuilderQuery(
signal="logs",
name="A",
filter_expression="service.name = 'api-service' AND severity_text = 'ERROR'",
)
],
).to_dict()
# Export logs with complex filter
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert (
len(jsonl_lines) == 1
), f"Expected 1 line (complex filter), got {len(jsonl_lines)}"
# Verify the filtered log
filtered_obj = json.loads(jsonl_lines[0])
assert filtered_obj["body"] == "API error occurred"
assert filtered_obj["severity_text"] == "ERROR"

View File

@@ -1,763 +0,0 @@
import csv
import io
import json
from datetime import datetime, timedelta, timezone
from http import HTTPStatus
from typing import Callable, List
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import (
BuilderQuery,
OrderBy,
QueryRangeRequest,
TelemetryFieldKey,
TraceOperatorQuery,
)
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
def test_export_raw_data_get_not_allowed(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
1. GET request to export_raw_data is rejected with 405 Method Not Allowed
"""
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data"),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.METHOD_NOT_ALLOWED
def test_export_traces_csv(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert 3 traces with different attributes.
Tests:
1. Export traces as CSV format
2. Verify CSV structure and content
3. Validate headers are present
4. Check trace data is correctly formatted
"""
http_service_trace_id = TraceIdGenerator.trace_id()
http_service_span_id = TraceIdGenerator.span_id()
http_service_db_span_id = TraceIdGenerator.span_id()
topic_service_trace_id = TraceIdGenerator.trace_id()
topic_service_span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=3),
trace_id=http_service_trace_id,
span_id=http_service_span_id,
parent_span_id="",
name="POST /integration",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"deployment.environment": "production",
"service.name": "http-service",
"os.type": "linux",
"host.name": "linux-000",
},
attributes={
"net.transport": "IP.TCP",
"http.scheme": "http",
"http.user_agent": "Integration Test",
"http.request.method": "POST",
"http.response.status_code": "200",
},
),
Traces(
timestamp=now - timedelta(seconds=3.5),
duration=timedelta(seconds=0.5),
trace_id=http_service_trace_id,
span_id=http_service_db_span_id,
parent_span_id=http_service_span_id,
name="SELECT",
kind=TracesKind.SPAN_KIND_CLIENT,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"deployment.environment": "production",
"service.name": "http-service",
"os.type": "linux",
"host.name": "linux-000",
},
attributes={
"db.name": "integration",
"db.operation": "SELECT",
"db.statement": "SELECT * FROM integration",
},
),
Traces(
timestamp=now - timedelta(seconds=1),
duration=timedelta(seconds=2),
trace_id=topic_service_trace_id,
span_id=topic_service_span_id,
parent_span_id="",
name="topic publish",
kind=TracesKind.SPAN_KIND_PRODUCER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"deployment.environment": "production",
"service.name": "topic-service",
"os.type": "linux",
"host.name": "linux-001",
},
attributes={
"message.type": "SENT",
"messaging.operation": "publish",
"messaging.message.id": "001",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[BuilderQuery(signal="traces", name="A", limit=1000)],
).to_dict()
# Export traces as CSV
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data"),
json=body,
timeout=30,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows, got {len(rows)}"
# Verify trace IDs are present in the exported data
trace_ids = [row.get("trace_id") for row in rows]
assert http_service_trace_id in trace_ids
assert topic_service_trace_id in trace_ids
# Verify span names are present
span_names = [row.get("name") for row in rows]
assert "POST /integration" in span_names
assert "SELECT" in span_names
assert "topic publish" in span_names
def test_export_traces_jsonl(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert 2 traces with different attributes.
Tests:
1. Export traces as JSONL format
2. Verify JSONL structure and content
3. Check each line is valid JSON
4. Validate trace data is correctly formatted
"""
http_service_trace_id = TraceIdGenerator.trace_id()
http_service_span_id = TraceIdGenerator.span_id()
topic_service_trace_id = TraceIdGenerator.trace_id()
topic_service_span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=3),
trace_id=http_service_trace_id,
span_id=http_service_span_id,
parent_span_id="",
name="POST /api/test",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "api-service",
"deployment.environment": "staging",
},
attributes={
"http.request.method": "POST",
"http.response.status_code": "201",
},
),
Traces(
timestamp=now - timedelta(seconds=2),
duration=timedelta(seconds=1),
trace_id=topic_service_trace_id,
span_id=topic_service_span_id,
parent_span_id="",
name="queue.process",
kind=TracesKind.SPAN_KIND_CONSUMER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "queue-service",
"deployment.environment": "staging",
},
attributes={
"messaging.operation": "process",
"messaging.system": "rabbitmq",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[BuilderQuery(signal="traces", name="A", limit=1000)],
).to_dict()
# Export traces as JSONL
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 2, f"Expected 2 lines, got {len(jsonl_lines)}"
# Verify each line is valid JSON
json_objects = []
for line in jsonl_lines:
obj = json.loads(line)
json_objects.append(obj)
assert "trace_id" in obj
assert "span_id" in obj
assert "name" in obj
# Verify trace IDs are present
trace_ids = [obj.get("trace_id") for obj in json_objects]
assert http_service_trace_id in trace_ids
assert topic_service_trace_id in trace_ids
# Verify span names are present
span_names = [obj.get("name") for obj in json_objects]
assert "POST /api/test" in span_names
assert "queue.process" in span_names
def test_export_traces_with_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert traces with different service names.
Tests:
1. Export traces with filter applied
2. Verify only filtered traces are returned
"""
service_a_trace_id = TraceIdGenerator.trace_id()
service_a_span_id = TraceIdGenerator.span_id()
service_b_trace_id = TraceIdGenerator.trace_id()
service_b_span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=1),
trace_id=service_a_trace_id,
span_id=service_a_span_id,
parent_span_id="",
name="operation-a",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "service-a",
},
attributes={},
),
Traces(
timestamp=now - timedelta(seconds=2),
duration=timedelta(seconds=1),
trace_id=service_b_trace_id,
span_id=service_b_span_id,
parent_span_id="",
name="operation-b",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "service-b",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[
BuilderQuery(
signal="traces",
name="A",
limit=1000,
filter_expression="service.name = 'service-a'",
)
],
).to_dict()
# Export traces with filter
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 1, f"Expected 1 line (filtered), got {len(jsonl_lines)}"
# Verify the filtered trace
filtered_obj = json.loads(jsonl_lines[0])
assert filtered_obj["trace_id"] == service_a_trace_id
assert filtered_obj["name"] == "operation-a"
def test_export_traces_with_limit(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert 5 traces.
Tests:
1. Export traces with limit applied
2. Verify only limited number of traces are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
traces = []
for i in range(5):
traces.append(
Traces(
timestamp=now - timedelta(seconds=i),
duration=timedelta(seconds=1),
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
parent_span_id="",
name=f"operation-{i}",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "test-service",
},
attributes={},
)
)
insert_traces(traces)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[BuilderQuery(signal="traces", name="A", limit=3)],
).to_dict()
# Export traces with limit
response = requests.post(
signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=csv"),
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows (limited), got {len(rows)}"
def test_export_traces_multiple_queries_rejected(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
1. POST with multiple builder queries but no trace operator is rejected
2. Verify 400 error is returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
request_type=None,
queries=[
BuilderQuery(
signal="traces",
name="A",
limit=1000,
filter_expression="service.name = 'service-a'",
),
BuilderQuery(
signal="traces",
name="B",
limit=1000,
filter_expression="service.name = 'service-b'",
),
],
).to_dict()
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
response = requests.post(
url,
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.BAD_REQUEST
def test_export_traces_with_composite_query_trace_operator(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert multiple traces with parent-child relationships.
Tests:
1. Export traces using trace operator in composite query (POST)
2. Verify trace operator query works correctly
"""
parent_trace_id = TraceIdGenerator.trace_id()
parent_span_id = TraceIdGenerator.span_id()
child_span_id_1 = TraceIdGenerator.span_id()
child_span_id_2 = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=10),
duration=timedelta(seconds=5),
trace_id=parent_trace_id,
span_id=parent_span_id,
parent_span_id="",
name="parent-operation",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "parent-service",
},
attributes={
"operation.type": "parent",
},
),
Traces(
timestamp=now - timedelta(seconds=9),
duration=timedelta(seconds=2),
trace_id=parent_trace_id,
span_id=child_span_id_1,
parent_span_id=parent_span_id,
name="child-operation-1",
kind=TracesKind.SPAN_KIND_INTERNAL,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "parent-service",
},
attributes={
"operation.type": "child",
},
),
Traces(
timestamp=now - timedelta(seconds=7),
duration=timedelta(seconds=1),
trace_id=parent_trace_id,
span_id=child_span_id_2,
parent_span_id=parent_span_id,
name="child-operation-2",
kind=TracesKind.SPAN_KIND_INTERNAL,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "parent-service",
},
attributes={
"operation.type": "child",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
# A: spans with operation.type = 'parent'
query_a = BuilderQuery(
signal="traces",
name="A",
limit=1000,
filter_expression="operation.type = 'parent'",
)
# B: spans with operation.type = 'child'
query_b = BuilderQuery(
signal="traces",
name="B",
limit=1000,
filter_expression="operation.type = 'child'",
)
# Trace operator: find traces where A has a direct descendant B
query_c = TraceOperatorQuery(
name="C",
expression="A => B",
return_spans_from="A",
limit=1000,
order=[OrderBy(TelemetryFieldKey("timestamp", "string", "span"), "desc")],
)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[query_a, query_b, query_c],
).to_dict()
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
response = requests.post(
url,
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 1, f"Expected at least 1 line, got {len(jsonl_lines)}"
# Verify all returned spans belong to the matched trace
json_objects = [json.loads(line) for line in jsonl_lines]
trace_ids = [obj.get("trace_id") for obj in json_objects]
assert all(tid == parent_trace_id for tid in trace_ids)
# Verify the parent span (returnSpansFrom = "A") is present
span_names = [obj.get("name") for obj in json_objects]
assert "parent-operation" in span_names
def test_export_traces_with_select_fields(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert traces with various attributes.
Tests:
1. Export traces with specific select fields via POST
2. Verify only specified fields are returned in the output
"""
trace_id = TraceIdGenerator.trace_id()
span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=10),
duration=timedelta(seconds=2),
trace_id=trace_id,
span_id=span_id,
parent_span_id="",
name="test-operation",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "test-service",
"deployment.environment": "production",
"host.name": "server-01",
},
attributes={
"http.method": "POST",
"http.status_code": "201",
"user.id": "user123",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = QueryRangeRequest(
start=start_ns,
end=end_ns,
queries=[
BuilderQuery(
signal="traces",
name="A",
limit=1000,
select_fields=[
TelemetryFieldKey("trace_id", "string", "span"),
TelemetryFieldKey("span_id", "string", "span"),
TelemetryFieldKey("name", "string", "span"),
TelemetryFieldKey("service.name", "string", "resource"),
],
)
],
).to_dict()
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
response = requests.post(
url,
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 1
# Verify the selected fields are present
result = json.loads(jsonl_lines[0])
assert "trace_id" in result
assert "span_id" in result
assert "name" in result
# Verify values
assert result["trace_id"] == trace_id
assert result["span_id"] == span_id
assert result["name"] == "test-operation"