Compare commits

..

14 Commits

Author SHA1 Message Date
Srikanth Chekuri
7aa0eff6f4 Merge branch 'main' into tvats-export-traces 2026-02-27 15:23:28 +05:30
Tushar Vats
0e30a11231 fix: address comments 2026-02-27 03:39:08 +05:30
Srikanth Chekuri
5901074385 Merge branch 'main' into tvats-export-traces 2026-02-26 06:57:14 +05:30
Tushar Vats
5b27121530 fix: ran yarn generate:api 2026-02-26 02:56:37 +05:30
Tushar Vats
7090d1af17 fix: lint error 2026-02-26 02:39:43 +05:30
Tushar Vats
31152e01f5 fix: lint error 2026-02-25 21:54:25 +05:30
Tushar Vats
e19b37620d Merge branch 'main' into tvats-export-traces 2026-02-25 21:46:49 +05:30
Tushar Vats
820f0723a5 fix: address comments 2026-02-25 21:44:32 +05:30
Tushar Vats
b8c101e95a fix: renamed method 2026-02-19 10:53:33 +05:30
Tushar Vats
85c9daab12 fix: rebased main and ran generate cmd 2026-02-19 10:50:00 +05:30
Tushar Vats
4a80ef8f43 fix: go fmt 2026-02-19 10:47:33 +05:30
Tushar Vats
ac8600f5da fix: removed nits 2026-02-19 10:47:33 +05:30
Tushar Vats
b2655634fe fix: address comments 2026-02-19 10:47:25 +05:30
Tushar Vats
b2f20f7a64 feat: added trace export
feat: added types for export

feat: added support for complex queries

fix: added correct open api spec

fix: updated unit tests

fix: type handling logic

fix: improve order by

feat: added integration tests

fix: address comments
2026-02-19 10:44:45 +05:30
31 changed files with 3630 additions and 2235 deletions

View File

@@ -17,5 +17,7 @@
},
"[html]": {
"editor.defaultFormatter": "vscode.html-language-features"
}
},
"python-envs.defaultEnvManager": "ms-python.python:system",
"python-envs.pythonProjects": []
}

View File

@@ -2858,6 +2858,186 @@ paths:
summary: Update auth domain
tags:
- authdomains
/api/v1/export_raw_data:
get:
deprecated: false
description: This endpoints allows simple query exporting raw data for traces
and logs
operationId: HandleExportRawDataGET
parameters:
- in: query
name: format
schema:
default: csv
enum:
- csv
- jsonl
type: string
- in: query
name: signal
required: true
schema:
$ref: '#/components/schemas/TelemetrytypesSignal'
- deprecated: true
in: query
name: source
schema:
deprecated: true
type: string
- in: query
name: start
schema:
minimum: 0
type: integer
- in: query
name: end
schema:
minimum: 0
type: integer
- in: query
name: limit
schema:
default: 10000
maximum: 50000
minimum: 1
type: integer
- deprecated: true
in: query
name: filter
schema:
deprecated: true
type: string
- content:
application/json:
schema:
$ref: '#/components/schemas/Querybuildertypesv5Filter'
in: query
name: filterExpression
- deprecated: true
in: query
name: columns
schema:
deprecated: true
items:
type: string
type: array
- in: query
name: selectFields
schema:
items:
$ref: '#/components/schemas/TelemetrytypesTelemetryFieldKey'
type: array
- deprecated: true
in: query
name: order_by
schema:
deprecated: true
type: string
- in: query
name: order
schema:
items:
$ref: '#/components/schemas/Querybuildertypesv5OrderBy'
type: array
responses:
"200":
content:
application/json:
schema:
type: string
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Export raw data
tags:
- logs
- traces
post:
deprecated: false
description: This endpoints allows complex query exporting raw data for traces
and logs
operationId: HandleExportRawDataPOST
parameters:
- in: query
name: format
schema:
default: csv
enum:
- csv
- jsonl
type: string
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/Querybuildertypesv5QueryRangeRequest'
responses:
"200":
content:
application/json:
schema:
type: string
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: Export raw data
tags:
- logs
- traces
/api/v1/fields/keys:
get:
deprecated: false

View File

@@ -123,6 +123,7 @@ if err := router.Handle("/api/v1/things", handler.New(
Description: "This endpoint creates a thing",
Request: new(types.PostableThing),
RequestContentType: "application/json",
RequestQuery: new(types.QueryableThing),
Response: new(types.GettableThing),
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusCreated,
@@ -155,6 +156,8 @@ The `handler.New` function ties the HTTP handler to OpenAPI metadata via `OpenAP
- **Request / RequestContentType**:
- `Request` is a Go type that describes the request body or form.
- `RequestContentType` is usually `"application/json"` or `"application/x-www-form-urlencoded"` (for callbacks like SAML).
- **RequestQuery**:
- `RequestQuery` is a Go type that descirbes query url params.
- **RequestExamples**: An array of `handler.OpenAPIExample` that provide concrete request payloads in the generated spec. See [Adding request examples](#adding-request-examples) below.
- **Response / ResponseContentType**:
- `Response` is the Go type for the successful response payload.

View File

@@ -1,126 +0,0 @@
# Packages
All shared Go code in SigNoz lives under `pkg/`. Each package represents a distinct domain concept and exposes a clear public interface. This guide covers the conventions for creating, naming, and organising packages so the codebase stays consistent as it grows.
## How should I name a package?
Use short, lowercase, single-word names. No underscores or camelCase (`querier`, `cache`, `authz`, not `query_builder` or `dataStore`).
Names must be **domain-specific**. A package name should tell you what problem domain it deals with, not what data structure it wraps. Prefer `alertmanager` over `manager`, `licensing` over `checker`.
Avoid generic names like `util`, `helpers`, `common`, `misc`, or `base`. If you can't name it, the code probably belongs in an existing package.
## When should I create a new package?
Create a new package when:
- The functionality represents a **distinct domain concept** (e.g., `authz`, `licensing`, `cache`).
- Two or more other packages would import it; it serves as shared infrastructure.
- The code has a clear public interface that can stand on its own.
Do **not** create a new package when:
- There is already a package that covers the same domain. Extend the existing package instead.
- The code is only used in one place. Keep it local to the caller.
- You are splitting purely for file size. Use multiple files within the same package instead.
## How should I lay out a package?
A typical package looks like:
```
pkg/cache/
├── cache.go # Public interface + exported types
├── config.go # Configuration types if needed
├── memorycache/ # Implementation sub-package
├── rediscache/ # Another implementation
└── cachetest/ # Test helpers for consumers
```
Follow these rules:
1. **Interface-first file**: The file matching the package name (e.g., `cache.go` in `pkg/cache/`) should define the public interface and core exported types. Keep implementation details out of this file.
2. **One responsibility per file**: Name files after what they contain (`config.go`, `handler.go`, `service.go`), not after the package name. If a package merges two concerns, prefix files to group them (e.g., `memory_store.go`, `redis_store.go` in a storage package).
3. **Sub-packages for implementations**: When a package defines an interface with multiple implementations, put each implementation in its own sub-package (`memorycache/`, `rediscache/`). This keeps the parent package import-free of implementation dependencies.
4. **Test helpers in `{pkg}test/`**: If consumers need test mocks or builders, put them in a `{pkg}test/` sub-package (e.g., `cachetest/`, `sqlstoretest/`). This avoids polluting the main package with test-only code.
5. **Test files stay alongside source**: Unit tests go in `_test.go` files next to the code they test, in the same package.
## How should I name symbols?
### Exported symbols
- **Interfaces**: For single-method interfaces, follow the standard `-er` suffix convention (`Reader`, `Writer`, `Closer`). For multi-method interfaces, use clear nouns (`Cache`, `Store`, `Provider`).
- **Constructors**: `New<Type>(...)` (e.g., `NewMemoryCache()`).
- **Avoid stutter**: Since callers qualify with the package name, don't repeat it. Write `cache.Cache`, not `cache.CacheInterface`. Write `authz.FromRole`, not `authz.AuthzFromRole`.
### Unexported symbols
- Struct receivers: one or two characters (`c`, `f`, `br`).
- Helper functions: descriptive lowercase names (`parseToken`, `buildQuery`).
### Constants
- Use `PascalCase` for exported constants.
- When merging files from different origins into one package, watch out for **name collisions** across files. Prefix to disambiguate when two types share a natural name.
## How should I organise imports?
Group imports in three blocks separated by blank lines:
```go
import (
// 1. Standard library
"fmt"
"net/http"
// 2. External dependencies
"github.com/gorilla/mux"
// 3. Internal
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types"
)
```
Never introduce circular imports. If package A needs package B and B needs A, extract the shared types into a third package (often under `pkg/types/`).
## Where do shared types go?
Most types belong in `pkg/types/` under a domain-specific sub-package (e.g., `pkg/types/ruletypes`, `pkg/types/authtypes`).
Do not put domain logic in `pkg/types/`. Only data structures, constants, and simple methods.
## How do I merge or move packages?
When two packages are tightly coupled (one imports the other's constants, they cover the same domain), merge them:
1. Pick a domain-specific name for the combined package.
2. Prefix files to preserve origin (e.g., `memory_store.go`, `redis_store.go`).
3. Resolve symbol conflicts explicitly; rename with a prefix rather than silently shadowing.
4. Update all consumers in a single change.
5. Delete the old packages. Do not leave behind re-export shims.
6. Verify with `go build ./...`, `go test ./<new-pkg>/...`, and `go vet ./...`.
## When should I add documentation?
Add a `doc.go` with a package-level comment for any package that is non-trivial or has multiple consumers. Keep it to 13 sentences:
```go
// Package cache provides a caching interface with pluggable backends
// for in-memory and Redis-based storage.
package cache
```
## What should I remember?
- Package names are domain-specific and lowercase. Never generic names like `util` or `common`.
- The file matching the package name (e.g., `cache.go`) defines the public interface. Implementation details go elsewhere.
- Never introduce circular imports. Extract shared types into `pkg/types/` when needed.
- Watch for symbol name collisions when merging packages, prefix to disambiguate.
- Put test helpers in a `{pkg}test/` sub-package, not in the main package.
- Before submitting, verify with `go build ./...`, `go test ./<your-pkg>/...`, and `go vet ./...`.
- Update all consumers when you rename or move symbols.

View File

@@ -8,13 +8,4 @@ We adhere to three primary style guides as our foundation:
- [Code Review Comments](https://go.dev/wiki/CodeReviewComments) - For understanding common comments in code reviews
- [Google Style Guide](https://google.github.io/styleguide/go/) - Additional practices from Google
We **recommend** (almost enforce) reviewing these guides before contributing to the codebase. They provide valuable insights into writing idiomatic Go code and will help you understand our approach to backend development. In addition, we have a few additional rules that make certain areas stricter than the above which can be found in area-specific files in this package:
- [Packages](packages.md) — Naming, layout, and conventions for `pkg/` packages
- [Errors](errors.md) — Structured error handling
- [Handler](handler.md) — Writing HTTP handlers and OpenAPI integration
- [Endpoint](endpoint.md) — Endpoint conventions
- [SQL](sql.md) — Database query patterns
- [Provider](provider.md) — Provider pattern
- [Integration](integration.md) — Integration conventions
- [Flagger](flagger.md) — Feature flag conventions
We **recommend** (almost enforce) reviewing these guides before contributing to the codebase. They provide valuable insights into writing idiomatic Go code and will help you understand our approach to backend development. In addition, we have a few additional rules that make certain areas stricter than the above which can be found in area-specific files in this package.

View File

@@ -20,8 +20,11 @@ import { useMutation, useQuery } from 'react-query';
import type { BodyType, ErrorType } from '../../../generatedAPIInstance';
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
import type {
HandleExportRawDataGETParams,
HandleExportRawDataPOSTParams,
ListPromotedAndIndexedPaths200,
PromotetypesPromotePathDTO,
Querybuildertypesv5QueryRangeRequestDTO,
RenderErrorResponseDTO,
} from '../sigNoz.schemas';
@@ -29,6 +32,206 @@ type AwaitedInput<T> = PromiseLike<T> | T;
type Awaited<O> = O extends AwaitedInput<infer T> ? T : never;
/**
* This endpoints allows simple query exporting raw data for traces and logs
* @summary Export raw data
*/
export const handleExportRawDataGET = (
params?: HandleExportRawDataGETParams,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<string>({
url: `/api/v1/export_raw_data`,
method: 'GET',
params,
signal,
});
};
export const getHandleExportRawDataGETQueryKey = (
params?: HandleExportRawDataGETParams,
) => {
return [`/api/v1/export_raw_data`, ...(params ? [params] : [])] as const;
};
export const getHandleExportRawDataGETQueryOptions = <
TData = Awaited<ReturnType<typeof handleExportRawDataGET>>,
TError = ErrorType<RenderErrorResponseDTO>
>(
params?: HandleExportRawDataGETParams,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof handleExportRawDataGET>>,
TError,
TData
>;
},
) => {
const { query: queryOptions } = options ?? {};
const queryKey =
queryOptions?.queryKey ?? getHandleExportRawDataGETQueryKey(params);
const queryFn: QueryFunction<
Awaited<ReturnType<typeof handleExportRawDataGET>>
> = ({ signal }) => handleExportRawDataGET(params, signal);
return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
Awaited<ReturnType<typeof handleExportRawDataGET>>,
TError,
TData
> & { queryKey: QueryKey };
};
export type HandleExportRawDataGETQueryResult = NonNullable<
Awaited<ReturnType<typeof handleExportRawDataGET>>
>;
export type HandleExportRawDataGETQueryError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Export raw data
*/
export function useHandleExportRawDataGET<
TData = Awaited<ReturnType<typeof handleExportRawDataGET>>,
TError = ErrorType<RenderErrorResponseDTO>
>(
params?: HandleExportRawDataGETParams,
options?: {
query?: UseQueryOptions<
Awaited<ReturnType<typeof handleExportRawDataGET>>,
TError,
TData
>;
},
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
const queryOptions = getHandleExportRawDataGETQueryOptions(params, options);
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
queryKey: QueryKey;
};
query.queryKey = queryOptions.queryKey;
return query;
}
/**
* @summary Export raw data
*/
export const invalidateHandleExportRawDataGET = async (
queryClient: QueryClient,
params?: HandleExportRawDataGETParams,
options?: InvalidateOptions,
): Promise<QueryClient> => {
await queryClient.invalidateQueries(
{ queryKey: getHandleExportRawDataGETQueryKey(params) },
options,
);
return queryClient;
};
/**
* This endpoints allows complex query exporting raw data for traces and logs
* @summary Export raw data
*/
export const handleExportRawDataPOST = (
querybuildertypesv5QueryRangeRequestDTO: BodyType<Querybuildertypesv5QueryRangeRequestDTO>,
params?: HandleExportRawDataPOSTParams,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<string>({
url: `/api/v1/export_raw_data`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: querybuildertypesv5QueryRangeRequestDTO,
params,
signal,
});
};
export const getHandleExportRawDataPOSTMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
> => {
const mutationKey = ['handleExportRawDataPOST'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
}
> = (props) => {
const { data, params } = props ?? {};
return handleExportRawDataPOST(data, params);
};
return { mutationFn, ...mutationOptions };
};
export type HandleExportRawDataPOSTMutationResult = NonNullable<
Awaited<ReturnType<typeof handleExportRawDataPOST>>
>;
export type HandleExportRawDataPOSTMutationBody = BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
export type HandleExportRawDataPOSTMutationError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary Export raw data
*/
export const useHandleExportRawDataPOST = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
TError,
{
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
params?: HandleExportRawDataPOSTParams;
},
TContext
> => {
const mutationOptions = getHandleExportRawDataPOSTMutationOptions(options);
return useMutation(mutationOptions);
};
/**
* This endpoints promotes and indexes paths
* @summary Promote and index paths

View File

@@ -2732,6 +2732,76 @@ export type DeleteAuthDomainPathParameters = {
export type UpdateAuthDomainPathParameters = {
id: string;
};
export type HandleExportRawDataGETParams = {
/**
* @enum csv,jsonl
* @type string
* @description undefined
*/
format?: HandleExportRawDataGETFormat;
/**
* @enum logs,traces
* @type string
* @description undefined
*/
source?: HandleExportRawDataGETSource;
/**
* @type integer
* @minimum 0
* @description undefined
*/
start?: number;
/**
* @type integer
* @minimum 0
* @description undefined
*/
end?: number;
/**
* @type integer
* @maximum 50000
* @minimum 1
* @description undefined
*/
limit?: number;
/**
* @type string
* @description undefined
*/
filter?: string;
/**
* @type array
* @description undefined
*/
columns?: string[];
/**
* @type string
* @description undefined
*/
order_by?: string;
};
export enum HandleExportRawDataGETFormat {
csv = 'csv',
jsonl = 'jsonl',
}
export enum HandleExportRawDataGETSource {
logs = 'logs',
traces = 'traces',
}
export type HandleExportRawDataPOSTParams = {
/**
* @enum csv,jsonl
* @type string
* @description undefined
*/
format?: HandleExportRawDataPOSTFormat;
};
export enum HandleExportRawDataPOSTFormat {
csv = 'csv',
jsonl = 'jsonl',
}
export type GetFieldsKeysParams = {
/**
* @description undefined

View File

@@ -1,6 +1,6 @@
import { useState } from 'react';
import { CloudDownloadOutlined } from '@ant-design/icons';
import { Button, Dropdown, MenuProps } from 'antd';
import { Excel } from 'antd-table-saveas-excel';
import { unparse } from 'papaparse';
import { DownloadProps } from './Download.types';
@@ -8,36 +8,25 @@ import { DownloadProps } from './Download.types';
import './Download.styles.scss';
function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
const [isDownloading, setIsDownloading] = useState(false);
const downloadExcelFile = async (): Promise<void> => {
setIsDownloading(true);
try {
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
const updatedTitle = item
.split('_')
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
return {
title: updatedTitle,
dataIndex: item,
};
});
const excelLib = await import('antd-table-saveas-excel');
const excel = new excelLib.Excel();
excel
.addSheet(fileName)
.addColumns(headers)
.addDataSource(data, {
str2Percent: true,
})
.saveAs(`${fileName}.xlsx`);
} finally {
setIsDownloading(false);
}
const downloadExcelFile = (): void => {
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
const updatedTitle = item
.split('_')
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
return {
title: updatedTitle,
dataIndex: item,
};
});
const excel = new Excel();
excel
.addSheet(fileName)
.addColumns(headers)
.addDataSource(data, {
str2Percent: true,
})
.saveAs(`${fileName}.xlsx`);
};
const downloadCsvFile = (): void => {
@@ -70,7 +59,7 @@ function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
<Dropdown menu={menu} trigger={['click']}>
<Button
className="download-button"
loading={isLoading || isDownloading}
loading={isLoading}
size="small"
type="link"
>

View File

@@ -1,5 +1,5 @@
import { useState } from 'react';
import { Button, Popover, Typography } from 'antd';
import { Excel } from 'antd-table-saveas-excel';
import { FileDigit, FileDown, Sheet } from 'lucide-react';
import { unparse } from 'papaparse';
@@ -8,34 +8,25 @@ import { DownloadProps } from './DownloadV2.types';
import './DownloadV2.styles.scss';
function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
const [isDownloading, setIsDownloading] = useState(false);
const downloadExcelFile = async (): Promise<void> => {
setIsDownloading(true);
try {
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
const updatedTitle = item
.split('_')
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
return {
title: updatedTitle,
dataIndex: item,
};
});
const excelLib = await import('antd-table-saveas-excel');
const excel = new excelLib.Excel();
excel
.addSheet(fileName)
.addColumns(headers)
.addDataSource(data, {
str2Percent: true,
})
.saveAs(`${fileName}.xlsx`);
} finally {
setIsDownloading(false);
}
const downloadExcelFile = (): void => {
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
const updatedTitle = item
.split('_')
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
.join(' ');
return {
title: updatedTitle,
dataIndex: item,
};
});
const excel = new Excel();
excel
.addSheet(fileName)
.addColumns(headers)
.addDataSource(data, {
str2Percent: true,
})
.saveAs(`${fileName}.xlsx`);
};
const downloadCsvFile = (): void => {
@@ -63,7 +54,6 @@ function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
type="text"
onClick={downloadExcelFile}
className="action-btns"
loading={isDownloading}
>
Excel (.xlsx)
</Button>

View File

@@ -18,6 +18,7 @@ import (
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/preference"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
"github.com/SigNoz/signoz/pkg/modules/session"
"github.com/SigNoz/signoz/pkg/modules/user"
"github.com/SigNoz/signoz/pkg/querier"
@@ -46,6 +47,7 @@ type provider struct {
gatewayHandler gateway.Handler
fieldsHandler fields.Handler
authzHandler authz.Handler
rawDataExportHandler rawdataexport.Handler
zeusHandler zeus.Handler
querierHandler querier.Handler
}
@@ -67,6 +69,7 @@ func NewFactory(
gatewayHandler gateway.Handler,
fieldsHandler fields.Handler,
authzHandler authz.Handler,
rawDataExportHandler rawdataexport.Handler,
zeusHandler zeus.Handler,
querierHandler querier.Handler,
) factory.ProviderFactory[apiserver.APIServer, apiserver.Config] {
@@ -91,6 +94,7 @@ func NewFactory(
gatewayHandler,
fieldsHandler,
authzHandler,
rawDataExportHandler,
zeusHandler,
querierHandler,
)
@@ -117,6 +121,7 @@ func newProvider(
gatewayHandler gateway.Handler,
fieldsHandler fields.Handler,
authzHandler authz.Handler,
rawDataExportHandler rawdataexport.Handler,
zeusHandler zeus.Handler,
querierHandler querier.Handler,
) (apiserver.APIServer, error) {
@@ -141,6 +146,7 @@ func newProvider(
gatewayHandler: gatewayHandler,
fieldsHandler: fieldsHandler,
authzHandler: authzHandler,
rawDataExportHandler: rawDataExportHandler,
zeusHandler: zeusHandler,
querierHandler: querierHandler,
}
@@ -215,6 +221,10 @@ func (provider *provider) AddToRouter(router *mux.Router) error {
return err
}
if err := provider.addRawDataExportRoutes(router); err != nil {
return err
}
if err := provider.addZeusRoutes(router); err != nil {
return err
}

View File

@@ -0,0 +1,47 @@
package signozapiserver
import (
"net/http"
"github.com/SigNoz/signoz/pkg/http/handler"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/types/exporttypes"
v5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/gorilla/mux"
)
func (provider *provider) addRawDataExportRoutes(router *mux.Router) error {
if err := router.Handle("/api/v1/export_raw_data", handler.New(provider.authZ.ViewAccess(provider.rawDataExportHandler.ExportRawData), handler.OpenAPIDef{
ID: "HandleExportRawDataGET",
Tags: []string{"logs", "traces"},
Summary: "Export raw data",
Description: "This endpoints allows simple query exporting raw data for traces and logs",
RequestQuery: new(exporttypes.ExportRawDataQueryParams),
RequestContentType: "",
Response: nil,
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest},
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
})).Methods(http.MethodGet).GetError(); err != nil {
return err
}
if err := router.Handle("/api/v1/export_raw_data", handler.New(provider.authZ.ViewAccess(provider.rawDataExportHandler.ExportRawData), handler.OpenAPIDef{
ID: "HandleExportRawDataPOST",
Tags: []string{"logs", "traces"},
Summary: "Export raw data",
Description: "This endpoints allows complex query exporting raw data for traces and logs",
Request: new(v5.QueryRangeRequest),
RequestQuery: new(exporttypes.ExportRawDataFormatQueryParam),
RequestContentType: "application/json",
Response: nil,
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest},
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
})).Methods(http.MethodPost).GetError(); err != nil {
return err
}
return nil
}

View File

@@ -1,62 +0,0 @@
package cloudintegration
import (
"context"
"net/http"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
type Module interface {
GetName() cloudintegrationtypes.CloudProviderType
// AgentCheckIn is called by agent to heartbeat and get latest config in response.
AgentCheckIn(ctx context.Context, req *cloudintegrationtypes.PostableAgentCheckInPayload) (any, error)
GenerateConnectionParams(ctx context.Context) (*cloudintegrationtypes.GettableCloudIntegrationConnectionParams, error)
// GenerateConnectionArtifact generates cloud provider specific connection information, client side handles how this information is shown
GenerateConnectionArtifact(ctx context.Context, req *cloudintegrationtypes.PostableConnectionArtifact) (any, error)
// GetAccountStatus returns agent connection status for a cloud integration account
GetAccountStatus(ctx context.Context, orgID, accountID string) (*cloudintegrationtypes.GettableAccountStatus, error)
// ListConnectedAccounts lists accounts where agent is connected
ListConnectedAccounts(ctx context.Context, orgID string) (*cloudintegrationtypes.GettableConnectedAccountsList, error)
// LIstServices return list of services for a cloud provider attached with the accountID. This just returns a summary
ListServices(ctx context.Context, orgID string, accountID *string) (any, error) // returns either GettableAWSServices or GettableAzureServices
// GetServiceDetails returns service definition details for a serviceId. This returns config and other details required to show in service details page on client.
GetServiceDetails(ctx context.Context, req *cloudintegrationtypes.GetServiceDetailsReq) (any, error)
// GetDashboard returns dashboard json for a give cloud integration service dashboard.
// this only returns the dashboard when account is connected and service is enabled
GetDashboard(ctx context.Context, id string, orgID valuer.UUID) (*dashboardtypes.Dashboard, error)
// GetAvailableDashboards returns list of available dashboards across all connected cloud integration accounts in the org.
// this list gets added to dashboard list page
GetAvailableDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error)
// UpdateAccountConfig updates cloud integration account config
UpdateAccountConfig(ctx context.Context, orgId valuer.UUID, accountId string, config []byte) (any, error)
// UpdateServiceConfig updates cloud integration service config
UpdateServiceConfig(ctx context.Context, serviceId string, orgID valuer.UUID, config []byte) (any, error)
// DisconnectAccount soft deletes/removes a cloud integration account.
DisconnectAccount(ctx context.Context, orgID, accountID string) (*cloudintegrationtypes.CloudIntegration, error)
}
type Handler interface {
AgentCheckIn(http.ResponseWriter, *http.Request)
GenerateConnectionParams(http.ResponseWriter, *http.Request)
GenerateConnectionArtifact(http.ResponseWriter, *http.Request)
ListConnectedAccounts(http.ResponseWriter, *http.Request)
GetAccountStatus(http.ResponseWriter, *http.Request)
ListServices(http.ResponseWriter, *http.Request)
GetServiceDetails(http.ResponseWriter, *http.Request)
UpdateAccountConfig(http.ResponseWriter, *http.Request)
UpdateServiceConfig(http.ResponseWriter, *http.Request)
DisconnectAccount(http.ResponseWriter, *http.Request)
}

View File

@@ -6,18 +6,18 @@ import (
"fmt"
"io"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/http/binding"
"github.com/SigNoz/signoz/pkg/http/render"
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/exporttypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
@@ -31,129 +31,52 @@ func NewHandler(module rawdataexport.Module) rawdataexport.Handler {
return &handler{module: module}
}
// ExportRawData handles data export requests.
//
// API Documentation:
// Endpoint: GET /api/v1/export_raw_data
//
// Query Parameters:
//
// - source (optional): Type of data to export ["logs" (default), "metrics", "traces"]
// Note: Currently only "logs" is fully supported
//
// - format (optional): Output format ["csv" (default), "jsonl"]
//
// - start (required): Start time for query (Unix timestamp in nanoseconds)
//
// - end (required): End time for query (Unix timestamp in nanoseconds)
//
// - limit (optional): Maximum number of rows to export
// Constraints: Must be positive and cannot exceed MAX_EXPORT_ROW_COUNT_LIMIT
//
// - filter (optional): Filter expression to apply to the query
//
// - columns (optional): Specific columns to include in export
// Default: all columns are returned
// Format: ["context.field:type", "context.field", "field"]
//
// - order_by (optional): Sorting specification ["column:direction" or "context.field:type:direction"]
// Direction: "asc" or "desc"
// Default: ["timestamp:desc", "id:desc"]
//
// Response Headers:
// - Content-Type: "text/csv" or "application/x-ndjson"
// - Content-Encoding: "gzip" (handled by HTTP middleware)
// - Content-Disposition: "attachment; filename=\"data_exported.[format]\""
// - Cache-Control: "no-cache"
// - Vary: "Accept-Encoding"
// - Transfer-Encoding: "chunked"
// - Trailers: X-Response-Complete
//
// Response Format:
//
// CSV: Headers in first row, data in subsequent rows
// JSONL: One JSON object per line
//
// Example Usage:
//
// Basic CSV export:
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000
//
// Export with columns and format:
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000&format=jsonl
// &columns=timestamp&columns=severity&columns=message
//
// Export with filter and ordering:
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000
// &filter=severity="error"&order_by=timestamp:desc&limit=1000
func (handler *handler) ExportRawData(rw http.ResponseWriter, r *http.Request) {
source, err := getExportQuerySource(r.URL.Query())
if err != nil {
render.Error(rw, err)
return
}
var queryRangeRequest qbtypes.QueryRangeRequest
var format string
switch source {
case "logs":
handler.exportLogs(rw, r)
case "traces":
handler.exportTraces(rw, r)
case "metrics":
handler.exportMetrics(rw, r)
switch r.Method {
case http.MethodGet:
var params exporttypes.ExportRawDataQueryParams
if err := binding.Query.BindQuery(r.URL.Query(), &params); err != nil {
render.Error(rw, err)
return
}
params.Normalize()
if err := params.Validate(); err != nil {
render.Error(rw, err)
return
}
format = params.Format
queryRangeRequest = buildQueryRangeRequest(&params)
case http.MethodPost:
var formatParam exporttypes.ExportRawDataFormatQueryParam
if err := binding.Query.BindQuery(r.URL.Query(), &formatParam); err != nil {
render.Error(rw, err)
return
}
format = formatParam.Format
if err := json.NewDecoder(r.Body).Decode(&queryRangeRequest); err != nil {
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid request body: %v", err))
return
}
default:
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid source: must be logs"))
render.Error(rw, errors.Newf(errors.TypeMethodNotAllowed, errors.CodeMethodNotAllowed, "method not allowed, only GET/POST supported"))
return
}
}
func (handler *handler) exportMetrics(rw http.ResponseWriter, r *http.Request) {
render.Error(rw, errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported, "metrics export is not yet supported"))
}
func (handler *handler) exportTraces(rw http.ResponseWriter, r *http.Request) {
render.Error(rw, errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported, "traces export is not yet supported"))
}
func (handler *handler) exportLogs(rw http.ResponseWriter, r *http.Request) {
// Set up response headers
rw.Header().Set("Cache-Control", "no-cache")
rw.Header().Set("Vary", "Accept-Encoding") // Indicate that response varies based on Accept-Encoding
rw.Header().Set("Access-Control-Expose-Headers", "Content-Disposition, X-Response-Complete")
rw.Header().Set("Trailer", "X-Response-Complete")
rw.Header().Set("Transfer-Encoding", "chunked")
queryParams := r.URL.Query()
startTime, endTime, err := getExportQueryTimeRange(queryParams)
if err != nil {
if err := validateSpecForExport(&queryRangeRequest); err != nil {
render.Error(rw, err)
return
}
limit, err := getExportQueryLimit(queryParams)
if err != nil {
if err := validateAndApplyDefaultExportLimits(queryRangeRequest.CompositeQuery.Queries); err != nil {
render.Error(rw, err)
return
}
format, err := getExportQueryFormat(queryParams)
if err != nil {
render.Error(rw, err)
return
}
// Set appropriate content type and filename
filename := fmt.Sprintf("data_exported_%s.%s", time.Now().Format("2006-01-02_150405"), format)
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
filterExpression := queryParams.Get("filter")
orderByExpression, err := getExportQueryOrderBy(queryParams)
if err != nil {
render.Error(rw, err)
return
}
columns := getExportQueryColumns(queryParams)
// Use default OrderBy if not specified
queryRangeRequest.UseDefaultOrderBy()
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
@@ -167,70 +90,134 @@ func (handler *handler) exportLogs(rw http.ResponseWriter, r *http.Request) {
return
}
queryRangeRequest := qbtypes.QueryRangeRequest{
Start: startTime,
End: endTime,
RequestType: qbtypes.RequestTypeRaw,
CompositeQuery: qbtypes.CompositeQuery{
Queries: []qbtypes.QueryEnvelope{
{
Type: qbtypes.QueryTypeBuilder,
Spec: nil,
},
},
},
}
setExportResponseHeaders(rw, format)
spec := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Name: "raw",
Filter: &qbtypes.Filter{
Expression: filterExpression,
},
Limit: limit,
Order: orderByExpression,
}
spec.SelectFields = columns
queryRangeRequest.CompositeQuery.Queries[0].Spec = spec
// This will signal Export module to stop sending data
doneChan := make(chan any)
defer close(doneChan)
rowChan, errChan := handler.module.ExportRawData(r.Context(), orgID, &queryRangeRequest, doneChan)
var isComplete bool
isComplete, err := handler.executeExport(rowChan, errChan, format, rw)
if err != nil {
render.Error(rw, err)
return
}
rw.Header().Set("X-Response-Complete", strconv.FormatBool(isComplete))
}
// validateSpecForExport validates query specs
func validateSpecForExport(req *qbtypes.QueryRangeRequest) error {
queries := req.CompositeQuery.Queries
// If the trace operator query is not present, and there are multiple queries, return an error
if req.TraceOperatorQueryIndex() == -1 && len(queries) > 1 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "multiple queries not allowed without a trace operator query")
}
for idx := range queries {
switch spec := queries[idx].Spec.(type) {
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
qbtypes.QueryBuilderTraceOperator:
// Supported spec types
default:
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported query at index %d type: %T", idx, spec)
}
}
err := req.Validate(qbtypes.WithSkipLimitValidation())
if err != nil {
return err
}
return nil
}
func validateAndApplyDefaultExportLimits(queries []qbtypes.QueryEnvelope) error {
for idx := range queries {
limit := queries[idx].GetLimit()
if limit == 0 {
limit = DefaultExportRowCountLimit
} else if limit < 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "limit must be positive")
} else if limit > MaxExportRowCountLimit {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "limit cannot be more than %d", MaxExportRowCountLimit)
}
queries[idx].SetLimit(limit)
}
return nil
}
// buildQueryEnvelope creates a QueryEnvelope with a QueryBuilderQuery for the given aggregation type.
func buildQueryEnvelope[T any](signal telemetrytypes.Signal, filter *qbtypes.Filter, limit int, order []qbtypes.OrderBy, selectFields []telemetrytypes.TelemetryFieldKey) qbtypes.QueryEnvelope {
return qbtypes.QueryEnvelope{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[T]{
Signal: signal,
Filter: filter,
Limit: limit,
Order: order,
SelectFields: selectFields,
},
}
}
// buildQueryRangeRequest builds a QueryRangeRequest from already-bound and validated GET query params.
func buildQueryRangeRequest(params *exporttypes.ExportRawDataQueryParams) qbtypes.QueryRangeRequest {
var query qbtypes.QueryEnvelope
switch params.Signal {
case telemetrytypes.SignalLogs:
query = buildQueryEnvelope[qbtypes.LogAggregation](params.Signal, &params.Filter, params.Limit, params.Order, params.SelectFields)
case telemetrytypes.SignalTraces:
query = buildQueryEnvelope[qbtypes.TraceAggregation](params.Signal, &params.Filter, params.Limit, params.Order, params.SelectFields)
}
return qbtypes.QueryRangeRequest{
Start: params.Start,
End: params.End,
RequestType: qbtypes.RequestTypeRaw,
CompositeQuery: qbtypes.CompositeQuery{
Queries: []qbtypes.QueryEnvelope{query},
},
}
}
// setExportResponseHeaders sets common HTTP headers for export responses.
func setExportResponseHeaders(rw http.ResponseWriter, format string) {
rw.Header().Set("Cache-Control", "no-cache")
rw.Header().Set("Vary", "Accept-Encoding")
rw.Header().Set("Access-Control-Expose-Headers", "Content-Disposition, X-Response-Complete")
rw.Header().Set("Trailer", "X-Response-Complete")
rw.Header().Set("Transfer-Encoding", "chunked")
filename := fmt.Sprintf("data_exported_%s.%s", time.Now().Format("2006-01-02_150405"), format)
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
}
// executeExport streams data from rowChan to the response writer in the specified format.
func (handler *handler) executeExport(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, format string, rw http.ResponseWriter) (bool, error) {
switch format {
case "csv", "":
rw.Header().Set("Content-Type", "text/csv")
csvWriter := csv.NewWriter(rw)
isComplete, err = handler.exportLogsCSV(rowChan, errChan, csvWriter)
isComplete, err := handler.exportRawDataCSV(rowChan, errChan, csvWriter)
if err != nil {
render.Error(rw, err)
return
return false, err
}
csvWriter.Flush()
return isComplete, nil
case "jsonl":
rw.Header().Set("Content-Type", "application/x-ndjson")
isComplete, err = handler.exportLogsJSONL(rowChan, errChan, rw)
if err != nil {
render.Error(rw, err)
return
}
return handler.exportRawDataJSONL(rowChan, errChan, rw)
default:
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl"))
return
return false, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl")
}
rw.Header().Set("X-Response-Complete", strconv.FormatBool(isComplete))
}
func (handler *handler) exportLogsCSV(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, csvWriter *csv.Writer) (bool, error) {
var header []string
// exportRawDataCSV is a generic CSV export function that works with any raw data (logs, traces, etc.)
func (handler *handler) exportRawDataCSV(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, csvWriter *csv.Writer) (bool, error) {
headerToIndexMapping := make(map[string]int, len(header))
var header []string
headerToIndexMapping := make(map[string]int)
totalBytes := uint64(0)
for {
@@ -268,8 +255,8 @@ func (handler *handler) exportLogsCSV(rowChan <-chan *qbtypes.RawRow, errChan <-
}
}
func (handler *handler) exportLogsJSONL(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, writer io.Writer) (bool, error) {
// exportRawDataJSONL is a generic JSONL export function that works with any raw data (logs, traces, etc.)
func (handler *handler) exportRawDataJSONL(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, writer io.Writer) (bool, error) {
totalBytes := uint64(0)
for {
select {
@@ -277,9 +264,11 @@ func (handler *handler) exportLogsJSONL(rowChan <-chan *qbtypes.RawRow, errChan
if !ok {
return true, nil
}
// Handle JSON format (JSONL - one object per line)
jsonBytes, _ := json.Marshal(row.Data)
totalBytes += uint64(len(jsonBytes)) + 1 // +1 for newline
jsonBytes, err := json.Marshal(row.Data)
if err != nil {
return false, errors.NewUnexpectedf(errors.CodeInternal, "error marshaling JSON: %s", err)
}
totalBytes += uint64(len(jsonBytes)) + 1
if _, err := writer.Write(jsonBytes); err != nil {
return false, errors.NewUnexpectedf(errors.CodeInternal, "error writing JSON: %s", err)
@@ -299,74 +288,33 @@ func (handler *handler) exportLogsJSONL(rowChan <-chan *qbtypes.RawRow, errChan
}
}
func getExportQuerySource(queryParams url.Values) (string, error) {
switch queryParams.Get("source") {
case "logs", "":
return "logs", nil
case "metrics":
return "metrics", errors.NewInvalidInputf(errors.CodeInvalidInput, "metrics export not yet supported")
case "traces":
return "traces", errors.NewInvalidInputf(errors.CodeInvalidInput, "traces export not yet supported")
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid source: must be logs, metrics or traces")
}
}
func getExportQueryFormat(queryParams url.Values) (string, error) {
switch queryParams.Get("format") {
case "csv", "":
return "csv", nil
case "jsonl":
return "jsonl", nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl")
}
}
func getExportQueryLimit(queryParams url.Values) (int, error) {
limitStr := queryParams.Get("limit")
if limitStr == "" {
return DefaultExportRowCountLimit, nil
} else {
limit, err := strconv.Atoi(limitStr)
if err != nil {
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid limit format: %s", err.Error())
}
if limit <= 0 {
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "limit must be positive")
}
if limit > MaxExportRowCountLimit {
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "limit cannot be more than %d", MaxExportRowCountLimit)
}
return limit, nil
}
}
func getExportQueryTimeRange(queryParams url.Values) (uint64, uint64, error) {
startTimeStr := queryParams.Get("start")
endTimeStr := queryParams.Get("end")
if startTimeStr == "" || endTimeStr == "" {
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "start and end time are required")
}
startTime, err := strconv.ParseUint(startTimeStr, 10, 64)
if err != nil {
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid start time format: %s", err.Error())
}
endTime, err := strconv.ParseUint(endTimeStr, 10, 64)
if err != nil {
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid end time format: %s", err.Error())
}
return startTime, endTime, nil
}
// priorityColumns defines the columns that should appear first in the CSV output, in order.
var priorityColumns = []string{"timestamp", "id"}
func constructCSVHeaderFromQueryResponse(data map[string]any) []string {
header := make([]string, 0, len(data))
for key := range data {
header = append(header, key)
}
// This is to ensure CSV output is consistent across multiple queries
slices.SortFunc(header, func(a, b string) int {
ai, bi := slices.Index(priorityColumns, a), slices.Index(priorityColumns, b)
switch {
case ai != -1 && bi != -1:
return ai - bi
case ai != -1:
return -1
case bi != -1:
return 1
default:
if a < b {
return -1
} else if a > b {
return 1
}
return 0
}
})
return header
}
@@ -427,9 +375,12 @@ func constructCSVRecordFromQueryResponse(data map[string]any, headerToIndexMappi
valueStr = v.String()
default:
// For all other complex types (maps, structs, etc.)
jsonBytes, _ := json.Marshal(v)
valueStr = string(jsonBytes)
jsonBytes, err := json.Marshal(v)
if err != nil {
valueStr = fmt.Sprintf("%v", v)
} else {
valueStr = string(jsonBytes)
}
}
record[index] = sanitizeForCSV(valueStr)
@@ -438,26 +389,6 @@ func constructCSVRecordFromQueryResponse(data map[string]any, headerToIndexMappi
return record
}
// getExportQueryColumns parses the "columns" query parameters and returns a slice of TelemetryFieldKey structs.
// Each column should be a valid telemetry field key in the format "context.field:type" or "context.field" or "field"
func getExportQueryColumns(queryParams url.Values) []telemetrytypes.TelemetryFieldKey {
columnParams := queryParams["columns"]
columns := make([]telemetrytypes.TelemetryFieldKey, 0, len(columnParams))
for _, columnStr := range columnParams {
// Skip empty strings
columnStr = strings.TrimSpace(columnStr)
if columnStr == "" {
continue
}
columns = append(columns, telemetrytypes.GetFieldKeyFromKeyText(columnStr))
}
return columns
}
func getsizeOfStringSlice(slice []string) uint64 {
var totalBytes uint64
for _, str := range slice {
@@ -465,52 +396,3 @@ func getsizeOfStringSlice(slice []string) uint64 {
}
return totalBytes
}
// getExportQueryOrderBy parses the "order_by" query parameters and returns a slice of OrderBy structs.
// Each "order_by" parameter should be in the format "column:direction"
// Each "column" should be a valid telemetry field key in the format "context.field:type" or "context.field" or "field"
func getExportQueryOrderBy(queryParams url.Values) ([]qbtypes.OrderBy, error) {
orderByParam := queryParams.Get("order_by")
orderByParam = strings.TrimSpace(orderByParam)
if orderByParam == "" {
return telemetrylogs.DefaultLogsV2SortingOrder, nil
}
parts := strings.Split(orderByParam, ":")
if len(parts) != 2 && len(parts) != 3 {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order_by format: %s, should be <column>:<direction>", orderByParam)
}
column := strings.Join(parts[:len(parts)-1], ":")
direction := parts[len(parts)-1]
orderDirection, ok := qbtypes.OrderDirectionMap[direction]
if !ok {
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order_by direction: %s, should be one of %s, %s", direction, qbtypes.OrderDirectionAsc, qbtypes.OrderDirectionDesc)
}
orderByKey := telemetrytypes.GetFieldKeyFromKeyText(column)
orderBy := []qbtypes.OrderBy{
{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: orderByKey,
},
Direction: orderDirection,
},
}
// If we are ordering by the timestamp column, also order by the ID column
if orderByKey.Name == telemetrylogs.LogsV2TimestampColumn {
orderBy = append(orderBy, qbtypes.OrderBy{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
Direction: orderDirection,
})
}
return orderBy, nil
}

View File

@@ -2,162 +2,85 @@ package implrawdataexport
import (
"net/url"
"strconv"
"testing"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/http/binding"
"github.com/SigNoz/signoz/pkg/types/exporttypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
)
func TestGetExportQuerySource(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedSource string
expectedError bool
}{
{
name: "default logs source",
queryParams: url.Values{},
expectedSource: "logs",
expectedError: false,
},
{
name: "explicit logs source",
queryParams: url.Values{"source": {"logs"}},
expectedSource: "logs",
expectedError: false,
},
{
name: "metrics source - not supported",
queryParams: url.Values{"source": {"metrics"}},
expectedSource: "metrics",
expectedError: true,
},
{
name: "traces source - not supported",
queryParams: url.Values{"source": {"traces"}},
expectedSource: "traces",
expectedError: true,
},
{
name: "invalid source",
queryParams: url.Values{"source": {"invalid"}},
expectedSource: "",
expectedError: true,
},
}
func TestExportRawDataQueryParams_BindingDefaults(t *testing.T) {
var params exporttypes.ExportRawDataQueryParams
err := binding.Query.BindQuery(url.Values{}, &params)
assert.NoError(t, err)
assert.Equal(t, "csv", params.Format)
assert.Equal(t, DefaultExportRowCountLimit, params.Limit)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
source, err := getExportQuerySource(tt.queryParams)
assert.Equal(t, tt.expectedSource, source)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
func logQuery(limit int) qbtypes.QueryEnvelope {
return qbtypes.QueryEnvelope{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{Limit: limit},
}
}
func TestGetExportQueryFormat(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedFormat string
expectedError bool
}{
{
name: "default csv format",
queryParams: url.Values{},
expectedFormat: "csv",
expectedError: false,
},
{
name: "explicit csv format",
queryParams: url.Values{"format": {"csv"}},
expectedFormat: "csv",
expectedError: false,
},
{
name: "jsonl format",
queryParams: url.Values{"format": {"jsonl"}},
expectedFormat: "jsonl",
expectedError: false,
},
{
name: "invalid format",
queryParams: url.Values{"format": {"xml"}},
expectedFormat: "",
expectedError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
format, err := getExportQueryFormat(tt.queryParams)
assert.Equal(t, tt.expectedFormat, format)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
func traceQuery(limit int) qbtypes.QueryEnvelope {
return qbtypes.QueryEnvelope{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{Limit: limit},
}
}
func TestGetExportQueryLimit(t *testing.T) {
func traceOperatorQuery(limit int) qbtypes.QueryEnvelope {
return qbtypes.QueryEnvelope{
Type: qbtypes.QueryTypeTraceOperator,
Spec: qbtypes.QueryBuilderTraceOperator{Limit: limit, Expression: "A"},
}
}
func makeRequest(queries ...qbtypes.QueryEnvelope) qbtypes.QueryRangeRequest {
return qbtypes.QueryRangeRequest{
Start: 1000000000000,
End: 1000003600000,
RequestType: qbtypes.RequestTypeRaw,
CompositeQuery: qbtypes.CompositeQuery{Queries: queries},
}
}
func TestValidateSpecForExport(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedLimit int
req qbtypes.QueryRangeRequest
expectedError bool
}{
{
name: "default limit",
queryParams: url.Values{},
expectedLimit: DefaultExportRowCountLimit,
expectedError: false,
name: "single log query",
req: makeRequest(logQuery(0)),
},
{
name: "valid limit",
queryParams: url.Values{"limit": {"5000"}},
expectedLimit: 5000,
expectedError: false,
name: "single trace query",
req: makeRequest(traceQuery(0)),
},
{
name: "maximum limit",
queryParams: url.Values{"limit": {strconv.Itoa(MaxExportRowCountLimit)}},
expectedLimit: MaxExportRowCountLimit,
expectedError: false,
name: "trace operator alone",
req: makeRequest(traceOperatorQuery(0)),
},
{
name: "limit exceeds maximum",
queryParams: url.Values{"limit": {"100000"}},
expectedLimit: 0,
name: "multiple queries without trace operator",
req: makeRequest(logQuery(0), traceQuery(0)),
expectedError: true,
},
{
name: "invalid limit format",
queryParams: url.Values{"limit": {"invalid"}},
expectedLimit: 0,
expectedError: true,
},
{
name: "negative limit",
queryParams: url.Values{"limit": {"-100"}},
expectedLimit: 0,
name: "unsupported query type",
req: makeRequest(qbtypes.QueryEnvelope{Type: qbtypes.QueryTypeBuilder, Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{}}),
expectedError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
limit, err := getExportQueryLimit(tt.queryParams)
assert.Equal(t, tt.expectedLimit, limit)
err := validateSpecForExport(&tt.req)
if tt.expectedError {
assert.Error(t, err)
} else {
@@ -167,352 +90,69 @@ func TestGetExportQueryLimit(t *testing.T) {
}
}
func TestGetExportQueryTimeRange(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedStartTime uint64
expectedEndTime uint64
expectedError bool
}{
{
name: "valid time range",
queryParams: url.Values{
"start": {"1640995200"},
"end": {"1641081600"},
},
expectedStartTime: 1640995200,
expectedEndTime: 1641081600,
expectedError: false,
},
{
name: "missing start time",
queryParams: url.Values{"end": {"1641081600"}},
expectedError: true,
},
{
name: "missing end time",
queryParams: url.Values{"start": {"1640995200"}},
expectedError: true,
},
{
name: "missing both times",
queryParams: url.Values{},
expectedError: true,
},
{
name: "invalid start time format",
queryParams: url.Values{
"start": {"invalid"},
"end": {"1641081600"},
},
expectedError: true,
},
{
name: "invalid end time format",
queryParams: url.Values{
"start": {"1640995200"},
"end": {"invalid"},
},
expectedError: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
startTime, endTime, err := getExportQueryTimeRange(tt.queryParams)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, tt.expectedStartTime, startTime)
assert.Equal(t, tt.expectedEndTime, endTime)
}
})
}
}
func TestGetExportQueryColumns(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedColumns []telemetrytypes.TelemetryFieldKey
}{
{
name: "no columns specified",
queryParams: url.Values{},
expectedColumns: []telemetrytypes.TelemetryFieldKey{},
},
{
name: "single column",
queryParams: url.Values{
"columns": {"timestamp"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
},
},
{
name: "multiple columns",
queryParams: url.Values{
"columns": {"timestamp", "message", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "message"},
{Name: "level"},
},
},
{
name: "empty column name (should be skipped)",
queryParams: url.Values{
"columns": {"timestamp", "", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "level"},
},
},
{
name: "whitespace column name (should be skipped)",
queryParams: url.Values{
"columns": {"timestamp", " ", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "level"},
},
},
{
name: "valid column name with data type",
queryParams: url.Values{
"columns": {"timestamp", "attribute.user:string", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "user", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
{Name: "level"},
},
},
{
name: "valid column name with dot notation",
queryParams: url.Values{
"columns": {"timestamp", "attribute.user.string", "level"},
},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "user.string", FieldContext: telemetrytypes.FieldContextAttribute},
{Name: "level"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
columns := getExportQueryColumns(tt.queryParams)
assert.Equal(t, len(tt.expectedColumns), len(columns))
for i, expectedCol := range tt.expectedColumns {
assert.Equal(t, expectedCol, columns[i])
}
})
}
}
func TestGetExportQueryOrderBy(t *testing.T) {
func TestValidateAndApplyDefaultExportLimits(t *testing.T) {
tests := []struct {
name string
queryParams url.Values
expectedOrder []qbtypes.OrderBy
queries []qbtypes.QueryEnvelope
expectedError bool
checkQueries func(t *testing.T, queries []qbtypes.QueryEnvelope)
}{
{
name: "no order specified",
queryParams: url.Values{},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
name: "single log query, zero limit gets default",
queries: makeRequest(logQuery(0)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
},
expectedError: false,
},
{
name: "single order error, direction not specified",
queryParams: url.Values{
"order_by": {"timestamp"},
name: "single log query, valid limit kept",
queries: makeRequest(logQuery(1000)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, 1000, q[0].GetLimit())
},
expectedOrder: nil,
},
{
name: "single log query, max limit kept",
queries: makeRequest(logQuery(MaxExportRowCountLimit)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, MaxExportRowCountLimit, q[0].GetLimit())
},
},
{
name: "single log query, limit exceeds max",
queries: makeRequest(logQuery(MaxExportRowCountLimit + 1)).CompositeQuery.Queries,
expectedError: true,
},
{
name: "single order no error",
queryParams: url.Values{
"order_by": {"timestamp:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "multiple orders",
queryParams: url.Values{
"order_by": {"timestamp:asc", "body:desc", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "empty order name (should be skipped)",
queryParams: url.Values{
"order_by": {"timestamp:asc", "", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "whitespace order name (should be skipped)",
queryParams: url.Values{
"order_by": {"timestamp:asc", " ", "id:asc"},
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2TimestampColumn,
},
},
},
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: telemetrylogs.LogsV2IDColumn,
},
},
},
},
expectedError: false,
},
{
name: "invalid order name (should error out)",
queryParams: url.Values{
"order_by": {"attributes.user:", "id:asc"},
},
expectedOrder: nil,
name: "single log query, negative limit",
queries: makeRequest(logQuery(-1)).CompositeQuery.Queries,
expectedError: true,
},
{
name: "valid order name (should be included)",
queryParams: url.Values{
"order_by": {"attribute.user:string:desc", "id:asc"},
name: "single trace query, zero limit gets default",
queries: makeRequest(traceQuery(0)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "user",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
},
expectedError: false,
},
{
name: "valid order name (should be included)",
queryParams: url.Values{
"order_by": {"attribute.user.string:desc", "id:asc"},
name: "trace operator alone, zero limit gets default",
queries: makeRequest(traceOperatorQuery(0)).CompositeQuery.Queries,
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
},
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "user.string",
FieldContext: telemetrytypes.FieldContextAttribute,
},
},
},
},
expectedError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
order, err := getExportQueryOrderBy(tt.queryParams)
err := validateAndApplyDefaultExportLimits(tt.queries)
if tt.expectedError {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, len(tt.expectedOrder), len(order))
for i, expectedOrd := range tt.expectedOrder {
assert.Equal(t, expectedOrd, order[i])
if tt.checkQueries != nil {
tt.checkQueries(t, tt.queries)
}
}
})
@@ -529,13 +169,8 @@ func TestConstructCSVHeaderFromQueryResponse(t *testing.T) {
header := constructCSVHeaderFromQueryResponse(data)
// Since map iteration order is not guaranteed, check that all expected keys are present
expectedKeys := []string{"timestamp", "message", "level", "id"}
assert.Equal(t, len(expectedKeys), len(header))
for _, key := range expectedKeys {
assert.Contains(t, header, key)
}
// Priority columns come first in order, then the rest alphabetically.
assert.Equal(t, []string{"timestamp", "id", "level", "message"}, header)
}
func TestConstructCSVRecordFromQueryResponse(t *testing.T) {

View File

@@ -23,8 +23,18 @@ func NewModule(querier querier.Querier) rawdataexport.Module {
func (m *Module) ExportRawData(ctx context.Context, orgID valuer.UUID, rangeRequest *qbtypes.QueryRangeRequest, doneChan chan any) (chan *qbtypes.RawRow, chan error) {
spec := rangeRequest.CompositeQuery.Queries[0].Spec.(qbtypes.QueryBuilderQuery[qbtypes.LogAggregation])
rowCountLimit := spec.Limit
traceOperatorQueryIndex := rangeRequest.TraceOperatorQueryIndex()
queries := rangeRequest.CompositeQuery.Queries
// If the trace operator query is present, mark the queries other than trace operator as disabled
if traceOperatorQueryIndex > -1 {
for idx := range len(queries) {
if idx != traceOperatorQueryIndex {
queries[idx].SetDisabled(true)
}
}
}
rowChan := make(chan *qbtypes.RawRow, 1)
errChan := make(chan error, 1)
@@ -38,52 +48,62 @@ func (m *Module) ExportRawData(ctx context.Context, orgID valuer.UUID, rangeRequ
defer close(errChan)
defer close(rowChan)
rowCount := 0
for rowCount < rowCountLimit {
spec.Limit = min(ChunkSize, rowCountLimit-rowCount)
spec.Offset = rowCount
rangeRequest.CompositeQuery.Queries[0].Spec = spec
response, err := m.querier.QueryRange(contextWithTimeout, orgID, rangeRequest)
if err != nil {
errChan <- err
return
}
newRowsCount := 0
for _, result := range response.Data.Results {
resultData, ok := result.(*qbtypes.RawData)
if !ok {
errChan <- errors.NewInternalf(errors.CodeInternal, "expected RawData, got %T", result)
return
}
newRowsCount += len(resultData.Rows)
for _, row := range resultData.Rows {
select {
case rowChan <- row:
case <-doneChan:
return
case <-ctx.Done():
errChan <- ctx.Err()
return
}
}
}
// Break if we did not receive any new rows
if newRowsCount == 0 {
return
}
rowCount += newRowsCount
if traceOperatorQueryIndex > -1 {
// If the trace operator query is present, we need to export the data for the trace operator query only
exportRawDataForSingleQuery(m.querier, contextWithTimeout, orgID, rangeRequest, rowChan, errChan, doneChan, traceOperatorQueryIndex)
} else {
// If the trace operator query is not present, we need to export the data for the first query only
exportRawDataForSingleQuery(m.querier, contextWithTimeout, orgID, rangeRequest, rowChan, errChan, doneChan, 0)
}
}()
return rowChan, errChan
}
func exportRawDataForSingleQuery(querier querier.Querier, ctx context.Context, orgID valuer.UUID, rangeRequest *qbtypes.QueryRangeRequest, rowChan chan *qbtypes.RawRow, errChan chan error, doneChan chan any, queryIndex int) {
queries := rangeRequest.CompositeQuery.Queries
rowCountLimit := queries[queryIndex].GetLimit()
rowCount := 0
for rowCount < rowCountLimit {
chunkSize := min(ChunkSize, rowCountLimit-rowCount)
queries[queryIndex].SetLimit(chunkSize)
queries[queryIndex].SetOffset(rowCount)
response, err := querier.QueryRange(ctx, orgID, rangeRequest)
if err != nil {
errChan <- err
return
}
newRowsCount := 0
for _, result := range response.Data.Results {
resultData, ok := result.(*qbtypes.RawData)
if !ok {
errChan <- errors.NewInternalf(errors.CodeInternal, "expected RawData, got %T", result)
return
}
newRowsCount += len(resultData.Rows)
for _, row := range resultData.Rows {
select {
case rowChan <- row:
case <-doneChan:
return
case <-ctx.Done():
errChan <- ctx.Err()
return
}
}
}
rowCount += newRowsCount
// Stop if we received fewer rows than requested — no more data available
if newRowsCount < chunkSize {
return
}
}
}

View File

@@ -572,9 +572,6 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *middleware.AuthZ) {
aH.LicensingAPI.Activate(rw, req)
})).Methods(http.MethodGet)
// Export
router.HandleFunc("/api/v1/export_raw_data", am.ViewAccess(aH.Signoz.Handlers.RawDataExport.ExportRawData)).Methods(http.MethodGet)
router.HandleFunc("/api/v1/span_percentile", am.ViewAccess(aH.Signoz.Handlers.SpanPercentile.GetSpanPercentileDetails)).Methods(http.MethodPost)
// Query Filter Analyzer api used to extract metric names and grouping columns from a query

View File

@@ -22,6 +22,7 @@ import (
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/preference"
"github.com/SigNoz/signoz/pkg/modules/promote"
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
"github.com/SigNoz/signoz/pkg/modules/session"
"github.com/SigNoz/signoz/pkg/modules/user"
"github.com/SigNoz/signoz/pkg/querier"
@@ -57,6 +58,7 @@ func NewOpenAPI(ctx context.Context, instrumentation instrumentation.Instrumenta
struct{ gateway.Handler }{},
struct{ fields.Handler }{},
struct{ authz.Handler }{},
struct{ rawdataexport.Handler }{},
struct{ zeus.Handler }{},
struct{ querier.Handler }{},
).New(ctx, instrumentation.ToProviderSettings(), apiserver.Config{})

View File

@@ -253,6 +253,7 @@ func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.Au
handlers.GatewayHandler,
handlers.Fields,
handlers.AuthzHandler,
handlers.RawDataExport,
handlers.ZeusHandler,
handlers.QuerierHandler,
),

View File

@@ -1,643 +0,0 @@
// NOTE:
// - When Account keyword is used in struct names, it refers cloud integration account. CloudIntegration refers to DB schema.
// - When Account Config keyword is used in struct names, it refers to configuration for cloud integration accounts
// - When Service keyword is used in struct names, it refers to cloud integration service. CloudIntegrationService refers to DB schema.
// where `service` is services provided by each cloud provider like AWS S3, Azure BlobStorage etc.
// - When Service Config keyword is used in struct names, it refers to configuration for cloud integration services
package cloudintegrationtypes
import (
"database/sql/driver"
"encoding/json"
"strings"
"time"
"github.com/uptrace/bun"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/valuer"
)
// CloudProviderType type alias
type CloudProviderType struct{ valuer.String }
var (
CloudProviderTypeAWS = valuer.NewString("aws")
CloudProviderTypeAzure = valuer.NewString("azure")
)
var ErrCodeCloudProviderInvalidInput = errors.MustNewCode("invalid_cloud_provider")
// NewCloudProvider returns a new CloudProviderType from a string. It validates the input and returns an error if the input is not valid.
func NewCloudProvider(provider string) (CloudProviderType, error) {
switch provider {
case CloudProviderTypeAWS.String():
return CloudProviderType{CloudProviderTypeAWS}, nil
case CloudProviderTypeAzure.String():
return CloudProviderType{CloudProviderTypeAzure}, nil
default:
return CloudProviderType{}, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider)
}
}
var (
AWSIntegrationUserEmail = valuer.MustNewEmail("aws-integration@signoz.io")
AzureIntegrationUserEmail = valuer.MustNewEmail("azure-integration@signoz.io")
)
// CloudIntegrationUserEmails is the list of valid emails for Cloud One Click integrations.
// This is used for validation and restrictions in different contexts, across codebase.
var CloudIntegrationUserEmails = []valuer.Email{
AWSIntegrationUserEmail,
AzureIntegrationUserEmail,
}
func IsCloudIntegrationDashboardUuid(dashboardUuid string) bool {
parts := strings.SplitN(dashboardUuid, "--", 4)
if len(parts) != 4 {
return false
}
return parts[0] == "cloud-integration"
}
// GetCloudIntegrationDashboardID returns the cloud provider from dashboard id, if it's a cloud integration dashboard id.
// throws an error if invalid format or invalid cloud provider is provided in the dashboard id.
func GetCloudProviderFromDashboardID(dashboardUuid string) (CloudProviderType, error) {
parts := strings.SplitN(dashboardUuid, "--", 4)
if len(parts) != 4 {
return CloudProviderType{}, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid dashboard uuid: %s", dashboardUuid)
}
providerStr := parts[1]
cloudProvider, err := NewCloudProvider(providerStr)
if err != nil {
return CloudProviderType{}, err
}
return cloudProvider, nil
}
// Generic utility functions for JSON serialization/deserialization
// this is helpful to return right errors from a common place and avoid repeating the same code in multiple places.
// UnmarshalJSON is a generic function to unmarshal JSON data into any type
func UnmarshalJSON[T any](src []byte, target *T) error {
err := json.Unmarshal(src, target)
if err != nil {
return errors.WrapInternalf(
err, errors.CodeInternal, "couldn't deserialize JSON",
)
}
return nil
}
// MarshalJSON is a generic function to marshal any type to JSON
func MarshalJSON[T any](source *T) ([]byte, error) {
if source == nil {
return nil, errors.NewInternalf(errors.CodeInternal, "source is nil")
}
serialized, err := json.Marshal(source)
if err != nil {
return nil, errors.WrapInternalf(
err, errors.CodeInternal, "couldn't serialize to JSON",
)
}
return serialized, nil
}
// GettableConnectedAccountsList is the response for listing connected accounts for a cloud provider.
type GettableConnectedAccountsList struct {
Accounts []*Account `json:"accounts"`
}
// SigNozAgentConfig represents parameters required for agent deployment in cloud provider accounts
// these represent parameters passed during agent deployment, how they are passed might change for each cloud provider but the purpose is same.
type SigNozAgentConfig struct {
Region string `json:"region,omitempty"` // AWS-specific: The region in which SigNoz agent should be installed
IngestionUrl string `json:"ingestion_url"`
IngestionKey string `json:"ingestion_key"`
SigNozAPIUrl string `json:"signoz_api_url"`
SigNozAPIKey string `json:"signoz_api_key"`
Version string `json:"version,omitempty"`
}
// PostableConnectionArtifact represent request body for generating connection artifact API.
// Data is request body raw bytes since each cloud provider will have have different request body structure and generics hardly help in such cases.
// Artifact is a generic name for different types of connection methods like connection URL for AWS, connection command for Azure etc.
type PostableConnectionArtifact struct {
OrgID string
Data []byte // either PostableAWSConnectionUrl or PostableAzureConnectionCommand
}
// PostableAWSConnectionUrl is request body for AWS connection artifact API
type PostableAWSConnectionUrl struct {
AgentConfig *SigNozAgentConfig `json:"agent_config"`
AccountConfig *AWSAccountConfig `json:"account_config"`
}
// PostableAzureConnectionCommand is request body for Azure connection artifact API
type PostableAzureConnectionCommand struct {
AgentConfig *SigNozAgentConfig `json:"agent_config"`
AccountConfig *AzureAccountConfig `json:"account_config"`
}
// GettableAzureConnectionArtifact is Azure specific connection artifact which contains connection commands for agent deployment
type GettableAzureConnectionArtifact struct {
AzureShellConnectionCommand string `json:"az_shell_connection_command"`
AzureCliConnectionCommand string `json:"az_cli_connection_command"`
}
// GettableAWSConnectionUrl is AWS specific connection artifact which contains connection url for agent deployment
type GettableAWSConnectionUrl struct {
AccountId string `json:"account_id"`
ConnectionUrl string `json:"connection_url"`
}
// GettableAzureConnectionCommand is Azure specific connection artifact which contains connection commands for agent deployment
type GettableAzureConnectionCommand struct {
AccountId string `json:"account_id"`
AzureShellConnectionCommand string `json:"az_shell_connection_command"`
AzureCliConnectionCommand string `json:"az_cli_connection_command"`
}
// GettableAccountStatus is cloud integration account status response
type GettableAccountStatus struct {
Id string `json:"id"`
CloudAccountId *string `json:"cloud_account_id,omitempty"`
Status AccountStatus `json:"status"`
}
// PostableAgentCheckInPayload is request body for agent check-in API.
// This is used by agent to send heartbeat.
type PostableAgentCheckInPayload struct {
ID string `json:"account_id"`
AccountID string `json:"cloud_account_id"`
// Arbitrary cloud specific Agent data
Data map[string]any `json:"data,omitempty"`
OrgID string `json:"-"`
}
// AWSAgentIntegrationConfig is used by agent for deploying infra to send telemetry to SigNoz
type AWSAgentIntegrationConfig struct {
EnabledRegions []string `json:"enabled_regions"`
TelemetryCollectionStrategy *AWSCollectionStrategy `json:"telemetry,omitempty"`
}
// AzureAgentIntegrationConfig is used by agent for deploying infra to send telemetry to SigNoz
type AzureAgentIntegrationConfig struct {
DeploymentRegion string `json:"deployment_region"` // will not be changed once set
EnabledResourceGroups []string `json:"resource_groups"`
// TelemetryCollectionStrategy is map of service to telemetry config
TelemetryCollectionStrategy map[string]*AzureCollectionStrategy `json:"telemetry,omitempty"`
}
// GettableAgentCheckInRes is generic response from agent check-in API.
// AWSAgentIntegrationConfig and AzureAgentIntegrationConfig these configs are used by agent to deploy the infra and send telemetry to SigNoz
type GettableAgentCheckInRes[AgentConfigT any] struct {
AccountId string `json:"account_id"`
CloudAccountId string `json:"cloud_account_id"`
RemovedAt *time.Time `json:"removed_at"`
IntegrationConfig AgentConfigT `json:"integration_config"`
}
// UpdatableServiceConfig is generic
type UpdatableServiceConfig[ServiceConfigT any] struct {
CloudAccountId string `json:"cloud_account_id"`
Config ServiceConfigT `json:"config"`
}
// ServiceConfigTyped is a generic interface for cloud integration service's configuration
// this is generic interface to define helper functions for CloudIntegrationService.Config field.
type ServiceConfigTyped[definition Definition] interface {
Validate(def definition) error
IsMetricsEnabled() bool
IsLogsEnabled() bool
}
type AWSServiceConfig struct {
Logs *AWSServiceLogsConfig `json:"logs,omitempty"`
Metrics *AWSServiceMetricsConfig `json:"metrics,omitempty"`
}
type AWSServiceLogsConfig struct {
Enabled bool `json:"enabled"`
S3Buckets map[string][]string `json:"s3_buckets,omitempty"`
}
type AWSServiceMetricsConfig struct {
Enabled bool `json:"enabled"`
}
// IsMetricsEnabled returns true if metrics collection is configured and enabled
func (a *AWSServiceConfig) IsMetricsEnabled() bool {
return a.Metrics != nil && a.Metrics.Enabled
}
// IsLogsEnabled returns true if logs collection is configured and enabled
func (a *AWSServiceConfig) IsLogsEnabled() bool {
return a.Logs != nil && a.Logs.Enabled
}
type AzureServiceConfig struct {
Logs []*AzureServiceLogsConfig `json:"logs,omitempty"`
Metrics []*AzureServiceMetricsConfig `json:"metrics,omitempty"`
}
// AzureServiceLogsConfig is Azure specific service config for logs
type AzureServiceLogsConfig struct {
Enabled bool `json:"enabled"`
Name string `json:"name"`
}
// AzureServiceMetricsConfig is Azure specific service config for metrics
type AzureServiceMetricsConfig struct {
Enabled bool `json:"enabled"`
Name string `json:"name"`
}
// IsMetricsEnabled returns true if any metric is configured and enabled
func (a *AzureServiceConfig) IsMetricsEnabled() bool {
if a.Metrics == nil {
return false
}
for _, m := range a.Metrics {
if m.Enabled {
return true
}
}
return false
}
// IsLogsEnabled returns true if any log is configured and enabled
func (a *AzureServiceConfig) IsLogsEnabled() bool {
if a.Logs == nil {
return false
}
for _, l := range a.Logs {
if l.Enabled {
return true
}
}
return false
}
func (a *AWSServiceConfig) Validate(def *AWSDefinition) error {
if def.Id != S3Sync.String() && a.Logs != nil && a.Logs.S3Buckets != nil {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "s3 buckets can only be added to service-type[%s]", S3Sync)
} else if def.Id == S3Sync.String() && a.Logs != nil && a.Logs.S3Buckets != nil {
for region := range a.Logs.S3Buckets {
if _, found := ValidAWSRegions[region]; !found {
return errors.NewInvalidInputf(CodeInvalidCloudRegion, "invalid cloud region: %s", region)
}
}
}
return nil
}
func (a *AzureServiceConfig) Validate(def *AzureDefinition) error {
logsMap := make(map[string]bool)
metricsMap := make(map[string]bool)
if def.Strategy != nil && def.Strategy.Logs != nil {
for _, log := range def.Strategy.Logs {
logsMap[log.Name] = true
}
}
if def.Strategy != nil && def.Strategy.Metrics != nil {
for _, metric := range def.Strategy.Metrics {
metricsMap[metric.Name] = true
}
}
for _, log := range a.Logs {
if _, found := logsMap[log.Name]; !found {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid log name: %s", log.Name)
}
}
for _, metric := range a.Metrics {
if _, found := metricsMap[metric.Name]; !found {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid metric name: %s", metric.Name)
}
}
return nil
}
// UpdatableServiceConfigRes is response for UpdateServiceConfig API
// TODO: find a better way to name this
type UpdatableServiceConfigRes struct {
ServiceId string `json:"id"`
Config any `json:"config"`
}
// UpdatableAccountConfigTyped is a generic struct for updating cloud integration account config used in UpdateAccountConfig API
type UpdatableAccountConfigTyped[AccountConfigT any] struct {
Config *AccountConfigT `json:"config"`
}
type (
UpdatableAWSAccountConfig = UpdatableAccountConfigTyped[AWSAccountConfig]
UpdatableAzureAccountConfig = UpdatableAccountConfigTyped[AzureAccountConfig]
)
// AWSAccountConfig is the configuration for AWS cloud integration account
type AWSAccountConfig struct {
EnabledRegions []string `json:"regions"`
}
// AzureAccountConfig is the configuration for Azure cloud integration account
type AzureAccountConfig struct {
DeploymentRegion string `json:"deployment_region,omitempty"`
EnabledResourceGroups []string `json:"resource_groups,omitempty"`
}
// GettableServices is a generic struct for listing services of a cloud integration account used in ListServices API
type GettableServices[ServiceSummaryT any] struct {
Services []ServiceSummaryT `json:"services"`
}
type (
GettableAWSServices = GettableServices[AWSServiceSummary]
GettableAzureServices = GettableServices[AzureServiceSummary]
)
// GetServiceDetailsReq is a req struct for getting service definition details
type GetServiceDetailsReq struct {
OrgID valuer.UUID
ServiceId string
CloudAccountID *string
}
// ServiceSummary is a generic struct for service summary used in ListServices API
type ServiceSummary[ServiceConfigT any] struct {
DefinitionMetadata
Config *ServiceConfigT `json:"config"`
}
type (
AWSServiceSummary = ServiceSummary[AWSServiceConfig]
AzureServiceSummary = ServiceSummary[AzureServiceConfig]
)
// GettableServiceDetails is a generic struct for service details used in GetServiceDetails API
type GettableServiceDetails[DefinitionT any, ServiceConfigT any] struct {
Definition DefinitionT `json:",inline"`
Config ServiceConfigT `json:"config"`
ConnectionStatus *ServiceConnectionStatus `json:"status,omitempty"`
}
type (
GettableAWSServiceDetails = GettableServiceDetails[AWSDefinition, *AWSServiceConfig]
GettableAzureServiceDetails = GettableServiceDetails[AzureDefinition, *AzureServiceConfig]
)
// ServiceConnectionStatus represents integration connection status for a particular service
// this struct helps to check ingested data and determines connection status by whether data was ingested or not.
// this is composite struct for both metrics and logs
type ServiceConnectionStatus struct {
Logs []*SignalConnectionStatus `json:"logs"`
Metrics []*SignalConnectionStatus `json:"metrics"`
}
// SignalConnectionStatus represents connection status for a particular signal type (logs or metrics) for a service
// this struct is used in API responses for clients to show relevant information about the connection status.
type SignalConnectionStatus struct {
CategoryID string `json:"category"`
CategoryDisplayName string `json:"category_display_name"`
LastReceivedTsMillis int64 `json:"last_received_ts_ms"` // epoch milliseconds
LastReceivedFrom string `json:"last_received_from"` // resource identifier
}
// GettableCloudIntegrationConnectionParams is response for connection params API
type GettableCloudIntegrationConnectionParams struct {
IngestionUrl string `json:"ingestion_url,omitempty"`
IngestionKey string `json:"ingestion_key,omitempty"`
SigNozAPIUrl string `json:"signoz_api_url,omitempty"`
SigNozAPIKey string `json:"signoz_api_key,omitempty"`
}
// GettableIngestionKey is a struct for ingestion key returned from gateway
type GettableIngestionKey struct {
Name string `json:"name"`
Value string `json:"value"`
// other attributes from gateway response not included here since they are not being used.
}
// GettableIngestionKeysSearch is a struct for response of ingestion keys search API on gateway
type GettableIngestionKeysSearch struct {
Status string `json:"status"`
Data []GettableIngestionKey `json:"data"`
Error string `json:"error"`
}
// GettableCreateIngestionKey is a struct for response of create ingestion key API on gateway
type GettableCreateIngestionKey struct {
Status string `json:"status"`
Data GettableIngestionKey `json:"data"`
Error string `json:"error"`
}
// GettableDeployment is response struct for deployment details fetched from Zeus
type GettableDeployment struct {
Name string `json:"name"`
ClusterInfo struct {
Region struct {
DNS string `json:"dns"`
} `json:"region"`
} `json:"cluster"`
}
// --------------------------------------------------------------------------
// Cloud integration uses the cloud_integration table
// and cloud_integrations_service table
// --------------------------------------------------------------------------
type CloudIntegration struct {
bun.BaseModel `bun:"table:cloud_integration"`
types.Identifiable
types.TimeAuditable
Provider string `json:"provider" bun:"provider,type:text,unique:provider_id"`
Config *AccountConfig `json:"config" bun:"config,type:text"`
AccountID *string `json:"account_id" bun:"account_id,type:text"`
LastAgentReport *AgentReport `json:"last_agent_report" bun:"last_agent_report,type:text"`
RemovedAt *time.Time `json:"removed_at" bun:"removed_at,type:timestamp,nullzero"`
OrgID string `bun:"org_id,type:text,unique:provider_id"`
}
func (a *CloudIntegration) Status() AccountStatus {
status := AccountStatus{}
if a.LastAgentReport != nil {
lastHeartbeat := a.LastAgentReport.TimestampMillis
status.Integration.LastHeartbeatTsMillis = &lastHeartbeat
}
return status
}
func (a *CloudIntegration) Account() Account {
ca := Account{Id: a.ID.StringValue(), Status: a.Status()}
if a.AccountID != nil {
ca.CloudAccountId = *a.AccountID
}
if a.Config != nil {
ca.Config = *a.Config
} else {
ca.Config = DefaultAccountConfig()
}
return ca
}
type Account struct {
Id string `json:"id"`
CloudAccountId string `json:"cloud_account_id"`
Config AccountConfig `json:"config"`
Status AccountStatus `json:"status"`
}
type AccountStatus struct {
Integration AccountIntegrationStatus `json:"integration"`
}
type AccountIntegrationStatus struct {
LastHeartbeatTsMillis *int64 `json:"last_heartbeat_ts_ms"`
}
func DefaultAccountConfig() AccountConfig {
return AccountConfig{
EnabledRegions: []string{},
}
}
type AccountConfig struct {
EnabledRegions []string `json:"regions"`
}
// For serializing from db
func (c *AccountConfig) Scan(src any) error {
var data []byte
switch v := src.(type) {
case []byte:
data = v
case string:
data = []byte(v)
default:
return errors.NewInternalf(errors.CodeInternal, "tried to scan from %T instead of string or bytes", src)
}
return json.Unmarshal(data, c)
}
// For serializing to db
func (c *AccountConfig) Value() (driver.Value, error) {
if c == nil {
return nil, errors.NewInternalf(errors.CodeInternal, "cloud account config is nil")
}
serialized, err := json.Marshal(c)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't serialize cloud account config to JSON")
}
// Return as string instead of []byte to ensure PostgreSQL stores as text, not bytea
return string(serialized), nil
}
type AgentReport struct {
TimestampMillis int64 `json:"timestamp_millis"`
Data map[string]any `json:"data"`
}
// For serializing from db
func (r *AgentReport) Scan(src any) error {
var data []byte
switch v := src.(type) {
case []byte:
data = v
case string:
data = []byte(v)
default:
return errors.NewInternalf(errors.CodeInternal, "tried to scan from %T instead of string or bytes", src)
}
return json.Unmarshal(data, r)
}
// For serializing to db
func (r *AgentReport) Value() (driver.Value, error) {
if r == nil {
return nil, errors.NewInternalf(errors.CodeInternal, "agent report is nil")
}
serialized, err := json.Marshal(r)
if err != nil {
return nil, errors.WrapInternalf(
err, errors.CodeInternal, "couldn't serialize agent report to JSON",
)
}
// Return as string instead of []byte to ensure PostgreSQL stores as text, not bytea
return string(serialized), nil
}
type CloudIntegrationService struct {
bun.BaseModel `bun:"table:cloud_integration_service,alias:cis"`
types.Identifiable
types.TimeAuditable
Type string `bun:"type,type:text,notnull,unique:cloud_integration_id_type"`
Config CloudServiceConfig `bun:"config,type:text"`
CloudIntegrationID string `bun:"cloud_integration_id,type:text,notnull,unique:cloud_integration_id_type,references:cloud_integrations(id),on_delete:cascade"`
}
type CloudServiceLogsConfig struct {
Enabled bool `json:"enabled"`
S3Buckets map[string][]string `json:"s3_buckets,omitempty"`
}
type CloudServiceMetricsConfig struct {
Enabled bool `json:"enabled"`
}
type CloudServiceConfig struct {
Logs *CloudServiceLogsConfig `json:"logs,omitempty"`
Metrics *CloudServiceMetricsConfig `json:"metrics,omitempty"`
}
// For serializing from db
func (c *CloudServiceConfig) Scan(src any) error {
var data []byte
switch src := src.(type) {
case []byte:
data = src
case string:
data = []byte(src)
default:
return errors.NewInternalf(errors.CodeInternal, "tried to scan from %T instead of string or bytes", src)
}
return json.Unmarshal(data, c)
}
// For serializing to db
func (c *CloudServiceConfig) Value() (driver.Value, error) {
if c == nil {
return nil, errors.NewInternalf(errors.CodeInternal, "cloud service config is nil")
}
serialized, err := json.Marshal(c)
if err != nil {
return nil, errors.WrapInternalf(
err, errors.CodeInternal, "couldn't serialize cloud service config to JSON",
)
}
// Return as string instead of []byte to ensure PostgreSQL stores as text, not bytea
return string(serialized), nil
}

View File

@@ -1,103 +0,0 @@
package cloudintegrationtypes
import (
"github.com/SigNoz/signoz/pkg/errors"
)
var (
CodeInvalidCloudRegion = errors.MustNewCode("invalid_cloud_region")
CodeMismatchCloudProvider = errors.MustNewCode("cloud_provider_mismatch")
)
// List of all valid cloud regions on Amazon Web Services
var ValidAWSRegions = map[string]bool{
"af-south-1": true, // Africa (Cape Town).
"ap-east-1": true, // Asia Pacific (Hong Kong).
"ap-northeast-1": true, // Asia Pacific (Tokyo).
"ap-northeast-2": true, // Asia Pacific (Seoul).
"ap-northeast-3": true, // Asia Pacific (Osaka).
"ap-south-1": true, // Asia Pacific (Mumbai).
"ap-south-2": true, // Asia Pacific (Hyderabad).
"ap-southeast-1": true, // Asia Pacific (Singapore).
"ap-southeast-2": true, // Asia Pacific (Sydney).
"ap-southeast-3": true, // Asia Pacific (Jakarta).
"ap-southeast-4": true, // Asia Pacific (Melbourne).
"ca-central-1": true, // Canada (Central).
"ca-west-1": true, // Canada West (Calgary).
"eu-central-1": true, // Europe (Frankfurt).
"eu-central-2": true, // Europe (Zurich).
"eu-north-1": true, // Europe (Stockholm).
"eu-south-1": true, // Europe (Milan).
"eu-south-2": true, // Europe (Spain).
"eu-west-1": true, // Europe (Ireland).
"eu-west-2": true, // Europe (London).
"eu-west-3": true, // Europe (Paris).
"il-central-1": true, // Israel (Tel Aviv).
"me-central-1": true, // Middle East (UAE).
"me-south-1": true, // Middle East (Bahrain).
"sa-east-1": true, // South America (Sao Paulo).
"us-east-1": true, // US East (N. Virginia).
"us-east-2": true, // US East (Ohio).
"us-west-1": true, // US West (N. California).
"us-west-2": true, // US West (Oregon).
}
// List of all valid cloud regions for Microsoft Azure
var ValidAzureRegions = map[string]bool{
"australiacentral": true, // Australia Central
"australiacentral2": true, // Australia Central 2
"australiaeast": true, // Australia East
"australiasoutheast": true, // Australia Southeast
"austriaeast": true, // Austria East
"belgiumcentral": true, // Belgium Central
"brazilsouth": true, // Brazil South
"brazilsoutheast": true, // Brazil Southeast
"canadacentral": true, // Canada Central
"canadaeast": true, // Canada East
"centralindia": true, // Central India
"centralus": true, // Central US
"chilecentral": true, // Chile Central
"denmarkeast": true, // Denmark East
"eastasia": true, // East Asia
"eastus": true, // East US
"eastus2": true, // East US 2
"francecentral": true, // France Central
"francesouth": true, // France South
"germanynorth": true, // Germany North
"germanywestcentral": true, // Germany West Central
"indonesiacentral": true, // Indonesia Central
"israelcentral": true, // Israel Central
"italynorth": true, // Italy North
"japaneast": true, // Japan East
"japanwest": true, // Japan West
"koreacentral": true, // Korea Central
"koreasouth": true, // Korea South
"malaysiawest": true, // Malaysia West
"mexicocentral": true, // Mexico Central
"newzealandnorth": true, // New Zealand North
"northcentralus": true, // North Central US
"northeurope": true, // North Europe
"norwayeast": true, // Norway East
"norwaywest": true, // Norway West
"polandcentral": true, // Poland Central
"qatarcentral": true, // Qatar Central
"southafricanorth": true, // South Africa North
"southafricawest": true, // South Africa West
"southcentralus": true, // South Central US
"southindia": true, // South India
"southeastasia": true, // Southeast Asia
"spaincentral": true, // Spain Central
"swedencentral": true, // Sweden Central
"switzerlandnorth": true, // Switzerland North
"switzerlandwest": true, // Switzerland West
"uaecentral": true, // UAE Central
"uaenorth": true, // UAE North
"uksouth": true, // UK South
"ukwest": true, // UK West
"westcentralus": true, // West Central US
"westeurope": true, // West Europe
"westindia": true, // West India
"westus": true, // West US
"westus2": true, // West US 2
"westus3": true, // West US 3
}

View File

@@ -1,263 +0,0 @@
package cloudintegrationtypes
import (
"fmt"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
var S3Sync = valuer.NewString("s3sync")
// Generic interface for cloud service definition.
// This is implemented by AWSDefinition and AzureDefinition, which represent service definitions for AWS and Azure respectively.
// Generics work well so far because service definitions share a similar logic.
// We dont want to over-do generics as well, if the service definitions functionally diverge in the future consider breaking generics.
type Definition interface {
GetId() string
Validate() error
PopulateDashboardURLs(cloudProvider CloudProviderType, svcId string)
GetIngestionStatusCheck() *IngestionStatusCheck
GetAssets() Assets
}
// AWSDefinition represents AWS Service definition, which includes collection strategy, dashboards and meta info for integration
type AWSDefinition = ServiceDefinition[AWSCollectionStrategy]
// AzureDefinition represents Azure Service definition, which includes collection strategy, dashboards and meta info for integration
type AzureDefinition = ServiceDefinition[AzureCollectionStrategy]
// Making AWSDefinition and AzureDefinition satisfy Definition interface, so that they can be used in a generic way
var (
_ Definition = &AWSDefinition{}
_ Definition = &AzureDefinition{}
)
// ServiceDefinition represents generic struct for cloud service, regardless of the cloud provider.
// this struct must satify Definition interface.
// StrategyT is of either AWSCollectionStrategy or AzureCollectionStrategy, depending on the cloud provider.
type ServiceDefinition[StrategyT any] struct {
DefinitionMetadata
Overview string `json:"overview"` // markdown
Assets Assets `json:"assets"`
SupportedSignals SupportedSignals `json:"supported_signals"`
DataCollected DataCollected `json:"data_collected"`
IngestionStatusCheck *IngestionStatusCheck `json:"ingestion_status_check,omitempty"`
Strategy *StrategyT `json:"telemetry_collection_strategy"`
}
// Following methods are quite self explanatory, they are just to satisfy the Definition interface and provide some utility functions for service definitions.
func (def *ServiceDefinition[StrategyT]) GetId() string {
return def.Id
}
func (def *ServiceDefinition[StrategyT]) Validate() error {
seenDashboardIds := map[string]interface{}{}
if def.Strategy == nil {
return errors.NewInternalf(errors.CodeInternal, "telemetry_collection_strategy is required")
}
for _, dd := range def.Assets.Dashboards {
if _, seen := seenDashboardIds[dd.Id]; seen {
return errors.NewInternalf(errors.CodeInternal, "multiple dashboards found with id %s", dd.Id)
}
seenDashboardIds[dd.Id] = nil
}
return nil
}
func (def *ServiceDefinition[StrategyT]) PopulateDashboardURLs(cloudProvider CloudProviderType, svcId string) {
for i := range def.Assets.Dashboards {
dashboardId := def.Assets.Dashboards[i].Id
url := "/dashboard/" + GetCloudIntegrationDashboardID(cloudProvider, svcId, dashboardId)
def.Assets.Dashboards[i].Url = url
}
}
func (def *ServiceDefinition[StrategyT]) GetIngestionStatusCheck() *IngestionStatusCheck {
return def.IngestionStatusCheck
}
func (def *ServiceDefinition[StrategyT]) GetAssets() Assets {
return def.Assets
}
// DefinitionMetadata represents service definition metadata. This is useful for showing service overview
type DefinitionMetadata struct {
Id string `json:"id"`
Title string `json:"title"`
Icon string `json:"icon"`
}
// IngestionStatusCheckCategory represents a category of ingestion status check. Applies for both metrics and logs.
// A category can be "Overview" of metrics or "Enhanced" Metrics for AWS, and "Transaction" or "Capacity" metrics for Azure.
// Each category can have multiple checks (AND logic), if all checks pass,
// then we can be sure that data is being ingested for that category of the signal
type IngestionStatusCheckCategory struct {
Category string `json:"category"`
DisplayName string `json:"display_name"`
Checks []*IngestionStatusCheckAttribute `json:"checks"`
}
// IngestionStatusCheckAttribute represents a check or condition for ingestion status.
// Key can be metric name or part of log message
type IngestionStatusCheckAttribute struct {
Key string `json:"key"` // OPTIONAL search key (metric name or log message)
Attributes []*IngestionStatusCheckAttributeFilter `json:"attributes"`
}
// IngestionStatusCheck represents combined checks for metrics and logs for a service
type IngestionStatusCheck struct {
Metrics []*IngestionStatusCheckCategory `json:"metrics"`
Logs []*IngestionStatusCheckCategory `json:"logs"`
}
// IngestionStatusCheckAttributeFilter represents filter for a check, which can be used to filter specific log messages or metrics with specific attributes.
// For example, we can use it to filter logs with specific log level or metrics with specific dimensions.
type IngestionStatusCheckAttributeFilter struct {
Name string `json:"name"`
Operator string `json:"operator"`
Value string `json:"value"` // OPTIONAL
}
// Assets represents the collection of dashboards
type Assets struct {
Dashboards []Dashboard `json:"dashboards"`
}
// SupportedSignals for cloud provider's service
type SupportedSignals struct {
Logs bool `json:"logs"`
Metrics bool `json:"metrics"`
}
// DataCollected is curated static list of metrics and logs, this is shown as part of service overview
type DataCollected struct {
Logs []CollectedLogAttribute `json:"logs"`
Metrics []CollectedMetric `json:"metrics"`
}
// CollectedLogAttribute represents a log attribute that is present in all log entries for a service,
// this is shown as part of service overview
type CollectedLogAttribute struct {
Name string `json:"name"`
Path string `json:"path"`
Type string `json:"type"`
}
// CollectedMetric represents a metric that is collected for a service, this is shown as part of service overview
type CollectedMetric struct {
Name string `json:"name"`
Type string `json:"type"`
Unit string `json:"unit"`
Description string `json:"description"`
}
// AWSCollectionStrategy represents signal collection strategy for AWS services.
// this is AWS specific.
type AWSCollectionStrategy struct {
Metrics *AWSMetricsStrategy `json:"aws_metrics,omitempty"`
Logs *AWSLogsStrategy `json:"aws_logs,omitempty"`
S3Buckets map[string][]string `json:"s3_buckets,omitempty"` // Only available in S3 Sync Service Type in AWS
}
// AzureCollectionStrategy represents signal collection strategy for Azure services.
// this is Azure specific.
type AzureCollectionStrategy struct {
Metrics []*AzureMetricsStrategy `json:"azure_metrics,omitempty"`
Logs []*AzureLogsStrategy `json:"azure_logs,omitempty"`
}
// AWSMetricsStrategy represents metrics collection strategy for AWS services.
// this is AWS specific.
type AWSMetricsStrategy struct {
// to be used as https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudwatch-metricstream.html#cfn-cloudwatch-metricstream-includefilters
StreamFilters []struct {
// json tags here are in the shape expected by AWS API as detailed at
// https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-cloudwatch-metricstream-metricstreamfilter.html
Namespace string `json:"Namespace"`
MetricNames []string `json:"MetricNames,omitempty"`
} `json:"cloudwatch_metric_stream_filters"`
}
// AWSLogsStrategy represents logs collection strategy for AWS services.
// this is AWS specific.
type AWSLogsStrategy struct {
Subscriptions []struct {
// subscribe to all logs groups with specified prefix.
// eg: `/aws/rds/`
LogGroupNamePrefix string `json:"log_group_name_prefix"`
// https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html
// "" implies no filtering is required.
FilterPattern string `json:"filter_pattern"`
} `json:"cloudwatch_logs_subscriptions"`
}
// AzureMetricsStrategy represents metrics collection strategy for Azure services.
// this is Azure specific.
type AzureMetricsStrategy struct {
CategoryType string `json:"category_type"`
Name string `json:"name"`
}
// AzureLogsStrategy represents logs collection strategy for Azure services.
// this is Azure specific. Even though this is similar to AzureMetricsStrategy, keeping it separate for future flexibility and clarity.
type AzureLogsStrategy struct {
CategoryType string `json:"category_type"`
Name string `json:"name"`
}
// Dashboard represents a dashboard definition for cloud integration.
type Dashboard struct {
Id string `json:"id"`
Url string `json:"url"`
Title string `json:"title"`
Description string `json:"description"`
Image string `json:"image"`
Definition *dashboardtypes.StorableDashboardData `json:"definition,omitempty"`
}
// UTILS
// GetCloudIntegrationDashboardID returns the dashboard id for a cloud integration, given the cloud provider, service id, and dashboard id.
// This is used to generate unique dashboard ids for cloud integration, and also to parse the dashboard id to get the cloud provider and service id when needed.
func GetCloudIntegrationDashboardID(cloudProvider CloudProviderType, svcId, dashboardId string) string {
return fmt.Sprintf("cloud-integration--%s--%s--%s", cloudProvider, svcId, dashboardId)
}
// GetDashboardsFromAssets returns the list of dashboards for the cloud provider service from definition
func GetDashboardsFromAssets(
svcId string,
orgID valuer.UUID,
cloudProvider CloudProviderType,
createdAt *time.Time,
assets Assets,
) []*dashboardtypes.Dashboard {
dashboards := make([]*dashboardtypes.Dashboard, 0)
for _, d := range assets.Dashboards {
author := fmt.Sprintf("%s-integration", cloudProvider)
dashboards = append(dashboards, &dashboardtypes.Dashboard{
ID: GetCloudIntegrationDashboardID(cloudProvider, svcId, d.Id),
Locked: true,
OrgID: orgID,
Data: *d.Definition,
TimeAuditable: types.TimeAuditable{
CreatedAt: *createdAt,
UpdatedAt: *createdAt,
},
UserAuditable: types.UserAuditable{
CreatedBy: author,
UpdatedBy: author,
},
})
}
return dashboards
}

View File

@@ -1,57 +0,0 @@
package cloudintegrationtypes
import (
"context"
"time"
"github.com/SigNoz/signoz/pkg/query-service/model"
"github.com/SigNoz/signoz/pkg/types"
)
type CloudIntegrationAccountStore interface {
ListConnected(ctx context.Context, orgId string, provider string) ([]types.CloudIntegration, *model.ApiError)
Get(ctx context.Context, orgId string, provider string, id string) (*types.CloudIntegration, *model.ApiError)
GetConnectedCloudAccount(ctx context.Context, orgId string, provider string, accountID string) (*types.CloudIntegration, *model.ApiError)
// Insert an account or update it by (cloudProvider, id)
// for specified non-empty fields
Upsert(
ctx context.Context,
orgId string,
provider string,
id *string,
config *types.AccountConfig,
accountId *string,
agentReport *types.AgentReport,
removedAt *time.Time,
) (*types.CloudIntegration, *model.ApiError)
}
type CloudIntegrationServiceStore interface {
Get(
ctx context.Context,
orgID string,
cloudAccountId string,
serviceType string,
) (*types.CloudServiceConfig, *model.ApiError)
Upsert(
ctx context.Context,
orgID string,
cloudProvider string,
cloudAccountId string,
serviceId string,
config types.CloudServiceConfig,
) (*types.CloudServiceConfig, *model.ApiError)
GetAllForAccount(
ctx context.Context,
orgID string,
cloudAccountId string,
) (
configsBySvcId map[string]*types.CloudServiceConfig,
apiErr *model.ApiError,
)
}

View File

@@ -0,0 +1,120 @@
package exporttypes
import (
"strings"
"github.com/SigNoz/signoz/pkg/errors"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
// ExportRawDataQueryParams represents the query parameters for the export raw data endpoint
type ExportRawDataQueryParams struct {
ExportRawDataFormatQueryParam
// Signal specifies the type of data to export: "logs" or "traces"
Signal telemetrytypes.Signal `query:"signal" enum:"logs,traces" required:"true"`
// Source specifies the type of data to export: "logs" or "traces"
// Deprecated: Use Signal instead.
Source string `query:"source" deprecated:"true"`
// Start is the start time for the query (Unix timestamp in nanoseconds)
Start uint64 `query:"start"`
// End is the end time for the query (Unix timestamp in nanoseconds)
End uint64 `query:"end"`
// Limit specifies the maximum number of rows to export
Limit int `query:"limit,default=10000" default:"10000" minimum:"1" maximum:"50000"`
// Filter is a filter expression to apply to the query
FilterString string `query:"filter" deprecated:"true"`
Filter qbtypes.Filter `query:"filterExpression"`
// Columns specifies the columns to include in the export
// Format: ["context.field:type", "context.field", "field"]
Columns []string `query:"columns" deprecated:"true"`
// SelectFields specifies the columns to include in the export
SelectFields []telemetrytypes.TelemetryFieldKey `query:"selectFields"`
// OrderBy specifies the sorting order
// Format: "column:direction" or "context.field:type:direction"
// Direction can be "asc" or "desc"
// ** Deprecated **
OrderBy string `query:"order_by" deprecated:"true"`
// order by keys and directions
Order []qbtypes.OrderBy `query:"order"`
}
type ExportRawDataFormatQueryParam struct {
// Format specifies the output format: "csv" or "jsonl"
Format string `query:"format,default=csv" default:"csv" enum:"csv,jsonl"`
}
func (p *ExportRawDataQueryParams) Normalize() {
if len(p.Order) == 0 && len(p.OrderBy) > 0 {
p.Order = parseExportQueryOrderBy(p.OrderBy)
}
if len(p.SelectFields) == 0 && len(p.Columns) != 0 {
p.SelectFields = parseExportQueryColumns(p.Columns)
}
if len(p.Filter.Expression) == 0 && len(p.FilterString) > 0 {
p.Filter = qbtypes.Filter{Expression: p.FilterString}
}
if p.Signal == telemetrytypes.SignalUnspecified && p.Source != "" {
p.Signal = telemetrytypes.Signal{String: valuer.NewString(p.Source)}
}
}
func (p *ExportRawDataQueryParams) Validate() error {
if p.Signal != telemetrytypes.SignalLogs && p.Signal != telemetrytypes.SignalTraces {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid signal %s", p.Signal).WithAdditional("Allowed values: [logs, traces]")
}
return nil
}
// parseExportQueryColumns converts bound column strings to TelemetryFieldKey structs.
// Each column should be in the format "context.field:type" or "context.field" or "field"
func parseExportQueryColumns(columnParams []string) []telemetrytypes.TelemetryFieldKey {
columns := make([]telemetrytypes.TelemetryFieldKey, 0, len(columnParams))
for _, columnStr := range columnParams {
columnStr = strings.TrimSpace(columnStr)
if columnStr == "" {
continue
}
columns = append(columns, telemetrytypes.GetFieldKeyFromKeyText(columnStr))
}
return columns
}
// parseExportQueryOrderBy converts a bound order_by string to an OrderBy slice.
// The string should be in the format "column:direction" and is assumed already validated.
func parseExportQueryOrderBy(orderByParam string) []qbtypes.OrderBy {
orderByParam = strings.TrimSpace(orderByParam)
if orderByParam == "" {
return []qbtypes.OrderBy{}
}
parts := strings.Split(orderByParam, ":")
column := strings.Join(parts[:len(parts)-1], ":")
direction := parts[len(parts)-1]
return []qbtypes.OrderBy{
{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.GetFieldKeyFromKeyText(column),
},
Direction: qbtypes.OrderDirectionMap[direction],
},
}
}

View File

@@ -0,0 +1,174 @@
package exporttypes
import (
"testing"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/stretchr/testify/assert"
)
func TestParseExportQueryColumns(t *testing.T) {
tests := []struct {
name string
input []string
expectedColumns []telemetrytypes.TelemetryFieldKey
}{
{
name: "empty input",
input: []string{},
expectedColumns: []telemetrytypes.TelemetryFieldKey{},
},
{
name: "single column",
input: []string{"timestamp"},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
},
},
{
name: "multiple columns",
input: []string{"timestamp", "message", "level"},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "message"},
{Name: "level"},
},
},
{
name: "empty entry is skipped",
input: []string{"timestamp", "", "level"},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "level"},
},
},
{
name: "whitespace-only entry is skipped",
input: []string{"timestamp", " ", "level"},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "timestamp"},
{Name: "level"},
},
},
{
name: "column with context and type",
input: []string{"attribute.user:string"},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "user", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
},
},
{
name: "column with context, dot-notation name",
input: []string{"attribute.user.string"},
expectedColumns: []telemetrytypes.TelemetryFieldKey{
{Name: "user.string", FieldContext: telemetrytypes.FieldContextAttribute},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
columns := parseExportQueryColumns(tt.input)
assert.Equal(t, len(tt.expectedColumns), len(columns))
for i, expected := range tt.expectedColumns {
assert.Equal(t, expected, columns[i])
}
})
}
}
func TestParseExportQueryOrderBy(t *testing.T) {
tests := []struct {
name string
input string
expectedOrder []qbtypes.OrderBy
}{
{
name: "empty string returns empty slice",
input: "",
expectedOrder: []qbtypes.OrderBy{},
},
{
name: "simple column asc",
input: "timestamp:asc",
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "timestamp"},
},
},
},
},
{
name: "simple column desc",
input: "timestamp:desc",
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "timestamp"},
},
},
},
},
{
name: "column with context and type qualifier",
input: "attribute.user:string:desc",
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "user",
FieldContext: telemetrytypes.FieldContextAttribute,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
},
},
{
name: "column with context, dot-notation name",
input: "attribute.user.string:desc",
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionDesc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "user.string",
FieldContext: telemetrytypes.FieldContextAttribute,
},
},
},
},
},
{
name: "resource with context and type",
input: "resource.service.name:string:asc",
expectedOrder: []qbtypes.OrderBy{
{
Direction: qbtypes.OrderDirectionAsc,
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "service.name",
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
order := parseExportQueryOrderBy(tt.input)
assert.Equal(t, len(tt.expectedOrder), len(order))
for i, expected := range tt.expectedOrder {
assert.Equal(t, expected, order[i])
}
})
}
}

View File

@@ -393,6 +393,59 @@ func (r *QueryRangeRequest) HasOrderSpecified() bool {
return false
}
// UseDefaultOrderBy applies a default order unless query has an explicit order provided.
func (r *QueryRangeRequest) UseDefaultOrderBy() {
queries := r.CompositeQuery.Queries
for idx := range queries {
switch queries[idx].Spec.(type) {
case QueryBuilderQuery[TraceAggregation],
QueryBuilderTraceOperator:
if len(queries[idx].GetOrder()) == 0 {
queries[idx].SetOrder(
[]OrderBy{
{
Key: OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: "timestamp",
Signal: telemetrytypes.SignalTraces,
FieldContext: telemetrytypes.FieldContextSpan,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
},
},
Direction: OrderDirectionDesc,
},
},
)
}
case QueryBuilderQuery[LogAggregation]:
if len(queries[idx].GetOrder()) == 0 {
queries[idx].SetOrder(
[]OrderBy{
{
Key: OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "timestamp",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeNumber},
},
Direction: OrderDirectionDesc,
},
{
Key: OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "id",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString},
},
Direction: OrderDirectionDesc,
},
},
)
}
}
}
}
func (r *QueryRangeRequest) FuncsForQuery(name string) []Function {
funcs := []Function{}
for _, query := range r.CompositeQuery.Queries {
@@ -437,6 +490,16 @@ func (r *QueryRangeRequest) IsAnomalyRequest() (*QueryBuilderQuery[MetricAggrega
return &q, hasAnomaly
}
func (r *QueryRangeRequest) TraceOperatorQueryIndex() int {
for idx, query := range r.CompositeQuery.Queries {
switch query.Spec.(type) {
case TraceOperatorType:
return idx
}
}
return -1
}
// We do not support fill gaps for these queries. Maybe support in future?
func (r *QueryRangeRequest) SkipFillGaps(name string) bool {
for _, query := range r.CompositeQuery.Queries {

View File

@@ -0,0 +1,379 @@
package querybuildertypesv5
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
// GetExpression returns the expression string.
func (q *QueryEnvelope) GetExpression() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Expression
case QueryBuilderFormula:
return spec.Expression
}
return ""
}
// GetReturnSpansFrom returns the return-spans-from value.
func (q *QueryEnvelope) GetReturnSpansFrom() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.ReturnSpansFrom
}
return ""
}
// GetSignal returns the signal.
func (q *QueryEnvelope) GetSignal() telemetrytypes.Signal {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Signal
case QueryBuilderQuery[LogAggregation]:
return spec.Signal
case QueryBuilderQuery[MetricAggregation]:
return spec.Signal
}
return telemetrytypes.SignalUnspecified
}
// GetSource returns the source.
func (q *QueryEnvelope) GetSource() telemetrytypes.Source {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Source
case QueryBuilderQuery[LogAggregation]:
return spec.Source
case QueryBuilderQuery[MetricAggregation]:
return spec.Source
}
return telemetrytypes.SourceUnspecified
}
// GetQuery returns the raw query string.
func (q *QueryEnvelope) GetQuery() string {
switch spec := q.Spec.(type) {
case PromQuery:
return spec.Query
case ClickHouseQuery:
return spec.Query
}
return ""
}
// GetStats returns the PromQL stats flag.
func (q *QueryEnvelope) GetStats() bool {
switch spec := q.Spec.(type) {
case PromQuery:
return spec.Stats
}
return false
}
// GetLeft returns the left query reference of a join.
func (q *QueryEnvelope) GetLeft() QueryRef {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.Left
}
return QueryRef{}
}
// GetRight returns the right query reference of a join.
func (q *QueryEnvelope) GetRight() QueryRef {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.Right
}
return QueryRef{}
}
// GetJoinType returns the join type.
func (q *QueryEnvelope) GetJoinType() JoinType {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.Type
}
return JoinType{}
}
// GetOn returns the join ON condition.
func (q *QueryEnvelope) GetOn() string {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
return spec.On
}
return ""
}
// GetQueryName returns the name of the spec.
func (q *QueryEnvelope) GetQueryName() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Name
case QueryBuilderQuery[TraceAggregation]:
return spec.Name
case QueryBuilderQuery[LogAggregation]:
return spec.Name
case QueryBuilderQuery[MetricAggregation]:
return spec.Name
case QueryBuilderFormula:
return spec.Name
case QueryBuilderJoin:
return spec.Name
case PromQuery:
return spec.Name
case ClickHouseQuery:
return spec.Name
}
return ""
}
// IsDisabled returns whether the spec is disabled.
func (q *QueryEnvelope) IsDisabled() bool {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Disabled
case QueryBuilderQuery[TraceAggregation]:
return spec.Disabled
case QueryBuilderQuery[LogAggregation]:
return spec.Disabled
case QueryBuilderQuery[MetricAggregation]:
return spec.Disabled
case QueryBuilderFormula:
return spec.Disabled
case QueryBuilderJoin:
return spec.Disabled
case PromQuery:
return spec.Disabled
case ClickHouseQuery:
return spec.Disabled
}
return false
}
// GetLimit returns the row limit.
func (q *QueryEnvelope) GetLimit() int {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Limit
case QueryBuilderQuery[TraceAggregation]:
return spec.Limit
case QueryBuilderQuery[LogAggregation]:
return spec.Limit
case QueryBuilderQuery[MetricAggregation]:
return spec.Limit
case QueryBuilderFormula:
return spec.Limit
case QueryBuilderJoin:
return spec.Limit
}
return 0
}
// GetOffset returns the row offset.
func (q *QueryEnvelope) GetOffset() int {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Offset
case QueryBuilderQuery[TraceAggregation]:
return spec.Offset
case QueryBuilderQuery[LogAggregation]:
return spec.Offset
case QueryBuilderQuery[MetricAggregation]:
return spec.Offset
}
return 0
}
// GetType returns the QueryType of the envelope.
func (q *QueryEnvelope) GetType() QueryType {
return q.Type
}
// GetOrder returns the order-by clauses.
func (q *QueryEnvelope) GetOrder() []OrderBy {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Order
case QueryBuilderQuery[TraceAggregation]:
return spec.Order
case QueryBuilderQuery[LogAggregation]:
return spec.Order
case QueryBuilderQuery[MetricAggregation]:
return spec.Order
case QueryBuilderFormula:
return spec.Order
case QueryBuilderJoin:
return spec.Order
}
return nil
}
// GetGroupBy returns the group-by keys.
func (q *QueryEnvelope) GetGroupBy() []GroupByKey {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.GroupBy
case QueryBuilderQuery[TraceAggregation]:
return spec.GroupBy
case QueryBuilderQuery[LogAggregation]:
return spec.GroupBy
case QueryBuilderQuery[MetricAggregation]:
return spec.GroupBy
case QueryBuilderJoin:
return spec.GroupBy
}
return nil
}
// GetFilter returns the filter.
func (q *QueryEnvelope) GetFilter() *Filter {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Filter
case QueryBuilderQuery[TraceAggregation]:
return spec.Filter
case QueryBuilderQuery[LogAggregation]:
return spec.Filter
case QueryBuilderQuery[MetricAggregation]:
return spec.Filter
case QueryBuilderJoin:
return spec.Filter
}
return nil
}
// GetHaving returns the having clause.
func (q *QueryEnvelope) GetHaving() *Having {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Having
case QueryBuilderQuery[TraceAggregation]:
return spec.Having
case QueryBuilderQuery[LogAggregation]:
return spec.Having
case QueryBuilderQuery[MetricAggregation]:
return spec.Having
case QueryBuilderFormula:
return spec.Having
case QueryBuilderJoin:
return spec.Having
}
return nil
}
// GetFunctions returns the post-processing functions.
func (q *QueryEnvelope) GetFunctions() []Function {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Functions
case QueryBuilderQuery[TraceAggregation]:
return spec.Functions
case QueryBuilderQuery[LogAggregation]:
return spec.Functions
case QueryBuilderQuery[MetricAggregation]:
return spec.Functions
case QueryBuilderFormula:
return spec.Functions
case QueryBuilderJoin:
return spec.Functions
}
return nil
}
// GetSelectFields returns the selected fields.
func (q *QueryEnvelope) GetSelectFields() []telemetrytypes.TelemetryFieldKey {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.SelectFields
case QueryBuilderQuery[TraceAggregation]:
return spec.SelectFields
case QueryBuilderQuery[LogAggregation]:
return spec.SelectFields
case QueryBuilderQuery[MetricAggregation]:
return spec.SelectFields
case QueryBuilderJoin:
return spec.SelectFields
}
return nil
}
// GetLegend returns the legend label.
func (q *QueryEnvelope) GetLegend() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Legend
case QueryBuilderQuery[TraceAggregation]:
return spec.Legend
case QueryBuilderQuery[LogAggregation]:
return spec.Legend
case QueryBuilderQuery[MetricAggregation]:
return spec.Legend
case QueryBuilderFormula:
return spec.Legend
case PromQuery:
return spec.Legend
case ClickHouseQuery:
return spec.Legend
}
return ""
}
// GetCursor returns the pagination cursor.
func (q *QueryEnvelope) GetCursor() string {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.Cursor
case QueryBuilderQuery[TraceAggregation]:
return spec.Cursor
case QueryBuilderQuery[LogAggregation]:
return spec.Cursor
case QueryBuilderQuery[MetricAggregation]:
return spec.Cursor
}
return ""
}
// GetStepInterval returns the step interval.
func (q *QueryEnvelope) GetStepInterval() Step {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
return spec.StepInterval
case QueryBuilderQuery[TraceAggregation]:
return spec.StepInterval
case QueryBuilderQuery[LogAggregation]:
return spec.StepInterval
case QueryBuilderQuery[MetricAggregation]:
return spec.StepInterval
case PromQuery:
return spec.Step
}
return Step{}
}
// GetSecondaryAggregations returns the secondary aggregations.
func (q *QueryEnvelope) GetSecondaryAggregations() []SecondaryAggregation {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.SecondaryAggregations
case QueryBuilderQuery[LogAggregation]:
return spec.SecondaryAggregations
case QueryBuilderQuery[MetricAggregation]:
return spec.SecondaryAggregations
case QueryBuilderJoin:
return spec.SecondaryAggregations
}
return nil
}
// GetLimitBy returns the limit-by configuration.
func (q *QueryEnvelope) GetLimitBy() *LimitBy {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.LimitBy
case QueryBuilderQuery[LogAggregation]:
return spec.LimitBy
case QueryBuilderQuery[MetricAggregation]:
return spec.LimitBy
}
return nil
}

View File

@@ -0,0 +1,452 @@
package querybuildertypesv5
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
// SetExpression sets the expression string of the spec, if applicable.
func (q *QueryEnvelope) SetExpression(expression string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Expression = expression
q.Spec = spec
case QueryBuilderFormula:
spec.Expression = expression
q.Spec = spec
}
}
// SetReturnSpansFrom sets the return-spans-from value, if applicable.
func (q *QueryEnvelope) SetReturnSpansFrom(returnSpansFrom string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.ReturnSpansFrom = returnSpansFrom
q.Spec = spec
}
}
// SetSignal sets the signal of the spec, if applicable.
func (q *QueryEnvelope) SetSignal(signal telemetrytypes.Signal) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.Signal = signal
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Signal = signal
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Signal = signal
q.Spec = spec
}
}
// SetSource sets the source of the spec, if applicable.
func (q *QueryEnvelope) SetSource(source telemetrytypes.Source) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.Source = source
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Source = source
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Source = source
q.Spec = spec
}
}
// SetQuery sets the raw query string of the spec, if applicable.
func (q *QueryEnvelope) SetQuery(query string) {
switch spec := q.Spec.(type) {
case PromQuery:
spec.Query = query
q.Spec = spec
case ClickHouseQuery:
spec.Query = query
q.Spec = spec
}
}
// SetStats sets the PromQL stats flag, if applicable.
func (q *QueryEnvelope) SetStats(stats bool) {
switch spec := q.Spec.(type) {
case PromQuery:
spec.Stats = stats
q.Spec = spec
}
}
// SetLeft sets the left query reference of a join, if applicable.
func (q *QueryEnvelope) SetLeft(left QueryRef) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.Left = left
q.Spec = spec
}
}
// SetRight sets the right query reference of a join, if applicable.
func (q *QueryEnvelope) SetRight(right QueryRef) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.Right = right
q.Spec = spec
}
}
// SetJoinType sets the join type, if applicable.
func (q *QueryEnvelope) SetJoinType(joinType JoinType) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.Type = joinType
q.Spec = spec
}
}
// SetOn sets the join ON condition, if applicable.
func (q *QueryEnvelope) SetOn(on string) {
switch spec := q.Spec.(type) {
case QueryBuilderJoin:
spec.On = on
q.Spec = spec
}
}
// SetQueryName sets the name of the spec, if applicable.
func (q *QueryEnvelope) SetQueryName(name string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Name = name
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Name = name
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Name = name
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Name = name
q.Spec = spec
case QueryBuilderFormula:
spec.Name = name
q.Spec = spec
case QueryBuilderJoin:
spec.Name = name
q.Spec = spec
case PromQuery:
spec.Name = name
q.Spec = spec
case ClickHouseQuery:
spec.Name = name
q.Spec = spec
}
}
// SetDisabled sets the disabled flag of the spec, if applicable.
func (q *QueryEnvelope) SetDisabled(disabled bool) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderFormula:
spec.Disabled = disabled
q.Spec = spec
case QueryBuilderJoin:
spec.Disabled = disabled
q.Spec = spec
case PromQuery:
spec.Disabled = disabled
q.Spec = spec
case ClickHouseQuery:
spec.Disabled = disabled
q.Spec = spec
}
}
// SetLimit sets the row limit of the spec, if applicable.
func (q *QueryEnvelope) SetLimit(limit int) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Limit = limit
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Limit = limit
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Limit = limit
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Limit = limit
q.Spec = spec
case QueryBuilderFormula:
spec.Limit = limit
q.Spec = spec
case QueryBuilderJoin:
spec.Limit = limit
q.Spec = spec
}
}
// SetOffset sets the row offset of the spec, if applicable.
func (q *QueryEnvelope) SetOffset(offset int) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Offset = offset
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Offset = offset
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Offset = offset
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Offset = offset
q.Spec = spec
}
}
// SetType sets the QueryType of the envelope.
func (q *QueryEnvelope) SetType(t QueryType) {
q.Type = t
}
// SetOrder sets the order-by clauses of the spec, if applicable.
func (q *QueryEnvelope) SetOrder(order []OrderBy) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Order = order
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Order = order
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Order = order
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Order = order
q.Spec = spec
case QueryBuilderFormula:
spec.Order = order
q.Spec = spec
case QueryBuilderJoin:
spec.Order = order
q.Spec = spec
}
}
// SetGroupBy sets the group-by keys of the spec, if applicable.
func (q *QueryEnvelope) SetGroupBy(groupBy []GroupByKey) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.GroupBy = groupBy
q.Spec = spec
case QueryBuilderJoin:
spec.GroupBy = groupBy
q.Spec = spec
}
}
// SetFilter sets the filter of the spec, if applicable.
func (q *QueryEnvelope) SetFilter(filter *Filter) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Filter = filter
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Filter = filter
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Filter = filter
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Filter = filter
q.Spec = spec
case QueryBuilderJoin:
spec.Filter = filter
q.Spec = spec
}
}
// SetHaving sets the having clause of the spec, if applicable.
func (q *QueryEnvelope) SetHaving(having *Having) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Having = having
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Having = having
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Having = having
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Having = having
q.Spec = spec
case QueryBuilderFormula:
spec.Having = having
q.Spec = spec
case QueryBuilderJoin:
spec.Having = having
q.Spec = spec
}
}
// SetFunctions sets the post-processing functions of the spec, if applicable.
func (q *QueryEnvelope) SetFunctions(functions []Function) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Functions = functions
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Functions = functions
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Functions = functions
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Functions = functions
q.Spec = spec
case QueryBuilderFormula:
spec.Functions = functions
q.Spec = spec
case QueryBuilderJoin:
spec.Functions = functions
q.Spec = spec
}
}
// SetSelectFields sets the selected fields of the spec, if applicable.
func (q *QueryEnvelope) SetSelectFields(fields []telemetrytypes.TelemetryFieldKey) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.SelectFields = fields
q.Spec = spec
case QueryBuilderJoin:
spec.SelectFields = fields
q.Spec = spec
}
}
// SetLegend sets the legend label of the spec, if applicable.
func (q *QueryEnvelope) SetLegend(legend string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Legend = legend
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Legend = legend
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Legend = legend
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Legend = legend
q.Spec = spec
case QueryBuilderFormula:
spec.Legend = legend
q.Spec = spec
case PromQuery:
spec.Legend = legend
q.Spec = spec
case ClickHouseQuery:
spec.Legend = legend
q.Spec = spec
}
}
// SetCursor sets the pagination cursor of the spec, if applicable.
func (q *QueryEnvelope) SetCursor(cursor string) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.Cursor = cursor
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.Cursor = cursor
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.Cursor = cursor
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.Cursor = cursor
q.Spec = spec
}
}
// SetStepInterval sets the step interval of the spec, if applicable.
func (q *QueryEnvelope) SetStepInterval(step Step) {
switch spec := q.Spec.(type) {
case QueryBuilderTraceOperator:
spec.StepInterval = step
q.Spec = spec
case QueryBuilderQuery[TraceAggregation]:
spec.StepInterval = step
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.StepInterval = step
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.StepInterval = step
q.Spec = spec
case PromQuery:
spec.Step = step
q.Spec = spec
}
}
// SetSecondaryAggregations sets the secondary aggregations of the spec, if applicable.
func (q *QueryEnvelope) SetSecondaryAggregations(secondaryAggregations []SecondaryAggregation) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
case QueryBuilderJoin:
spec.SecondaryAggregations = secondaryAggregations
q.Spec = spec
}
}
// SetLimitBy sets the limit-by configuration of the spec, if applicable.
func (q *QueryEnvelope) SetLimitBy(limitBy *LimitBy) {
switch spec := q.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
spec.LimitBy = limitBy
q.Spec = spec
case QueryBuilderQuery[LogAggregation]:
spec.LimitBy = limitBy
q.Spec = spec
case QueryBuilderQuery[MetricAggregation]:
spec.LimitBy = limitBy
q.Spec = spec
}
}

View File

@@ -10,55 +10,9 @@ import (
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
// queryName returns the name from any query envelope spec type.
func (e QueryEnvelope) queryName() string {
switch spec := e.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Name
case QueryBuilderQuery[LogAggregation]:
return spec.Name
case QueryBuilderQuery[MetricAggregation]:
return spec.Name
case QueryBuilderFormula:
return spec.Name
case QueryBuilderTraceOperator:
return spec.Name
case QueryBuilderJoin:
return spec.Name
case PromQuery:
return spec.Name
case ClickHouseQuery:
return spec.Name
}
return ""
}
// isDisabled returns the disabled status from any query envelope spec type.
func (e QueryEnvelope) isDisabled() bool {
switch spec := e.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Disabled
case QueryBuilderQuery[LogAggregation]:
return spec.Disabled
case QueryBuilderQuery[MetricAggregation]:
return spec.Disabled
case QueryBuilderFormula:
return spec.Disabled
case QueryBuilderTraceOperator:
return spec.Disabled
case QueryBuilderJoin:
return spec.Disabled
case PromQuery:
return spec.Disabled
case ClickHouseQuery:
return spec.Disabled
}
return false
}
// getQueryIdentifier returns a friendly identifier for a query based on its type and name/content
func getQueryIdentifier(envelope QueryEnvelope, index int) string {
name := envelope.queryName()
name := envelope.GetQueryName()
var typeLabel string
switch envelope.Type {
@@ -89,50 +43,115 @@ const (
MaxQueryLimit = 10000
)
// Validate performs preliminary validation on QueryBuilderQuery
func (q *QueryBuilderQuery[T]) Validate(requestType RequestType) error {
// Validate signal
// ValidationOption is a functional option for configuring validation behaviour.
type ValidationOption func(*validationConfig)
type validationConfig struct {
skipLimitValidation bool
skipAggregationValidation bool
skipHavingValidation bool
skipAggregationOrderBy bool
skipSelectFieldValidation bool
skipGroupByValidation bool
}
func applyValidationOptions(opts []ValidationOption) validationConfig {
cfg := validationConfig{}
for _, opt := range opts {
opt(&cfg)
}
return cfg
}
// SkipLimitValidation returns a ValidationOption that skips the limit range check.
// Use this when the caller has already validated limits with different constraints.
func WithSkipLimitValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipLimitValidation = true
}
}
// SkipAggregationValidation skips aggregation validation.
// Used for raw/trace request types where aggregations are not required.
func WithSkipAggregationValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipAggregationValidation = true
}
}
// SkipHavingValidation skips having-clause validation.
// Used for raw/trace request types where having clauses do not apply.
func WithSkipHavingValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipHavingValidation = true
}
}
// SkipAggregationOrderBy skips the aggregation-specific order-by key validation.
// Used for raw/trace request types where order-by keys are not restricted to group-by or aggregation keys.
func WithSkipAggregationOrderBy() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipAggregationOrderBy = true
}
}
// SkipSelectFieldValidation skips select-field validation.
// Used for aggregation request types where select fields do not apply.
func WithSkipSelectFieldValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipSelectFieldValidation = true
}
}
// SkipGroupByValidation skips group-by validation.
// Used for raw/trace request types where group-by does not apply.
func WithSkipGroupByValidation() ValidationOption {
return func(cfg *validationConfig) {
cfg.skipGroupByValidation = true
}
}
// Validate performs preliminary validation on QueryBuilderQuery.
func (q *QueryBuilderQuery[T]) Validate(opts ...ValidationOption) error {
cfg := applyValidationOptions(opts)
if err := q.validateSignal(); err != nil {
return err
}
if err := q.validateAggregations(requestType); err != nil {
if err := q.validateAggregations(cfg); err != nil {
return err
}
if err := q.validateGroupBy(requestType); err != nil {
if err := q.validateGroupBy(cfg); err != nil {
return err
}
// Validate limit and pagination
if err := q.validateLimitAndPagination(); err != nil {
if err := q.validateLimitAndPagination(cfg); err != nil {
return err
}
// Validate functions
if err := q.validateFunctions(); err != nil {
return err
}
// Validate secondary aggregations
if err := q.validateSecondaryAggregations(); err != nil {
return err
}
if err := q.validateOrderBy(requestType); err != nil {
if err := q.validateOrderBy(cfg); err != nil {
return err
}
if err := q.validateSelectFields(requestType); err != nil {
if err := q.validateSelectFields(cfg); err != nil {
return err
}
return nil
}
func (q *QueryBuilderQuery[T]) validateSelectFields(requestType RequestType) error {
// selectFields don't apply to aggregation queries, skip validation
if requestType.IsAggregation() {
func (q *QueryBuilderQuery[T]) validateSelectFields(cfg validationConfig) error {
if cfg.skipSelectFieldValidation {
return nil
}
@@ -148,9 +167,8 @@ func (q *QueryBuilderQuery[T]) validateSelectFields(requestType RequestType) err
return nil
}
func (q *QueryBuilderQuery[T]) validateGroupBy(requestType RequestType) error {
// groupBy doesn't apply to non-aggregation queries, skip validation
if !requestType.IsAggregation() {
func (q *QueryBuilderQuery[T]) validateGroupBy(cfg validationConfig) error {
if cfg.skipGroupByValidation {
return nil
}
for idx, item := range q.GroupBy {
@@ -183,9 +201,8 @@ func (q *QueryBuilderQuery[T]) validateSignal() error {
}
}
func (q *QueryBuilderQuery[T]) validateAggregations(requestType RequestType) error {
// aggregations don't apply to non-aggregation queries, skip validation
if !requestType.IsAggregation() {
func (q *QueryBuilderQuery[T]) validateAggregations(cfg validationConfig) error {
if cfg.skipAggregationValidation {
return nil
}
@@ -265,24 +282,25 @@ func (q *QueryBuilderQuery[T]) validateAggregations(requestType RequestType) err
return nil
}
func (q *QueryBuilderQuery[T]) validateLimitAndPagination() error {
// Validate limit
if q.Limit < 0 {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
"limit must be non-negative, got %d",
q.Limit,
)
}
func (q *QueryBuilderQuery[T]) validateLimitAndPagination(cfg validationConfig) error {
if !cfg.skipLimitValidation {
if q.Limit < 0 {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
"limit must be non-negative, got %d",
q.Limit,
)
}
if q.Limit > MaxQueryLimit {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
"limit exceeds maximum allowed value of %d",
MaxQueryLimit,
).WithAdditional(
fmt.Sprintf("Provided limit: %d", q.Limit),
)
if q.Limit > MaxQueryLimit {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
"limit exceeds maximum allowed value of %d",
MaxQueryLimit,
).WithAdditional(
fmt.Sprintf("Provided limit: %d", q.Limit),
)
}
}
// Validate offset
@@ -329,7 +347,7 @@ func (q *QueryBuilderQuery[T]) validateSecondaryAggregations() error {
return nil
}
func (q *QueryBuilderQuery[T]) validateOrderBy(requestType RequestType) error {
func (q *QueryBuilderQuery[T]) validateOrderBy(cfg validationConfig) error {
for i, order := range q.Order {
// Direction validation is handled by the OrderDirection type
if order.Direction != OrderDirectionAsc && order.Direction != OrderDirectionDesc {
@@ -348,8 +366,7 @@ func (q *QueryBuilderQuery[T]) validateOrderBy(requestType RequestType) error {
}
}
// aggregation-specific order key validation only applies to aggregation queries
if requestType.IsAggregation() {
if !cfg.skipAggregationOrderBy {
return q.validateOrderByForAggregation()
}
@@ -431,8 +448,8 @@ func (q *QueryBuilderQuery[T]) validateOrderByForAggregation() error {
return nil
}
// ValidateQueryRangeRequest validates the entire query range request
func (r *QueryRangeRequest) Validate() error {
// Validate validates the entire query range request.
func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
// Validate time range
if r.RequestType != RequestTypeRawStream && r.Start >= r.End {
return errors.NewInvalidInputf(
@@ -443,8 +460,10 @@ func (r *QueryRangeRequest) Validate() error {
// Validate request type
switch r.RequestType {
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTimeSeries, RequestTypeScalar, RequestTypeTrace:
// Valid request types
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace:
opts = append(opts, getValidationOptions(false)...)
case RequestTypeTimeSeries, RequestTypeScalar:
opts = append(opts, getValidationOptions(true)...)
default:
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
@@ -456,7 +475,7 @@ func (r *QueryRangeRequest) Validate() error {
}
// Validate composite query
if err := r.validateCompositeQuery(); err != nil {
if err := r.CompositeQuery.Validate(opts...); err != nil {
return err
}
@@ -471,7 +490,7 @@ func (r *QueryRangeRequest) Validate() error {
// validateAllQueriesNotDisabled validates that at least one query in the composite query is enabled
func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error {
for _, envelope := range r.CompositeQuery.Queries {
if !envelope.isDisabled() {
if !envelope.IsDisabled() {
return nil
}
}
@@ -482,12 +501,8 @@ func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error {
)
}
func (r *QueryRangeRequest) validateCompositeQuery() error {
return r.CompositeQuery.Validate(r.RequestType)
}
// Validate performs validation on CompositeQuery
func (c *CompositeQuery) Validate(requestType RequestType) error {
func (c *CompositeQuery) Validate(opts ...ValidationOption) error {
if len(c.Queries) == 0 {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
@@ -499,14 +514,14 @@ func (c *CompositeQuery) Validate(requestType RequestType) error {
queryNames := make(map[string]bool)
for i, envelope := range c.Queries {
if err := validateQueryEnvelope(envelope, requestType); err != nil {
if err := validateQueryEnvelope(envelope, opts...); err != nil {
queryId := getQueryIdentifier(envelope, i)
return wrapValidationError(err, queryId, "invalid %s: %s")
}
// Check name uniqueness for builder queries
if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery {
name := envelope.queryName()
name := envelope.GetQueryName()
if name != "" {
if queryNames[name] {
return errors.NewInvalidInputf(
@@ -523,16 +538,16 @@ func (c *CompositeQuery) Validate(requestType RequestType) error {
return nil
}
func validateQueryEnvelope(envelope QueryEnvelope, requestType RequestType) error {
func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) error {
switch envelope.Type {
case QueryTypeBuilder, QueryTypeSubQuery:
switch spec := envelope.Spec.(type) {
case QueryBuilderQuery[TraceAggregation]:
return spec.Validate(requestType)
return spec.Validate(opts...)
case QueryBuilderQuery[LogAggregation]:
return spec.Validate(requestType)
return spec.Validate(opts...)
case QueryBuilderQuery[MetricAggregation]:
return spec.Validate(requestType)
return spec.Validate(opts...)
default:
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
@@ -618,3 +633,11 @@ func validateQueryEnvelope(envelope QueryEnvelope, requestType RequestType) erro
)
}
}
func getValidationOptions(isAggregationQuery bool) []ValidationOption {
if isAggregationQuery {
return []ValidationOption{WithSkipSelectFieldValidation()}
}
return []ValidationOption{WithSkipAggregationValidation(), WithSkipHavingValidation(), WithSkipAggregationOrderBy(), WithSkipGroupByValidation()}
}

View File

@@ -743,7 +743,7 @@ func TestValidateQueryEnvelope(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateQueryEnvelope(tt.envelope, tt.requestType)
err := validateQueryEnvelope(tt.envelope)
if tt.wantErr {
if err == nil {
t.Errorf("validateQueryEnvelope() expected error but got none")
@@ -816,7 +816,7 @@ func TestQueryEnvelope_Helpers(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.envelope.queryName()
got := tt.envelope.GetQueryName()
if got != tt.want {
t.Errorf("queryName() = %q, want %q", got, tt.want)
}
@@ -868,7 +868,7 @@ func TestQueryEnvelope_Helpers(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := tt.envelope.isDisabled()
got := tt.envelope.IsDisabled()
if got != tt.want {
t.Errorf("isDisabled() = %v, want %v", got, tt.want)
}
@@ -1107,7 +1107,7 @@ func TestQueryRangeRequest_ValidateOrderByForAggregation(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := tt.query.Validate(RequestTypeTimeSeries)
err := tt.query.Validate(getValidationOptions(true)...)
if tt.wantErr {
if err == nil {
t.Errorf("validateOrderByForAggregation() expected error but got none")
@@ -1161,7 +1161,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}},
},
}
err := query.Validate(RequestTypeRaw)
err := query.Validate(getValidationOptions(false)...)
if err != nil {
t.Errorf("expected no error for groupBy with raw request type, got: %v", err)
}
@@ -1178,7 +1178,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: ""}},
},
}
err := query.Validate(RequestTypeTimeSeries)
err := query.Validate(getValidationOptions(true)...)
if err == nil {
t.Errorf("expected error for empty groupBy key with timeseries request type")
}
@@ -1190,7 +1190,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
Signal: telemetrytypes.SignalLogs,
Having: &Having{Expression: "count() > 10"},
}
err := query.Validate(RequestTypeRaw)
err := query.Validate(getValidationOptions(false)...)
if err != nil {
t.Errorf("expected no error for having with raw request type, got: %v", err)
}
@@ -1202,7 +1202,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
Signal: telemetrytypes.SignalTraces,
Having: &Having{Expression: "count() > 10"},
}
err := query.Validate(RequestTypeTrace)
err := query.Validate(getValidationOptions(false)...)
if err != nil {
t.Errorf("expected no error for having with trace request type, got: %v", err)
}
@@ -1216,7 +1216,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{Expression: "count()"},
},
}
err := query.Validate(RequestTypeRaw)
err := query.Validate(getValidationOptions(false)...)
if err != nil {
t.Errorf("expected no error for aggregations with raw request type, got: %v", err)
}
@@ -1230,7 +1230,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
{Expression: "count()"},
},
}
err := query.Validate(RequestTypeRawStream)
err := query.Validate(getValidationOptions(false)...)
if err != nil {
t.Errorf("expected no error for aggregations with raw_stream request type, got: %v", err)
}
@@ -1248,12 +1248,12 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
},
}
// Should error for raw (selectFields are validated)
err := query.Validate(RequestTypeRaw)
err := query.Validate(getValidationOptions(false)...)
if err == nil {
t.Errorf("expected error for isRoot in selectFields with raw request type")
}
// Should pass for timeseries (selectFields skipped)
err = query.Validate(RequestTypeTimeSeries)
err = query.Validate(getValidationOptions(true)...)
if err != nil {
t.Errorf("expected no error for isRoot in selectFields with timeseries request type, got: %v", err)
}

View File

@@ -0,0 +1,634 @@
import csv
import io
import json
from datetime import datetime, timedelta, timezone
from http import HTTPStatus
from typing import Callable, List
from urllib.parse import urlencode
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.logs import Logs
def test_export_logs_csv(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert 3 logs with different severity levels and attributes.
Tests:
1. Export logs as CSV format
2. Verify CSV structure and content
3. Validate headers are present
4. Check log data is correctly formatted
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="Application started successfully",
severity_text="INFO",
resources={
"service.name": "api-service",
"deployment.environment": "production",
"host.name": "server-01",
},
attributes={
"http.method": "GET",
"http.status_code": 200,
"user.id": "user123",
},
),
Logs(
timestamp=now - timedelta(seconds=8),
body="Connection to database failed",
severity_text="ERROR",
resources={
"service.name": "api-service",
"deployment.environment": "production",
"host.name": "server-01",
},
attributes={
"error.type": "ConnectionError",
"db.name": "production_db",
},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Request processed",
severity_text="DEBUG",
resources={
"service.name": "worker-service",
"deployment.environment": "production",
"host.name": "server-02",
},
attributes={
"request.id": "req-456",
"duration_ms": 150.5,
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
}
# Export logs as CSV (default format, no source needed)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=30,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows, got {len(rows)}"
# Verify log bodies are present in the exported data
bodies = [row.get("body") for row in rows]
assert "Application started successfully" in bodies
assert "Connection to database failed" in bodies
assert "Request processed" in bodies
# Verify severity levels
severities = [row.get("severity_text") for row in rows]
assert "INFO" in severities
assert "ERROR" in severities
assert "DEBUG" in severities
def test_export_logs_jsonl(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert 2 logs with different attributes.
Tests:
1. Export logs as JSONL format
2. Verify JSONL structure and content
3. Check each line is valid JSON
4. Validate log data is correctly formatted
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="User logged in",
severity_text="INFO",
resources={
"service.name": "auth-service",
"deployment.environment": "staging",
},
attributes={
"user.email": "test@example.com",
"session.id": "sess-789",
},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Payment processed successfully",
severity_text="INFO",
resources={
"service.name": "payment-service",
"deployment.environment": "staging",
},
attributes={
"transaction.id": "txn-123",
"amount": 99.99,
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"format": "jsonl",
"source": "logs",
}
# Export logs as JSONL
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 2, f"Expected 2 lines, got {len(jsonl_lines)}"
# Verify each line is valid JSON
json_objects = []
for line in jsonl_lines:
obj = json.loads(line)
json_objects.append(obj)
assert "id" in obj
assert "timestamp" in obj
assert "body" in obj
assert "severity_text" in obj
# Verify log bodies
bodies = [obj.get("body") for obj in json_objects]
assert "User logged in" in bodies
assert "Payment processed successfully" in bodies
def test_export_logs_with_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs with different severity levels.
Tests:
1. Export logs with filter applied
2. Verify only filtered logs are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="Info message",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=8),
body="Error message",
severity_text="ERROR",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Another error message",
severity_text="ERROR",
resources={
"service.name": "test-service",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"format": "jsonl",
"source": "logs",
"filter": "severity_text = 'ERROR'",
}
# Export logs with filter
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 2, f"Expected 2 lines (filtered), got {len(jsonl_lines)}"
# Verify only ERROR logs are returned
for line in jsonl_lines:
obj = json.loads(line)
assert obj["severity_text"] == "ERROR"
assert "error message" in obj["body"].lower()
def test_export_logs_with_limit(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert 5 logs.
Tests:
1. Export logs with limit applied
2. Verify only limited number of logs are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
logs = []
for i in range(5):
logs.append(
Logs(
timestamp=now - timedelta(seconds=i),
body=f"Log message {i}",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={
"index": i,
},
)
)
insert_logs(logs)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"format": "csv",
"source": "logs",
"limit": 3,
}
# Export logs with limit
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows (limited), got {len(rows)}"
def test_export_logs_with_columns(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs with various attributes.
Tests:
1. Export logs with specific columns
2. Verify only specified columns are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="Test log message",
severity_text="INFO",
resources={
"service.name": "test-service",
"deployment.environment": "production",
},
attributes={
"http.method": "GET",
"http.status_code": 200,
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
# Request only specific columns
params = {
"start": start_ns,
"end": end_ns,
"format": "csv",
"source": "logs",
"columns": ["timestamp", "severity_text", "body"],
}
# Export logs with specific columns
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params, doseq=True)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 1
# Verify the specified columns are present
row = rows[0]
assert "timestamp" in row
assert "severity_text" in row
assert "body" in row
assert row["severity_text"] == "INFO"
assert row["body"] == "Test log message"
def test_export_logs_with_order_by(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs at different timestamps.
Tests:
1. Export logs with ascending timestamp order
2. Verify logs are returned in correct order
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="First log",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="Second log",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=1),
body="Third log",
severity_text="INFO",
resources={
"service.name": "test-service",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"format": "jsonl",
"source": "logs",
"order_by": "timestamp:asc",
}
# Export logs with ascending order
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 3
# Verify order - first log should be "First log" (oldest)
json_objects = [json.loads(line) for line in jsonl_lines]
assert json_objects[0]["body"] == "First log"
assert json_objects[1]["body"] == "Second log"
assert json_objects[2]["body"] == "Third log"
def test_export_logs_with_complex_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_logs: Callable[[List[Logs]], None],
) -> None:
"""
Setup:
Insert logs with various service names and severity levels.
Tests:
1. Export logs with complex filter (multiple conditions)
2. Verify only logs matching all conditions are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_logs(
[
Logs(
timestamp=now - timedelta(seconds=10),
body="API error occurred",
severity_text="ERROR",
resources={
"service.name": "api-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=8),
body="Worker info message",
severity_text="INFO",
resources={
"service.name": "worker-service",
},
attributes={},
),
Logs(
timestamp=now - timedelta(seconds=5),
body="API info message",
severity_text="INFO",
resources={
"service.name": "api-service",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
# Filter for api-service AND ERROR severity
params = {
"start": start_ns,
"end": end_ns,
"format": "jsonl",
"source": "logs",
"filter": "service.name = 'api-service' AND severity_text = 'ERROR'",
}
# Export logs with complex filter
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert (
len(jsonl_lines) == 1
), f"Expected 1 line (complex filter), got {len(jsonl_lines)}"
# Verify the filtered log
filtered_obj = json.loads(jsonl_lines[0])
assert filtered_obj["body"] == "API error occurred"
assert filtered_obj["severity_text"] == "ERROR"

View File

@@ -0,0 +1,782 @@
import csv
import io
import json
from datetime import datetime, timedelta, timezone
from http import HTTPStatus
from typing import Callable, List
from urllib.parse import urlencode
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
def test_export_traces_csv(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert 3 traces with different attributes.
Tests:
1. Export traces as CSV format
2. Verify CSV structure and content
3. Validate headers are present
4. Check trace data is correctly formatted
"""
http_service_trace_id = TraceIdGenerator.trace_id()
http_service_span_id = TraceIdGenerator.span_id()
http_service_db_span_id = TraceIdGenerator.span_id()
topic_service_trace_id = TraceIdGenerator.trace_id()
topic_service_span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=3),
trace_id=http_service_trace_id,
span_id=http_service_span_id,
parent_span_id="",
name="POST /integration",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"deployment.environment": "production",
"service.name": "http-service",
"os.type": "linux",
"host.name": "linux-000",
},
attributes={
"net.transport": "IP.TCP",
"http.scheme": "http",
"http.user_agent": "Integration Test",
"http.request.method": "POST",
"http.response.status_code": "200",
},
),
Traces(
timestamp=now - timedelta(seconds=3.5),
duration=timedelta(seconds=0.5),
trace_id=http_service_trace_id,
span_id=http_service_db_span_id,
parent_span_id=http_service_span_id,
name="SELECT",
kind=TracesKind.SPAN_KIND_CLIENT,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"deployment.environment": "production",
"service.name": "http-service",
"os.type": "linux",
"host.name": "linux-000",
},
attributes={
"db.name": "integration",
"db.operation": "SELECT",
"db.statement": "SELECT * FROM integration",
},
),
Traces(
timestamp=now - timedelta(seconds=1),
duration=timedelta(seconds=2),
trace_id=topic_service_trace_id,
span_id=topic_service_span_id,
parent_span_id="",
name="topic publish",
kind=TracesKind.SPAN_KIND_PRODUCER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"deployment.environment": "production",
"service.name": "topic-service",
"os.type": "linux",
"host.name": "linux-001",
},
attributes={
"message.type": "SENT",
"messaging.operation": "publish",
"messaging.message.id": "001",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"source": "traces",
"limit": 1000,
}
# Export traces as CSV (GET for simple queries)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=30,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows, got {len(rows)}"
# Verify trace IDs are present in the exported data
trace_ids = [row.get("trace_id") for row in rows]
assert http_service_trace_id in trace_ids
assert topic_service_trace_id in trace_ids
# Verify span names are present
span_names = [row.get("name") for row in rows]
assert "POST /integration" in span_names
assert "SELECT" in span_names
assert "topic publish" in span_names
def test_export_traces_jsonl(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert 2 traces with different attributes.
Tests:
1. Export traces as JSONL format
2. Verify JSONL structure and content
3. Check each line is valid JSON
4. Validate trace data is correctly formatted
"""
http_service_trace_id = TraceIdGenerator.trace_id()
http_service_span_id = TraceIdGenerator.span_id()
topic_service_trace_id = TraceIdGenerator.trace_id()
topic_service_span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=3),
trace_id=http_service_trace_id,
span_id=http_service_span_id,
parent_span_id="",
name="POST /api/test",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "api-service",
"deployment.environment": "staging",
},
attributes={
"http.request.method": "POST",
"http.response.status_code": "201",
},
),
Traces(
timestamp=now - timedelta(seconds=2),
duration=timedelta(seconds=1),
trace_id=topic_service_trace_id,
span_id=topic_service_span_id,
parent_span_id="",
name="queue.process",
kind=TracesKind.SPAN_KIND_CONSUMER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "queue-service",
"deployment.environment": "staging",
},
attributes={
"messaging.operation": "process",
"messaging.system": "rabbitmq",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"format": "jsonl",
"source": "traces",
"limit": 1000,
}
# Export traces as JSONL (GET for simple queries)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
assert "attachment" in response.headers.get("Content-Disposition", "")
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 2, f"Expected 2 lines, got {len(jsonl_lines)}"
# Verify each line is valid JSON
json_objects = []
for line in jsonl_lines:
obj = json.loads(line)
json_objects.append(obj)
assert "trace_id" in obj
assert "span_id" in obj
assert "name" in obj
# Verify trace IDs are present
trace_ids = [obj.get("trace_id") for obj in json_objects]
assert http_service_trace_id in trace_ids
assert topic_service_trace_id in trace_ids
# Verify span names are present
span_names = [obj.get("name") for obj in json_objects]
assert "POST /api/test" in span_names
assert "queue.process" in span_names
def test_export_traces_with_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert traces with different service names.
Tests:
1. Export traces with filter applied
2. Verify only filtered traces are returned
"""
service_a_trace_id = TraceIdGenerator.trace_id()
service_a_span_id = TraceIdGenerator.span_id()
service_b_trace_id = TraceIdGenerator.trace_id()
service_b_span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=4),
duration=timedelta(seconds=1),
trace_id=service_a_trace_id,
span_id=service_a_span_id,
parent_span_id="",
name="operation-a",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "service-a",
},
attributes={},
),
Traces(
timestamp=now - timedelta(seconds=2),
duration=timedelta(seconds=1),
trace_id=service_b_trace_id,
span_id=service_b_span_id,
parent_span_id="",
name="operation-b",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "service-b",
},
attributes={},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"format": "jsonl",
"source": "traces",
"limit": 1000,
"filter": "service.name = 'service-a'",
}
# Export traces with filter (GET supports filter param)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 1, f"Expected 1 line (filtered), got {len(jsonl_lines)}"
# Verify the filtered trace
filtered_obj = json.loads(jsonl_lines[0])
assert filtered_obj["trace_id"] == service_a_trace_id
assert filtered_obj["name"] == "operation-a"
def test_export_traces_with_limit(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert 5 traces.
Tests:
1. Export traces with limit applied
2. Verify only limited number of traces are returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
traces = []
for i in range(5):
traces.append(
Traces(
timestamp=now - timedelta(seconds=i),
duration=timedelta(seconds=1),
trace_id=TraceIdGenerator.trace_id(),
span_id=TraceIdGenerator.span_id(),
parent_span_id="",
name=f"operation-{i}",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "test-service",
},
attributes={},
)
)
insert_traces(traces)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
params = {
"start": start_ns,
"end": end_ns,
"format": "csv",
"source": "traces",
"limit": 3,
}
# Export traces with limit (GET supports limit param)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/export_raw_data?{urlencode(params)}"
),
timeout=10,
headers={
"authorization": f"Bearer {token}",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "text/csv"
# Parse CSV content
csv_content = response.text
csv_reader = csv.DictReader(io.StringIO(csv_content))
rows = list(csv_reader)
assert len(rows) == 3, f"Expected 3 rows (limited), got {len(rows)}"
def test_export_traces_multiple_queries_rejected(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""
Tests:
1. POST with multiple builder queries but no trace operator is rejected
2. Verify 400 error is returned
"""
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
body = {
"start": start_ns,
"end": end_ns,
"compositeQuery": {
"queries": [
{
"type": "builder_query",
"spec": {
"signal": "traces",
"name": "A",
"limit": 1000,
"filter": {"expression": "service.name = 'service-a'"},
},
},
{
"type": "builder_query",
"spec": {
"signal": "traces",
"name": "B",
"limit": 1000,
"filter": {"expression": "service.name = 'service-b'"},
},
},
]
},
}
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
response = requests.post(
url,
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.BAD_REQUEST
def test_export_traces_with_composite_query_trace_operator(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert multiple traces with parent-child relationships.
Tests:
1. Export traces using trace operator in composite query (POST)
2. Verify trace operator query works correctly
"""
parent_trace_id = TraceIdGenerator.trace_id()
parent_span_id = TraceIdGenerator.span_id()
child_span_id_1 = TraceIdGenerator.span_id()
child_span_id_2 = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=10),
duration=timedelta(seconds=5),
trace_id=parent_trace_id,
span_id=parent_span_id,
parent_span_id="",
name="parent-operation",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "parent-service",
},
attributes={
"operation.type": "parent",
},
),
Traces(
timestamp=now - timedelta(seconds=9),
duration=timedelta(seconds=2),
trace_id=parent_trace_id,
span_id=child_span_id_1,
parent_span_id=parent_span_id,
name="child-operation-1",
kind=TracesKind.SPAN_KIND_INTERNAL,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "parent-service",
},
attributes={
"operation.type": "child",
},
),
Traces(
timestamp=now - timedelta(seconds=7),
duration=timedelta(seconds=1),
trace_id=parent_trace_id,
span_id=child_span_id_2,
parent_span_id=parent_span_id,
name="child-operation-2",
kind=TracesKind.SPAN_KIND_INTERNAL,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "parent-service",
},
attributes={
"operation.type": "child",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
# A: spans with operation.type = 'parent'
query_a = {
"type": "builder_query",
"spec": {
"signal": "traces",
"name": "A",
"limit": 1000,
"filter": {"expression": "operation.type = 'parent'"},
},
}
# B: spans with operation.type = 'child'
query_b = {
"type": "builder_query",
"spec": {
"signal": "traces",
"name": "B",
"limit": 1000,
"filter": {"expression": "operation.type = 'child'"},
},
}
# Trace operator: find traces where A has a direct descendant B
query_c = {
"type": "builder_trace_operator",
"spec": {
"name": "C",
"expression": "A => B",
"returnSpansFrom": "A",
"limit": 1000,
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
},
}
body = {
"start": start_ns,
"end": end_ns,
"requestType": "raw",
"compositeQuery": {
"queries": [query_a, query_b, query_c],
},
}
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
response = requests.post(
url,
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
print(response.text)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 1, f"Expected at least 1 line, got {len(jsonl_lines)}"
# Verify all returned spans belong to the matched trace
json_objects = [json.loads(line) for line in jsonl_lines]
trace_ids = [obj.get("trace_id") for obj in json_objects]
assert all(tid == parent_trace_id for tid in trace_ids)
# Verify the parent span (returnSpansFrom = "A") is present
span_names = [obj.get("name") for obj in json_objects]
assert "parent-operation" in span_names
def test_export_traces_with_select_fields(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_traces: Callable[[List[Traces]], None],
) -> None:
"""
Setup:
Insert traces with various attributes.
Tests:
1. Export traces with specific select fields via POST
2. Verify only specified fields are returned in the output
"""
trace_id = TraceIdGenerator.trace_id()
span_id = TraceIdGenerator.span_id()
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
insert_traces(
[
Traces(
timestamp=now - timedelta(seconds=10),
duration=timedelta(seconds=2),
trace_id=trace_id,
span_id=span_id,
parent_span_id="",
name="test-operation",
kind=TracesKind.SPAN_KIND_SERVER,
status_code=TracesStatusCode.STATUS_CODE_OK,
status_message="",
resources={
"service.name": "test-service",
"deployment.environment": "production",
"host.name": "server-01",
},
attributes={
"http.method": "POST",
"http.status_code": "201",
"user.id": "user123",
},
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Calculate timestamps in nanoseconds
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
end_ns = int(now.timestamp() * 1e9)
body = {
"start": start_ns,
"end": end_ns,
"requestType": "raw",
"compositeQuery": {
"queries": [
{
"type": "builder_query",
"spec": {
"signal": "traces",
"name": "A",
"limit": 1000,
"selectFields": [
{
"name": "trace_id",
"fieldDataType": "string",
"fieldContext": "span",
"signal": "traces",
},
{
"name": "span_id",
"fieldDataType": "string",
"fieldContext": "span",
"signal": "traces",
},
{
"name": "name",
"fieldDataType": "string",
"fieldContext": "span",
"signal": "traces",
},
{
"name": "service.name",
"fieldDataType": "string",
"fieldContext": "resource",
"signal": "traces",
},
],
},
}
]
},
}
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
response = requests.post(
url,
json=body,
timeout=10,
headers={
"authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status_code == HTTPStatus.OK
assert response.headers["Content-Type"] == "application/x-ndjson"
# Parse JSONL content
jsonl_lines = response.text.strip().split("\n")
assert len(jsonl_lines) == 1
# Verify the selected fields are present
result = json.loads(jsonl_lines[0])
assert "trace_id" in result
assert "span_id" in result
assert "name" in result
# Verify values
assert result["trace_id"] == trace_id
assert result["span_id"] == span_id
assert result["name"] == "test-operation"