mirror of
https://github.com/SigNoz/signoz.git
synced 2026-03-01 03:22:55 +00:00
Compare commits
14 Commits
testing-fe
...
tvats-expo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7aa0eff6f4 | ||
|
|
0e30a11231 | ||
|
|
5901074385 | ||
|
|
5b27121530 | ||
|
|
7090d1af17 | ||
|
|
31152e01f5 | ||
|
|
e19b37620d | ||
|
|
820f0723a5 | ||
|
|
b8c101e95a | ||
|
|
85c9daab12 | ||
|
|
4a80ef8f43 | ||
|
|
ac8600f5da | ||
|
|
b2655634fe | ||
|
|
b2f20f7a64 |
10
.github/workflows/jsci.yaml
vendored
10
.github/workflows/jsci.yaml
vendored
@@ -61,13 +61,3 @@ jobs:
|
||||
with:
|
||||
PRIMUS_REF: main
|
||||
JS_SRC: frontend
|
||||
md-languages:
|
||||
if: |
|
||||
(github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
|
||||
(github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: validate md languages
|
||||
run: bash frontend/scripts/validate-md-languages.sh
|
||||
|
||||
4
.vscode/settings.json
vendored
4
.vscode/settings.json
vendored
@@ -17,5 +17,7 @@
|
||||
},
|
||||
"[html]": {
|
||||
"editor.defaultFormatter": "vscode.html-language-features"
|
||||
}
|
||||
},
|
||||
"python-envs.defaultEnvManager": "ms-python.python:system",
|
||||
"python-envs.pythonProjects": []
|
||||
}
|
||||
|
||||
@@ -2858,6 +2858,186 @@ paths:
|
||||
summary: Update auth domain
|
||||
tags:
|
||||
- authdomains
|
||||
/api/v1/export_raw_data:
|
||||
get:
|
||||
deprecated: false
|
||||
description: This endpoints allows simple query exporting raw data for traces
|
||||
and logs
|
||||
operationId: HandleExportRawDataGET
|
||||
parameters:
|
||||
- in: query
|
||||
name: format
|
||||
schema:
|
||||
default: csv
|
||||
enum:
|
||||
- csv
|
||||
- jsonl
|
||||
type: string
|
||||
- in: query
|
||||
name: signal
|
||||
required: true
|
||||
schema:
|
||||
$ref: '#/components/schemas/TelemetrytypesSignal'
|
||||
- deprecated: true
|
||||
in: query
|
||||
name: source
|
||||
schema:
|
||||
deprecated: true
|
||||
type: string
|
||||
- in: query
|
||||
name: start
|
||||
schema:
|
||||
minimum: 0
|
||||
type: integer
|
||||
- in: query
|
||||
name: end
|
||||
schema:
|
||||
minimum: 0
|
||||
type: integer
|
||||
- in: query
|
||||
name: limit
|
||||
schema:
|
||||
default: 10000
|
||||
maximum: 50000
|
||||
minimum: 1
|
||||
type: integer
|
||||
- deprecated: true
|
||||
in: query
|
||||
name: filter
|
||||
schema:
|
||||
deprecated: true
|
||||
type: string
|
||||
- content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Querybuildertypesv5Filter'
|
||||
in: query
|
||||
name: filterExpression
|
||||
- deprecated: true
|
||||
in: query
|
||||
name: columns
|
||||
schema:
|
||||
deprecated: true
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
- in: query
|
||||
name: selectFields
|
||||
schema:
|
||||
items:
|
||||
$ref: '#/components/schemas/TelemetrytypesTelemetryFieldKey'
|
||||
type: array
|
||||
- deprecated: true
|
||||
in: query
|
||||
name: order_by
|
||||
schema:
|
||||
deprecated: true
|
||||
type: string
|
||||
- in: query
|
||||
name: order
|
||||
schema:
|
||||
items:
|
||||
$ref: '#/components/schemas/Querybuildertypesv5OrderBy'
|
||||
type: array
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: string
|
||||
description: OK
|
||||
"400":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Bad Request
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- VIEWER
|
||||
- tokenizer:
|
||||
- VIEWER
|
||||
summary: Export raw data
|
||||
tags:
|
||||
- logs
|
||||
- traces
|
||||
post:
|
||||
deprecated: false
|
||||
description: This endpoints allows complex query exporting raw data for traces
|
||||
and logs
|
||||
operationId: HandleExportRawDataPOST
|
||||
parameters:
|
||||
- in: query
|
||||
name: format
|
||||
schema:
|
||||
default: csv
|
||||
enum:
|
||||
- csv
|
||||
- jsonl
|
||||
type: string
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/Querybuildertypesv5QueryRangeRequest'
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: string
|
||||
description: OK
|
||||
"400":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Bad Request
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- VIEWER
|
||||
- tokenizer:
|
||||
- VIEWER
|
||||
summary: Export raw data
|
||||
tags:
|
||||
- logs
|
||||
- traces
|
||||
/api/v1/fields/keys:
|
||||
get:
|
||||
deprecated: false
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
# Abstractions
|
||||
|
||||
This document provides rules for deciding when a new type, interface, or intermediate representation is warranted in Go code. The goal is to keep the codebase navigable by ensuring every abstraction earns its place.
|
||||
|
||||
## The cost of a new abstraction
|
||||
|
||||
Every exported type, interface, or wrapper is a permanent commitment. It must be named, documented, tested, and understood by every future contributor. It creates a new concept in the codebase vocabulary. Before introducing one, verify that the cost is justified by a concrete benefit that cannot be achieved with existing mechanisms.
|
||||
|
||||
## Before you introduce anything new
|
||||
|
||||
Answer these four questions. If writing a PR, include the answers in the description.
|
||||
|
||||
1. **What already exists?** Name the specific type, function, interface, or library that covers this ground today.
|
||||
2. **What does the new abstraction add?** Name the concrete operation, guarantee, or capability. "Cleaner" or "more reusable" are not sufficient; name what the caller can do that it could not do before.
|
||||
3. **What does the new abstraction drop?** If it wraps or mirrors an existing structure, list what it cannot represent. Every gap must be either justified or handled with an explicit error.
|
||||
4. **Who consumes it?** List the call sites. If there is only one producer and one consumer in the same call chain, you likely need a function, not a type.
|
||||
|
||||
## Rules
|
||||
|
||||
### 1. Prefer functions over types
|
||||
|
||||
If a piece of logic has one input and one output, write a function. Do not create a struct to hold intermediate state that is built in one place and read in one place. A function is easier to test, easier to inline, and does not expand the vocabulary of the codebase.
|
||||
|
||||
```go
|
||||
// Prefer this:
|
||||
func ConvertConfig(src ExternalConfig) (InternalConfig, error)
|
||||
|
||||
// Over this:
|
||||
type ConfigAdapter struct { ... }
|
||||
func NewConfigAdapter(src ExternalConfig) *ConfigAdapter
|
||||
func (a *ConfigAdapter) ToInternal() (InternalConfig, error)
|
||||
```
|
||||
|
||||
The two-step version is only justified when `ConfigAdapter` has multiple distinct consumers that use it in different ways.
|
||||
|
||||
### 2. Do not duplicate structures you do not own
|
||||
|
||||
When a library or external package produces a structured output, operate on that output directly. Do not create a parallel type that mirrors a subset of its fields.
|
||||
|
||||
A partial copy will:
|
||||
- **Silently lose data** when the source has fields or variants the copy does not account for.
|
||||
- **Drift** when the source evolves and the copy is not updated in lockstep.
|
||||
- **Add a conversion step** that doubles the code surface and the opportunity for bugs.
|
||||
|
||||
If you need to shield consumers from a dependency, define a narrow interface over the dependency's type rather than copying its shape into a new struct.
|
||||
|
||||
### 3. Never silently discard input
|
||||
|
||||
If your code receives structured input and cannot handle part of it, return an error. Do not silently return nil, skip the element, or produce a partial result. Silent data loss is the hardest class of bug to detect because the code appears to work, it just produces wrong results.
|
||||
|
||||
```go
|
||||
// Wrong: silently ignores the unrecognized case.
|
||||
default:
|
||||
return nil
|
||||
|
||||
// Right: makes the gap visible.
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported %T value: %v", v, v)
|
||||
```
|
||||
|
||||
This applies broadly: type switches, format conversions, data migrations, enum mappings, configuration parsing. Anywhere a `default` or `else` branch can swallow input, it should surface an error instead.
|
||||
|
||||
### 4. Do not expose methods that lose information
|
||||
|
||||
A method on a structured type should not strip meaning from the structure it belongs to. If a caller needs to iterate over elements for a specific purpose (validation, aggregation, logging), write that logic as a standalone function that operates on the structure with full context, rather than adding a method that returns a reduced view.
|
||||
|
||||
```go
|
||||
// Problematic: callers cannot distinguish how items were related.
|
||||
func (o *Order) AllLineItems() []LineItem { ... }
|
||||
|
||||
// Better: the validation logic operates on the full structure.
|
||||
func ValidateOrder(o *Order) error { ... }
|
||||
```
|
||||
|
||||
Public methods shape how a type is used. Once a lossy accessor exists, callers will depend on it, and the lost information becomes unrecoverable at those call sites.
|
||||
|
||||
### 5. Interfaces should be discovered, not predicted
|
||||
|
||||
Do not define an interface before you have at least two concrete implementations that need it. An interface with one implementation is not abstraction; it is indirection that makes it harder to navigate from call site to implementation.
|
||||
|
||||
The exception is interfaces required for testing (e.g., for mocking an external dependency). In that case, define the interface in the **consuming** package, not the providing package, following the Go convention of [accepting interfaces and returning structs](https://go.dev/wiki/CodeReviewComments#interfaces).
|
||||
|
||||
## When a new type IS warranted
|
||||
|
||||
See [Types](types.md#when-a-new-type-is-warranted) for the criteria that justify introducing a new type.
|
||||
|
||||
## What should I remember?
|
||||
|
||||
- A function is almost always simpler than a type. Start with a function; promote to a type only when you have evidence of need.
|
||||
- Never silently drop data. If you cannot handle it, error.
|
||||
- If your new type mirrors an existing one, you need a strong reason beyond "nicer to work with".
|
||||
- If your type has one producer and one consumer, it is indirection, not abstraction.
|
||||
- Interfaces come from need (multiple implementations), not from prediction.
|
||||
- When in doubt, do not add it. It is easier to add an abstraction later when the need is clear than to remove one after it has spread through the codebase.
|
||||
|
||||
## Further reading
|
||||
|
||||
These works and our own lessions shaped the above guidelines
|
||||
|
||||
- [The Wrong Abstraction](https://sandimetz.com/blog/2016/1/20/the-wrong-abstraction) - Sandi Metz. The wrong abstraction is worse than duplication. If you find yourself passing parameters and adding conditional paths through shared code, inline it back into every caller and let the duplication show you what the right abstraction is.
|
||||
- [Write code that is easy to delete, not easy to extend](https://programmingisterrible.com/post/139222674273/write-code-that-is-easy-to-delete-not-easy-to) - tef. Every abstraction is a bet on the future. Optimize for how cheaply you can remove code when the bet is wrong, not for how easily you can extend it when the bet is right.
|
||||
- [Goodbye, Clean Code](https://overreacted.io/goodbye-clean-code/) - Dan Abramov. A refactoring that removes duplication can look cleaner while making the code harder to change. Clean-looking and easy-to-change are not the same thing.
|
||||
- [A Philosophy of Software Design](https://www.amazon.com/Philosophy-Software-Design-John-Ousterhout/dp/1732102201) - John Ousterhout. Good abstractions are deep: simple interface, complex implementation. A "false abstraction" omits important details while appearing simple, and is worse than no abstraction at all. ([Summary by Pragmatic Engineer](https://blog.pragmaticengineer.com/a-philosophy-of-software-design-review/))
|
||||
- [Simplicity is Complicated](https://go.dev/talks/2015/simplicity-is-complicated.slide) - Rob Pike. Go-specific. Fewer orthogonal concepts that compose predictably beat many overlapping ones. Features were left out of Go deliberately; the same discipline applies to your own code.
|
||||
@@ -123,6 +123,7 @@ if err := router.Handle("/api/v1/things", handler.New(
|
||||
Description: "This endpoint creates a thing",
|
||||
Request: new(types.PostableThing),
|
||||
RequestContentType: "application/json",
|
||||
RequestQuery: new(types.QueryableThing),
|
||||
Response: new(types.GettableThing),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusCreated,
|
||||
@@ -155,6 +156,8 @@ The `handler.New` function ties the HTTP handler to OpenAPI metadata via `OpenAP
|
||||
- **Request / RequestContentType**:
|
||||
- `Request` is a Go type that describes the request body or form.
|
||||
- `RequestContentType` is usually `"application/json"` or `"application/x-www-form-urlencoded"` (for callbacks like SAML).
|
||||
- **RequestQuery**:
|
||||
- `RequestQuery` is a Go type that descirbes query url params.
|
||||
- **RequestExamples**: An array of `handler.OpenAPIExample` that provide concrete request payloads in the generated spec. See [Adding request examples](#adding-request-examples) below.
|
||||
- **Response / ResponseContentType**:
|
||||
- `Response` is the Go type for the successful response payload.
|
||||
|
||||
@@ -1,169 +0,0 @@
|
||||
# Packages
|
||||
|
||||
All shared Go code in SigNoz lives under `pkg/`. Each package represents a distinct domain concept and exposes a clear public interface. This guide covers the conventions for creating, naming, and organising packages so the codebase stays consistent as it grows.
|
||||
|
||||
## How should I name a package?
|
||||
|
||||
Use short, lowercase, single-word names. No underscores or camelCase (`querier`, `cache`, `authz`, not `query_builder` or `dataStore`).
|
||||
|
||||
Names must be **domain-specific**. A package name should tell you what problem domain it deals with, not what data structure it wraps. Prefer `alertmanager` over `manager`, `licensing` over `checker`.
|
||||
|
||||
Avoid generic names like `util`, `helpers`, `common`, `misc`, or `base`. If you can't name it, the code probably belongs in an existing package.
|
||||
|
||||
## When should I create a new package?
|
||||
|
||||
Create a new package when:
|
||||
|
||||
- The functionality represents a **distinct domain concept** (e.g., `authz`, `licensing`, `cache`).
|
||||
- Two or more other packages would import it; it serves as shared infrastructure.
|
||||
- The code has a clear public interface that can stand on its own.
|
||||
|
||||
Do **not** create a new package when:
|
||||
|
||||
- There is already a package that covers the same domain. Extend the existing package instead.
|
||||
- The code is only used in one place. Keep it local to the caller.
|
||||
- You are splitting purely for file size. Use multiple files within the same package instead.
|
||||
|
||||
## How should I lay out a package?
|
||||
|
||||
A typical package looks like:
|
||||
|
||||
```
|
||||
pkg/cache/
|
||||
├── cache.go # Public interface + exported types
|
||||
├── config.go # Configuration types if needed
|
||||
├── memorycache/ # Implementation sub-package
|
||||
├── rediscache/ # Another implementation
|
||||
└── cachetest/ # Test helpers for consumers
|
||||
```
|
||||
|
||||
Follow these rules:
|
||||
|
||||
1. **Interface-first file**: The file matching the package name (e.g., `cache.go` in `pkg/cache/`) should define the public interface and core exported types. Keep implementation details out of this file.
|
||||
|
||||
2. **One responsibility per file**: Name files after what they contain (`config.go`, `handler.go`, `service.go`), not after the package name. If a package merges two concerns, prefix files to group them (e.g., `memory_store.go`, `redis_store.go` in a storage package).
|
||||
|
||||
3. **Sub-packages for implementations**: When a package defines an interface with multiple implementations, put each implementation in its own sub-package (`memorycache/`, `rediscache/`). This keeps the parent package import-free of implementation dependencies.
|
||||
|
||||
4. **Test helpers in `{pkg}test/`**: If consumers need test mocks or builders, put them in a `{pkg}test/` sub-package (e.g., `cachetest/`, `sqlstoretest/`). This avoids polluting the main package with test-only code.
|
||||
|
||||
5. **Test files stay alongside source**: Unit tests go in `_test.go` files next to the code they test, in the same package.
|
||||
|
||||
## How should I order code within a file?
|
||||
|
||||
Within a single `.go` file, declarations should follow this order:
|
||||
|
||||
1. Constants
|
||||
2. Variables
|
||||
3. Types (structs, interfaces)
|
||||
4. Constructor functions (`New...`)
|
||||
5. Exported methods and functions
|
||||
6. Unexported methods and functions
|
||||
|
||||
```go
|
||||
// 1. Constants
|
||||
const defaultTimeout = 30 * time.Second
|
||||
|
||||
// 2. Variables
|
||||
var ErrNotFound = errors.New(errors.TypeNotFound, errors.CodeNotFound, "resource not found")
|
||||
|
||||
// 3. Types
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
}
|
||||
|
||||
// 4. Constructors
|
||||
func NewStore(db *sql.DB) *Store {
|
||||
return &Store{db: db}
|
||||
}
|
||||
|
||||
// 5. Exported methods
|
||||
func (s *Store) Get(ctx context.Context, id string) (*Resource, error) { ... }
|
||||
|
||||
// 6. Unexported methods
|
||||
func (s *Store) buildQuery(id string) string { ... }
|
||||
```
|
||||
|
||||
This ordering makes files predictable. A reader scanning from top to bottom sees the contract (constants, types, constructors) before the implementation (methods), and exported behavior before internal helpers.
|
||||
|
||||
## How should I name symbols?
|
||||
|
||||
### Exported symbols
|
||||
|
||||
- **Interfaces**: For single-method interfaces, follow the standard `-er` suffix convention (`Reader`, `Writer`, `Closer`). For multi-method interfaces, use clear nouns (`Cache`, `Store`, `Provider`).
|
||||
- **Constructors**: `New<Type>(...)` (e.g., `NewMemoryCache()`).
|
||||
- **Avoid stutter**: Since callers qualify with the package name, don't repeat it. Write `cache.Cache`, not `cache.CacheInterface`. Write `authz.FromRole`, not `authz.AuthzFromRole`.
|
||||
|
||||
### Unexported symbols
|
||||
|
||||
- Struct receivers: one or two characters (`c`, `f`, `br`).
|
||||
- Helper functions: descriptive lowercase names (`parseToken`, `buildQuery`).
|
||||
|
||||
### Constants
|
||||
|
||||
- Use `PascalCase` for exported constants.
|
||||
- When merging files from different origins into one package, watch out for **name collisions** across files. Prefix to disambiguate when two types share a natural name.
|
||||
|
||||
## How should I organise imports?
|
||||
|
||||
Group imports in three blocks separated by blank lines:
|
||||
|
||||
```go
|
||||
import (
|
||||
// 1. Standard library
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
// 2. External dependencies
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
// 3. Internal
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
)
|
||||
```
|
||||
|
||||
Never introduce circular imports. If package A needs package B and B needs A, extract the shared types into a third package (often under `pkg/types/`).
|
||||
|
||||
## Where do shared types go?
|
||||
|
||||
See [Types](types.md) for full conventions on type placement, naming variants, composition, and constructors.
|
||||
|
||||
## How do I merge or move packages?
|
||||
|
||||
When two packages are tightly coupled (one imports the other's constants, they cover the same domain), merge them:
|
||||
|
||||
1. Pick a domain-specific name for the combined package.
|
||||
2. Prefix files to preserve origin (e.g., `memory_store.go`, `redis_store.go`).
|
||||
3. Resolve symbol conflicts explicitly; rename with a prefix rather than silently shadowing.
|
||||
4. Update all consumers in a single change.
|
||||
5. Delete the old packages. Do not leave behind re-export shims.
|
||||
6. Verify with `go build ./...`, `go test ./<new-pkg>/...`, and `go vet ./...`.
|
||||
|
||||
## When should I use valuer types?
|
||||
|
||||
See [Types](types.md#typed-domain-values-pkgvaluer) for valuer types, when to use them, and the enum pattern using `valuer.String`.
|
||||
|
||||
## When should I add documentation?
|
||||
|
||||
Add a `doc.go` with a package-level comment for any package that is non-trivial or has multiple consumers. Keep it to 1–3 sentences:
|
||||
|
||||
```go
|
||||
// Package cache provides a caching interface with pluggable backends
|
||||
// for in-memory and Redis-based storage.
|
||||
package cache
|
||||
```
|
||||
|
||||
## What should I remember?
|
||||
|
||||
- Package names are domain-specific and lowercase. Never generic names like `util` or `common`.
|
||||
- The file matching the package name (e.g., `cache.go`) defines the public interface. Implementation details go elsewhere.
|
||||
- Within a file, order declarations: constants, variables, types, constructors, exported functions, unexported functions.
|
||||
- Segregate types across files by responsibility. A file with 5 unrelated types is harder to navigate than 5 files with one type each.
|
||||
- Use valuer types (`valuer.String`, `valuer.Email`, `valuer.UUID`, `valuer.TextDuration`) for domain values that need validation, normalization, or cross-boundary serialization. See [Types](types.md#typed-domain-values-pkgvaluer) for details.
|
||||
- Avoid `init()` functions. If you need to initialize a variable, use a package-level `var` with a function call or a `sync.Once`. `init()` hides execution order, makes testing harder, and has caused subtle bugs in large codebases.
|
||||
- Never introduce circular imports. Extract shared types into `pkg/types/` when needed.
|
||||
- Watch for symbol name collisions when merging packages, prefix to disambiguate.
|
||||
- Put test helpers in a `{pkg}test/` sub-package, not in the main package.
|
||||
- Before submitting, verify with `go build ./...`, `go test ./<your-pkg>/...`, and `go vet ./...`.
|
||||
- Update all consumers when you rename or move symbols.
|
||||
@@ -8,41 +8,4 @@ We adhere to three primary style guides as our foundation:
|
||||
- [Code Review Comments](https://go.dev/wiki/CodeReviewComments) - For understanding common comments in code reviews
|
||||
- [Google Style Guide](https://google.github.io/styleguide/go/) - Additional practices from Google
|
||||
|
||||
We **recommend** (almost enforce) reviewing these guides before contributing to the codebase. They provide valuable insights into writing idiomatic Go code and will help you understand our approach to backend development.
|
||||
|
||||
**Discover before inventing.** Before writing new code, search the codebase for existing solutions. SigNoz has established patterns for common problems: `pkg/valuer` for typed domain values, `pkg/errors` for structured errors, `pkg/factory` for provider wiring, `{pkg}test/` sub-packages for test helpers, and shared fixtures for integration tests. Duplicating what already exists creates drift and maintenance burden. When you find an existing pattern, use it. When you don't find one, check with the maintainers before building your own.
|
||||
|
||||
## How to approach a feature
|
||||
|
||||
Building a feature is not one task, it is a sequence of concerns that build on each other. Work through them in this order:
|
||||
|
||||
1. **Domain design (types).** Define the types that represent your domain. What are the entities, what are their relationships, what are the constraints? This is where you decide your data model. Get this right first because everything else depends on it. See [Packages](packages.md) and [Abstractions](abstractions.md).
|
||||
|
||||
2. **Structure (services / modules / handlers).** Place your code in the right layer given the current infrastructure. If the current structure does not work for your feature, that is the time to open a discussion and write a technical document, not to silently reshape things in the same PR. See [Handler](handler.md) and [Provider](provider.md).
|
||||
|
||||
3. **HTTP endpoints (paths, status codes, errors).** Pay close attention to detail here. Paths, methods, request/response shapes, status codes, error codes. These are the contract with consumers and are expensive to change after release. See [Endpoint](endpoint.md) and [Handler](handler.md).
|
||||
|
||||
4. **Database constraints (org_id, foreign keys, migrations).** Ensure org scoping, schema consistency, and migration correctness. See [SQL](sql.md).
|
||||
|
||||
5. **Business logic (module layer).** With the types, structure, endpoints, and storage in place, the focus narrows to the actual logic. This is where review should concentrate on correctness, edge cases, and error handling.
|
||||
|
||||
This ordering also gives you a natural way to split PRs. Each layer affects a different area and requires a different lens for review. A PR that mixes refactoring with new feature logic is hard to review and risky to ship. Separate them.
|
||||
|
||||
For large refactors or features that touch multiple subsystems, write a short technical document outlining the design and get relevant stakeholders aligned before starting implementation. This saves significant back-and-forth during review.
|
||||
|
||||
## Area-specific guides
|
||||
|
||||
In addition, we have a few additional rules that make certain areas stricter than the above which can be found in area-specific files in this package:
|
||||
|
||||
- [Abstractions](abstractions.md) - When to introduce new types and intermediate representations
|
||||
- [Errors](errors.md) - Structured error handling
|
||||
- [Endpoint](endpoint.md) - HTTP endpoint patterns
|
||||
- [Flagger](flagger.md) - Feature flag patterns
|
||||
- [Handler](handler.md) - HTTP handler patterns
|
||||
- [Integration](integration.md) - Integration testing
|
||||
- [Provider](provider.md) - Dependency injection and provider patterns
|
||||
- [Packages](packages.md) - Naming, layout, and conventions for `pkg/` packages
|
||||
- [Service](service.md) - Managed service lifecycle with `factory.Service`
|
||||
- [SQL](sql.md) - Database and SQL patterns
|
||||
- [Testing](testing.md) - Writing tests that catch bugs without becoming a maintenance burden
|
||||
- [Types](types.md) - Type placement, naming variants, composition, and constructors
|
||||
We **recommend** (almost enforce) reviewing these guides before contributing to the codebase. They provide valuable insights into writing idiomatic Go code and will help you understand our approach to backend development. In addition, we have a few additional rules that make certain areas stricter than the above which can be found in area-specific files in this package.
|
||||
|
||||
@@ -1,269 +0,0 @@
|
||||
# Service
|
||||
|
||||
A service is a component with a managed lifecycle: it starts, runs for the lifetime of the application, and stops gracefully.
|
||||
|
||||
Services are distinct from [providers](provider.md). A provider adapts an external dependency behind an interface. A service has a managed lifecycle that is tied to the lifetime of the application.
|
||||
|
||||
## When do you need a service?
|
||||
|
||||
You need a service when your component needs to do work that outlives a single method call:
|
||||
|
||||
- **Periodic work**: polling an external system, garbage-collecting expired data, syncing state on an interval.
|
||||
- **Graceful shutdown**: holding resources (connections, caches, buffers) that must be flushed or closed before the process exits.
|
||||
- **Blocking on readiness**: waiting for an external dependency to become available before the application can proceed.
|
||||
|
||||
If your component only responds to calls and holds no state that requires cleanup, it is a provider, not a service. If it does both (responds to calls *and* needs a lifecycle), embed `factory.Service` in the provider interface; see [How to create a service](#how-to-create-a-service).
|
||||
|
||||
## The interface
|
||||
|
||||
The `factory.Service` interface in `pkg/factory/service.go` defines two methods:
|
||||
|
||||
```go
|
||||
type Service interface {
|
||||
// Starts a service. It should block and should not return until the service is stopped or it fails.
|
||||
Start(context.Context) error
|
||||
// Stops a service.
|
||||
Stop(context.Context) error
|
||||
}
|
||||
```
|
||||
|
||||
`Start` **must block**. It should not return until the service is stopped (returning `nil`) or something goes wrong (returning an error). If `Start` returns an error, the entire application shuts down.
|
||||
|
||||
`Stop` should cause `Start` to unblock and return. It must be safe to call from a different goroutine than the one running `Start`.
|
||||
|
||||
## Shutdown coordination
|
||||
|
||||
Every service uses a `stopC chan struct{}` to coordinate shutdown:
|
||||
|
||||
- **Constructor**: `stopC: make(chan struct{})`
|
||||
- **Start**: blocks on `<-stopC` (or uses it in a `select` loop)
|
||||
- **Stop**: `close(stopC)` to unblock `Start`
|
||||
|
||||
This is the standard pattern. Do not use `context.WithCancel` or other mechanisms for service-level shutdown coordination. See the examples in the next section.
|
||||
|
||||
## Service shapes
|
||||
|
||||
Two shapes recur across the codebase (these are not exhaustive, if a new shape is needed, bring it up for discussion before going ahead with the implementation), implemented by convention rather than base classes.
|
||||
|
||||
### Idle service
|
||||
|
||||
The service does work during startup or shutdown but has nothing to do while running. `Start` blocks on `<-stopC`. `Stop` closes `stopC` and optionally does cleanup.
|
||||
|
||||
The JWT tokenizer (`pkg/tokenizer/jwttokenizer/provider.go`) is a good example. It validates and creates tokens on demand via method calls, but has no periodic work to do. It still needs the service lifecycle so the registry can manage its lifetime:
|
||||
|
||||
```go
|
||||
// pkg/tokenizer/jwttokenizer/provider.go
|
||||
|
||||
func (provider *provider) Start(ctx context.Context) error {
|
||||
<-provider.stopC
|
||||
return nil
|
||||
}
|
||||
|
||||
func (provider *provider) Stop(ctx context.Context) error {
|
||||
close(provider.stopC)
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
The instrumentation SDK (`pkg/instrumentation/sdk.go`) is idle while running but does real cleanup in `Stop` shutting down its OpenTelemetry tracer and meter providers:
|
||||
|
||||
```go
|
||||
// pkg/instrumentation/sdk.go
|
||||
|
||||
func (i *SDK) Start(ctx context.Context) error {
|
||||
<-i.startCh
|
||||
return nil
|
||||
}
|
||||
|
||||
func (i *SDK) Stop(ctx context.Context) error {
|
||||
close(i.startCh)
|
||||
return errors.Join(
|
||||
i.sdk.Shutdown(ctx),
|
||||
i.meterProviderShutdownFunc(ctx),
|
||||
)
|
||||
}
|
||||
```
|
||||
|
||||
### Scheduled service
|
||||
|
||||
The service runs an operation repeatedly on a fixed interval. `Start` runs a ticker loop with a `select` on `stopC` and the ticker channel.
|
||||
|
||||
The opaque tokenizer (`pkg/tokenizer/opaquetokenizer/provider.go`) garbage-collects expired tokens and flushes cached last-observed-at timestamps to the database on a configurable interval:
|
||||
|
||||
```go
|
||||
// pkg/tokenizer/opaquetokenizer/provider.go
|
||||
|
||||
func (provider *provider) Start(ctx context.Context) error {
|
||||
ticker := time.NewTicker(provider.config.Opaque.GC.Interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-provider.stopC:
|
||||
return nil
|
||||
case <-ticker.C:
|
||||
orgs, err := provider.orgGetter.ListByOwnedKeyRange(ctx)
|
||||
if err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to get orgs data", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
for _, org := range orgs {
|
||||
if err := provider.gc(ctx, org); err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to garbage collect tokens", "error", err, "org_id", org.ID)
|
||||
}
|
||||
|
||||
if err := provider.flushLastObservedAt(ctx, org); err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to flush tokens", "error", err, "org_id", org.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Its `Stop` does a final gc and flush before returning, so no data is lost on shutdown:
|
||||
|
||||
```go
|
||||
// pkg/tokenizer/opaquetokenizer/provider.go
|
||||
|
||||
func (provider *provider) Stop(ctx context.Context) error {
|
||||
close(provider.stopC)
|
||||
|
||||
orgs, err := provider.orgGetter.ListByOwnedKeyRange(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, org := range orgs {
|
||||
if err := provider.gc(ctx, org); err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to garbage collect tokens", "error", err, "org_id", org.ID)
|
||||
}
|
||||
|
||||
if err := provider.flushLastObservedAt(ctx, org); err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to flush tokens", "error", err, "org_id", org.ID)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
The key points:
|
||||
|
||||
- In the loop, `select` on `stopC` and the ticker. Errors in iterations are logged but do not cause the service to return (which would shut down the application).
|
||||
- Only return an error from `Start` if the failure is unrecoverable.
|
||||
- Use `Stop` to flush or drain any in-memory state before the process exits.
|
||||
|
||||
## How to create a service
|
||||
|
||||
There are two cases: a standalone service and a provider that is also a service.
|
||||
|
||||
### Standalone service
|
||||
|
||||
A standalone service only has the `factory.Service` lifecycle i.e it does not serve as a dependency for other packages. The user reconciliation service is an example.
|
||||
|
||||
1. Define the service interface in your package. Embed `factory.Service`:
|
||||
|
||||
```go
|
||||
// pkg/modules/user/service.go
|
||||
package user
|
||||
|
||||
type Service interface {
|
||||
factory.Service
|
||||
}
|
||||
```
|
||||
|
||||
2. Create the implementation in an `impl` sub-package. Use an unexported struct with an exported constructor that returns the interface:
|
||||
|
||||
```go
|
||||
// pkg/modules/user/impluser/service.go
|
||||
package impluser
|
||||
|
||||
type service struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
// ... dependencies ...
|
||||
stopC chan struct{}
|
||||
}
|
||||
|
||||
func NewService(
|
||||
providerSettings factory.ProviderSettings,
|
||||
// ... dependencies ...
|
||||
) user.Service {
|
||||
return &service{
|
||||
settings: factory.NewScopedProviderSettings(providerSettings, "go.signoz.io/pkg/modules/user"),
|
||||
// ... dependencies ...
|
||||
stopC: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *service) Start(ctx context.Context) error { ... }
|
||||
func (s *service) Stop(ctx context.Context) error { ... }
|
||||
```
|
||||
|
||||
### Provider that is also a service
|
||||
|
||||
Many providers need a managed lifecycle: they poll, sync, or garbage-collect in the background. In this case, embed `factory.Service` in the provider interface. The implementation satisfies both the provider methods and `Start`/`Stop`.
|
||||
|
||||
```go
|
||||
// pkg/tokenizer/tokenizer.go
|
||||
package tokenizer
|
||||
|
||||
type Tokenizer interface {
|
||||
factory.Service
|
||||
CreateToken(context.Context, *authtypes.Identity, map[string]string) (*authtypes.Token, error)
|
||||
GetIdentity(context.Context, string) (*authtypes.Identity, error)
|
||||
// ... other methods ...
|
||||
}
|
||||
```
|
||||
|
||||
The implementation (e.g. `pkg/tokenizer/opaquetokenizer/provider.go`) implements `Start`, `Stop`, and all the provider methods on the same struct. See the [provider guide](provider.md) for how to set up the factory, config, and constructor. The `stopC` channel and `Start`/`Stop` methods follow the same patterns described above.
|
||||
|
||||
## How to wire it up
|
||||
|
||||
Wiring happens in `pkg/signoz/signoz.go`.
|
||||
|
||||
### 1. Instantiate the service
|
||||
|
||||
For a standalone service, call the constructor directly:
|
||||
|
||||
```go
|
||||
userService := impluser.NewService(providerSettings, store, module, orgGetter, authz, config.User.Root)
|
||||
```
|
||||
|
||||
For a provider that is also a service, use `factory.NewProviderFromNamedMap` as described in the [provider guide](provider.md). The returned value already implements `factory.Service`.
|
||||
|
||||
### 2. Register in the registry
|
||||
|
||||
Wrap the service with `factory.NewNamedService` and pass it to `factory.NewRegistry`:
|
||||
|
||||
```go
|
||||
registry, err := factory.NewRegistry(
|
||||
instrumentation.Logger(),
|
||||
// ... other services ...
|
||||
factory.NewNamedService(factory.MustNewName("user"), userService),
|
||||
)
|
||||
```
|
||||
|
||||
The name must be unique across all services. The registry handles the rest:
|
||||
|
||||
- **Start**: launches all services concurrently in goroutines.
|
||||
- **Wait**: blocks until a service returns an error, the context is cancelled, or a SIGINT/SIGTERM is received. Any service error triggers application shutdown.
|
||||
- **Stop**: stops all services concurrently, collects errors via `errors.Join`.
|
||||
|
||||
You do not call `Start` or `Stop` on individual services. The registry does it.
|
||||
|
||||
## What should I remember?
|
||||
|
||||
- A service has a managed lifecycle: `Start` blocks, `Stop` unblocks it.
|
||||
- Use `stopC chan struct{}` for shutdown coordination. `close(stopC)` in `Stop`, `<-stopC` in `Start`.
|
||||
- Service shapes: idle (block on `stopC`) and scheduled (ticker loop with `select`).
|
||||
- Unexported struct, exported `NewService` constructor returning the interface.
|
||||
- First constructor parameter is `factory.ProviderSettings`. Create scoped settings with `factory.NewScopedProviderSettings`.
|
||||
- Register in `factory.Registry` with `factory.NewNamedService`. The registry starts and stops everything.
|
||||
- Only return an error from `Start` if the failure is unrecoverable. Log and continue for transient errors in polling loops.
|
||||
|
||||
## Further reading
|
||||
|
||||
- [Google Guava - ServiceExplained](https://github.com/google/guava/wiki/ServiceExplained) - the service lifecycle pattern takes inspiration from this
|
||||
- [OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector) - Worth studying for its approach to building composable components
|
||||
@@ -1,260 +0,0 @@
|
||||
# Testing
|
||||
|
||||
This document provides rules for writing tests that catch real bugs and do not become a maintenance burden. It covers both how to write good tests and how to recognize bad ones.
|
||||
|
||||
## Why we write tests
|
||||
|
||||
Tests exist to give confidence that the system behaves correctly. A good test suite lets you change code and know immediately (or in a reasonable time) whether you broke something. A bad test suite lets you change code (and then spend hours figuring out whether the failures are real) and still lets the bugs slip in.
|
||||
|
||||
Every test should be written to answer one question: **if this test fails, does that mean a user-visible behavior is broken?** If the answer is no, reconsider whether the test should exist.
|
||||
|
||||
Not all tests are equal. Different scopes serve different purposes, and the balance matters.
|
||||
|
||||
- **Unit tests**: Fast, focused, test a single function or type in isolation. These form the foundation. They should run in milliseconds, have no I/O, and be fully deterministic.
|
||||
- **Integration tests**: Verify that components work together against real dependencies (ClickHouse, PostgreSQL, etc.). Slower, but catch problems that unit tests cannot: real query behavior, configuration issues, serialization mismatches.
|
||||
- **End-to-end tests**: Validate full system behavior from the outside. Expensive to write and maintain, but necessary for critical user flows.
|
||||
|
||||
When a test can be written at a smaller scope, prefer the smaller scope. But do not force a unit test where an integration test is the natural fit.
|
||||
|
||||
## What to test
|
||||
|
||||
### Test behaviors, not implementations
|
||||
|
||||
A test should verify what the code does, not how it does it (unless the goal of the test is specifically how something happen). If you can refactor the internals of a function e.g, change a query, rename a variable, restructure the logic and no user-visible behavior changes, no test should break.
|
||||
|
||||
```go
|
||||
// Good: tests the behavior "given this input, expect this output."
|
||||
func TestDiscountApplied(t *testing.T) {
|
||||
order := NewOrder(item("widget", 100))
|
||||
order.ApplyDiscount(10)
|
||||
assert.Equal(t, 90, order.Total())
|
||||
}
|
||||
|
||||
// Bad: tests the implementation "did it call the right internal method?"
|
||||
func TestDiscountApplied(t *testing.T) {
|
||||
mockPricer := new(MockPricer)
|
||||
mockPricer.On("CalculateDiscount", 100, 10).Return(90)
|
||||
order := NewOrder(item("widget", 100), WithPricer(mockPricer))
|
||||
order.ApplyDiscount(10)
|
||||
mockPricer.AssertCalled(t, "CalculateDiscount", 100, 10)
|
||||
}
|
||||
```
|
||||
|
||||
The first test survives a refactoring of how discounts are calculated. The second test breaks the moment you rename the method, change its signature, or inline the logic.
|
||||
|
||||
**The refactoring test**: before committing a test, ask if someone refactors the internals tomorrow without changing any behavior, will this test break? If yes, consider updating the test.
|
||||
|
||||
### Output format as behavior
|
||||
|
||||
Some functions exist specifically to produce a formatted output: a query builder generates SQL, a serializer generates JSON, a code generator produces source code. In these cases, the output string *is* the behavior and asserting on it is valid and necessary. The function's contract is the exact output it produces.
|
||||
|
||||
This is different from testing a function that *uses* a query internally. If a function's job is to fetch data from a database, the query it sends is an implementation detail and the returned data is the behavior. If its job is to *build* a query for someone else to execute, the query string is the behavior.
|
||||
|
||||
The distinction: **is the formatted output the function's product, or the function's mechanism?** Test the product, not the mechanism.
|
||||
|
||||
### Test at the public API boundary
|
||||
|
||||
Write tests against the exported functions and methods that consumers actually call. Do not test unexported helpers directly. If an unexported function has complex logic worth testing, that is a signal it should be extracted into its own package with its own public API.
|
||||
|
||||
### Test edge cases and error paths
|
||||
|
||||
The most valuable tests cover the cases that are easy to get wrong:
|
||||
|
||||
- Empty inputs, nil inputs, zero values.
|
||||
- Boundary conditions (off-by-one, first element, last element).
|
||||
- Error conditions (what happens when the dependency fails?).
|
||||
- Concurrent access, if the code is designed for it.
|
||||
|
||||
A test for the happy path of a trivial function adds little value. A test for the error path of a complex function prevents real bugs.
|
||||
|
||||
### The Beyonce Rule
|
||||
|
||||
"If you liked it, then you should have put a test on it." Any behavior you want to preserve such as correctness, performance characteristics, security constraints, error handling should be covered by a test. If it breaks and there is no test, that is not a regression; it is an untested assumption.
|
||||
|
||||
## How to write a test
|
||||
|
||||
### Structure: arrange, act, assert
|
||||
|
||||
Every test should have three clearly separated sections:
|
||||
|
||||
```go
|
||||
func TestTransferInsufficientFunds(t *testing.T) {
|
||||
// Arrange: set up the preconditions.
|
||||
from := NewAccount(50)
|
||||
to := NewAccount(0)
|
||||
|
||||
// Act: perform the operation being tested.
|
||||
err := Transfer(from, to, 100)
|
||||
|
||||
// Assert: verify the outcome.
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, 50, from.Balance())
|
||||
assert.Equal(t, 0, to.Balance())
|
||||
}
|
||||
```
|
||||
|
||||
Do not interleave setup and assertions. Do not put assertions in helper functions that also perform setup. Keep the three sections visually distinct.
|
||||
|
||||
### One behavior per test
|
||||
|
||||
Each test function should verify one behavior. If a test name needs "and" in it, split it into two tests.
|
||||
|
||||
```go
|
||||
// Good: one behavior per test.
|
||||
func TestParseValidInput(t *testing.T) { ... }
|
||||
func TestParseEmptyInput(t *testing.T) { ... }
|
||||
func TestParseMalformedInput(t *testing.T) { ... }
|
||||
|
||||
// Bad: multiple behaviors in one test.
|
||||
func TestParse(t *testing.T) {
|
||||
// test valid input
|
||||
// test empty input
|
||||
// test malformed input
|
||||
}
|
||||
```
|
||||
|
||||
Table-driven tests are fine when the behavior is the same and only the inputs/outputs vary.
|
||||
|
||||
### Name tests after behaviors
|
||||
|
||||
Test names should describe the scenario and the expected outcome, not the function being tested.
|
||||
|
||||
```go
|
||||
// Good: describes the behavior.
|
||||
func TestWithdrawal_InsufficientFunds_ReturnsError(t *testing.T)
|
||||
func TestWithdrawal_ZeroBalance_ReturnsError(t *testing.T)
|
||||
|
||||
// Bad: describes the function.
|
||||
func TestWithdraw(t *testing.T)
|
||||
func TestWithdrawError(t *testing.T)
|
||||
```
|
||||
|
||||
### Eliminate logic in tests
|
||||
|
||||
Tests should be straight-line code. No `if`, no `for`, no `switch`. If you feel the need to add control flow to a test, either split it into multiple tests or restructure the test data.
|
||||
|
||||
A test with logic in it needs its own tests. That is a sign something has gone wrong.
|
||||
|
||||
### Write clear failure messages
|
||||
|
||||
When a test fails, the failure message should tell you what went wrong without reading the test source.
|
||||
|
||||
```go
|
||||
// Good: failure message explains the context.
|
||||
assert.Equal(t, expected, actual, "discount should be applied to order total")
|
||||
|
||||
// Bad: failure message is just the default.
|
||||
assert.Equal(t, expected, actual)
|
||||
```
|
||||
|
||||
Use `require` for preconditions that must hold for the rest of the test to make sense. Use `assert` for the actual verifications. This avoids cascading failures from a single root cause.
|
||||
|
||||
## How to recognize a bad test
|
||||
|
||||
A bad test costs more to maintain than the bugs it prevents. Learning to identify bad tests is as important as learning to write good ones. Always evaluate a test critically before commiting it.
|
||||
|
||||
### Tests that duplicate the implementation
|
||||
|
||||
If a test contains the same logic as the code it tests, it verifies nothing. It will pass when the code is wrong in the same way the test is wrong, and it will break whenever the code changes even if the change is correct.
|
||||
|
||||
A common form: mocking a database, setting up canned rows, calling a function that queries and scans those rows, then asserting that the function returned exactly those rows. The test encodes the query, the row structure, and the scan logic. The same things the production code does. If the function has no branching logic beyond "query and scan," this test is a mirror of the implementation, not a check on it. An integration test against a real database verifies the actual behavior; the mock-based test verifies that the code matches the test author's expectations of the code.
|
||||
|
||||
### Tests for functions with no interesting logic
|
||||
|
||||
Not every function needs a test. A function that prepares a query, sends it, and scans the result has no branching, no edge cases, and no logic that could be wrong independently of the query being correct. Unit-testing it means mocking the database, which means the test does not verify the query works. It only verifies the function calls the mock in the expected way.
|
||||
|
||||
Ask: **what bug would this test catch that would not be caught by the integration test or by the tests of the calling code?** If the answer is nothing, skip the unit test. A missing test is better than a test that provides false confidence.
|
||||
|
||||
### Tests that rebuild the dependency boundary
|
||||
|
||||
When a test creates an in-package mock of an external interface (database driver, HTTP client, file system) and that mock contains non-trivial logic (reflection-based scanning, response simulation, state machines), the test is now testing its own mock as much as the production code. Bugs in the mock produce false passes or false failures, and the mock must be maintained alongside the real dependency.
|
||||
|
||||
If the mock is complex enough to have its own bugs, you have rebuilt the dependency boundary rather than testing against it. Use the real dependency (via integration test) or use a well-maintained fake provided by the dependency's authors.
|
||||
|
||||
### Tests that exist for coverage
|
||||
|
||||
A test that exercises a function without meaningfully verifying its output adds coverage without adding confidence. Calling a type-conversion function with every numeric type and asserting it does not panic covers lines but does not catch regressions. The function would need to be rewritten to fail, and any such rewrite would be caught by the callers' tests.
|
||||
|
||||
Before writing a test, identify the specific failure mode it guards against. If you cannot name one, the test is not worth writing.
|
||||
|
||||
### Tests that test the language
|
||||
|
||||
Do not test that language type system, standard library, or well-known third-party libraries work correctly. Testing that `reflect.Kind` returns the right value for each type, that pointer dereferencing works, or that a type switch dispatches correctly adds maintenance burden without catching any plausible bug in your code.
|
||||
|
||||
## Brittle tests
|
||||
|
||||
A brittle test is one that fails when production code changes without an actual bug being introduced. Brittle tests are expensive: they slow down development, train people to ignore failures, and provide no real safety net. Common sources of brittleness:
|
||||
|
||||
- **Asserting on implementation details**: Verifying which internal methods were called, in what order, or with what intermediate values. If the method is renamed or the order changes but the output is the same, the test breaks for no reason.
|
||||
- **Asserting on serialized representations when the format is not the contract**: Matching exact SQL strings, JSON output, or log messages produced by a function whose job is not to produce that format.
|
||||
- **Over-constrained mocks**: Setting up a mock that expects specific arguments in a specific sequence. Any refactoring of the call pattern breaks the mock setup even if behavior is preserved.
|
||||
- **Shared mutable state**: Tests that depend on data left behind by other tests. A change in execution order or a new test case causes unrelated failures.
|
||||
- **Time-dependence**: Tests that use `time.Now()`, `time.Sleep()`, or real timers. These produce flaky results and break under load.
|
||||
|
||||
When you encounter a brittle test, fix or delete it. Do not work around it.
|
||||
|
||||
## DAMP
|
||||
|
||||
Test code should prioritize clarity (DAMP: Descriptive And Meaningful Phrases).
|
||||
|
||||
```go
|
||||
// DAMP: each test is self-contained and readable.
|
||||
func TestCreateUser(t *testing.T) {
|
||||
user := User{Name: "Alice", Email: "alice@example.com"}
|
||||
err := store.Create(ctx, user)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestCreateDuplicateUser(t *testing.T) {
|
||||
user := User{Name: "Alice", Email: "alice@example.com"}
|
||||
_ = store.Create(ctx, user)
|
||||
err := store.Create(ctx, user)
|
||||
assert.ErrorIs(t, err, ErrAlreadyExists)
|
||||
}
|
||||
```
|
||||
|
||||
Shared setup helpers are fine for constructing objects with sensible defaults. But each test should explicitly set the values it depends on rather than relying on hidden defaults in a shared fixture.
|
||||
|
||||
## Flaky tests
|
||||
|
||||
A flaky test is one that sometimes passes and sometimes fails without any code change. Flaky tests erode trust in the entire suite. Once people learn to re-run and ignore failures, real bugs slip through.
|
||||
|
||||
Common causes and fixes:
|
||||
|
||||
- **Timing and sleeps**: Replace `time.Sleep` with channels, condition variables, or polling with a timeout.
|
||||
- **Uncontrolled concurrency**: Use deterministic synchronization rather than relying on goroutine scheduling.
|
||||
- **Shared state between tests**: Each test should set up and tear down its own state.
|
||||
|
||||
If a test is flaky and you cannot fix the root cause quickly, skip or delete it. A skipped test with an explanation is better than a flaky test that trains everyone to ignore red builds.
|
||||
|
||||
## Code coverage
|
||||
|
||||
Code coverage measures which lines were executed, not whether the code is correct. A function that is called but whose output is never checked has 100% coverage and 0% verification.
|
||||
|
||||
Do not use coverage as a target to hit. Use it as a tool to find gaps such as untested error paths, unreachable branches, dead code. A codebase with 60% meaningful coverage is better than one with 95% coverage achieved by testing trivial getters.
|
||||
|
||||
## Tests are code
|
||||
|
||||
Tests must be maintained and they are not second-class citizen. You should apply the same standards for readability, naming, and structure that you apply to production code. We do not tolerate complexity in tests just because they are tests.
|
||||
|
||||
However, tests should be simpler than production code. If a test requires its own helper library, complex setup, or nested control flow, step back and ask whether you are testing the right thing at the right level. This is not a blanket rule but a prompt to pause, assess the situation, and check whether the complexity is justified.
|
||||
|
||||
## What should I remember?
|
||||
|
||||
- If refactoring internals breaks your test but no behavior changed, the test is likely bad. Delete it or consider updating it.
|
||||
- Test what the code does, not how it does it. Verify outputs and state, not method calls.
|
||||
- Output format is behavior when the function's job is to produce that format. It is not behavior when the function uses it internally.
|
||||
- Ask what specific bug this test catches. If you cannot name one, do not write it.
|
||||
- Always evaluate whether the test adds confidence, not just lines.
|
||||
- One behavior per test. Name it after the scenario, not the function.
|
||||
- No logic in tests. Straight-line code only.
|
||||
- Flaky tests are not acceptable. Fix the root cause or nuke the test code.
|
||||
- Coverage measures execution, not correctness.
|
||||
|
||||
## Mandatory reading
|
||||
|
||||
- What to look for in a code review: Tests - https://google.github.io/eng-practices/review/reviewer/looking-for.html#tests
|
||||
- Testing Overview - https://abseil.io/resources/swe-book/html/ch11.html
|
||||
- Unit Testing - https://abseil.io/resources/swe-book/html/ch12.html
|
||||
- Test Doubles - https://abseil.io/resources/swe-book/html/ch13.html
|
||||
- Larger Testing - https://abseil.io/resources/swe-book/html/ch14.html
|
||||
@@ -1,272 +0,0 @@
|
||||
# Types
|
||||
|
||||
This guide covers how types are organised, named, constructed, and composed so you can add new ones consistently.
|
||||
|
||||
## Where do types live?
|
||||
|
||||
Types live in `pkg/types/` and its sub-packages:
|
||||
|
||||
```
|
||||
pkg/types/
|
||||
├── auditable.go # TimeAuditable, UserAuditable
|
||||
├── identity.go # Identifiable (UUID primary key)
|
||||
├── user.go # User, PostableRegisterOrgAndAdmin, UserStore
|
||||
├── alertmanagertypes/ # Alert manager domain types
|
||||
│ ├── channel.go
|
||||
│ ├── receiver.go
|
||||
│ └── config.go
|
||||
├── authtypes/ # Auth domain types
|
||||
└── ruletypes/ # Rule domain types
|
||||
└── maintenance.go
|
||||
```
|
||||
|
||||
Follow these rules:
|
||||
|
||||
1. **Embeddable building blocks** go in `pkg/types/` directly `Identifiable`, `TimeAuditable`, `UserAuditable`.
|
||||
2. **Domain-specific types** go in a sub-package named `pkg/types/<domain>types/` (e.g., `alertmanagertypes`, `ruletypes`, `authtypes`).
|
||||
3. **No domain logic** in type packages. Only data structures, constants, and simple methods. Domain services import from type packages, not the other way around.
|
||||
4. **Domain services import types, not vice versa.** If a type needs a service, the design is likely wrong and you should restructure so the service operates on the type.
|
||||
|
||||
## Type variants
|
||||
|
||||
A domain entity often has multiple representations depending on where it appears in the system. We use naming prefixes to distinguish them:
|
||||
|
||||
| Prefix | Purpose | Example |
|
||||
|---|---|---|
|
||||
| `Postable<Type>` | API request input | `PostableRegisterOrgAndAdmin` |
|
||||
| `Gettable<Type>` | API response output | `GettablePlannedMaintenance` |
|
||||
| `Storable<Type>` | Database model (embeds `bun.BaseModel`) | `StorablePlannedMaintenance` |
|
||||
| Plain `<Type>` | Domain logic type | `User` |
|
||||
|
||||
Not every entity needs all four variants. Start with the plain type and add variants only when the API or database representation genuinely differs.
|
||||
|
||||
Here is a concrete example from `pkg/types/ruletypes/maintenance.go`:
|
||||
|
||||
```go
|
||||
// Database model embeds bun.BaseModel and composition types
|
||||
type StorablePlannedMaintenance struct {
|
||||
bun.BaseModel `bun:"table:planned_maintenance"`
|
||||
types.Identifiable
|
||||
types.TimeAuditable
|
||||
types.UserAuditable
|
||||
Name string `bun:"name,type:text,notnull"`
|
||||
Description string `bun:"description,type:text"`
|
||||
Schedule *Schedule `bun:"schedule,type:text,notnull"`
|
||||
OrgID string `bun:"org_id,type:text"`
|
||||
}
|
||||
|
||||
// API response: flat struct with JSON tags, computed fields like Status
|
||||
type GettablePlannedMaintenance struct {
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Schedule *Schedule `json:"schedule"`
|
||||
RuleIDs []string `json:"alertIds"`
|
||||
CreatedAt time.Time `json:"createdAt"`
|
||||
CreatedBy string `json:"createdBy"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
UpdatedBy string `json:"updatedBy"`
|
||||
Status string `json:"status"`
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
```
|
||||
|
||||
When the API shape exactly matches the domain type, use a type alias instead of duplicating fields:
|
||||
|
||||
```go
|
||||
// From pkg/types/user.go
|
||||
type GettableUser = User
|
||||
```
|
||||
|
||||
## Composition via embedding
|
||||
|
||||
`pkg/types/` provides small, reusable structs that you embed into your domain types:
|
||||
|
||||
```go
|
||||
// pkg/types/identity.go
|
||||
type Identifiable struct {
|
||||
ID valuer.UUID `json:"id" bun:"id,pk,type:text"`
|
||||
}
|
||||
|
||||
// pkg/types/auditable.go
|
||||
type TimeAuditable struct {
|
||||
CreatedAt time.Time `bun:"created_at" json:"createdAt"`
|
||||
UpdatedAt time.Time `bun:"updated_at" json:"updatedAt"`
|
||||
}
|
||||
|
||||
type UserAuditable struct {
|
||||
CreatedBy string `bun:"created_by,type:text" json:"createdBy"`
|
||||
UpdatedBy string `bun:"updated_by,type:text" json:"updatedBy"`
|
||||
}
|
||||
```
|
||||
|
||||
Compose them in a database model:
|
||||
|
||||
```go
|
||||
type StorablePlannedMaintenance struct {
|
||||
bun.BaseModel `bun:"table:planned_maintenance"`
|
||||
types.Identifiable // adds ID (UUID primary key)
|
||||
types.TimeAuditable // adds CreatedAt, UpdatedAt
|
||||
types.UserAuditable // adds CreatedBy, UpdatedBy
|
||||
Name string `bun:"name,type:text,notnull"`
|
||||
Description string `bun:"description,type:text"`
|
||||
}
|
||||
```
|
||||
|
||||
See [SQL](sql.md) for full database patterns including migrations and queries.
|
||||
|
||||
## Constructors
|
||||
|
||||
Constructors validate inputs and return a ready-to-use value:
|
||||
|
||||
```go
|
||||
// New<Type> validates and returns a pointer + error
|
||||
func NewUser(displayName string, email valuer.Email, role Role, orgID valuer.UUID) (*User, error) {
|
||||
if email.IsZero() {
|
||||
return nil, errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "email is required")
|
||||
}
|
||||
if role == "" {
|
||||
return nil, errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "role is required")
|
||||
}
|
||||
if orgID.IsZero() {
|
||||
return nil, errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "orgID is required")
|
||||
}
|
||||
|
||||
return &User{
|
||||
Identifiable: Identifiable{ID: valuer.GenerateUUID()},
|
||||
DisplayName: displayName,
|
||||
Email: email,
|
||||
Role: role,
|
||||
OrgID: orgID,
|
||||
TimeAuditable: TimeAuditable{CreatedAt: time.Now(), UpdatedAt: time.Now()},
|
||||
}, nil
|
||||
}
|
||||
```
|
||||
|
||||
Follow these conventions:
|
||||
|
||||
- **`New<Type>(args) (*Type, error)`**: validates inputs, returns an error on failure. Use this in production code.
|
||||
- **Validation at construction**: check required fields, format constraints, and invariants in the constructor. Callers should not need to validate after construction.
|
||||
- **Generate IDs internally**: constructors call `valuer.GenerateUUID()` callers do not pass IDs in.
|
||||
- **Set timestamps internally**: constructors set `CreatedAt` and `UpdatedAt` to `time.Now()`.
|
||||
|
||||
## Typed domain values (`pkg/valuer/`)
|
||||
|
||||
The `pkg/valuer` package provides typed wrappers for common domain values. These types carry validation, normalization, and consistent serialization (JSON, SQL, text) that raw Go primitives do not.
|
||||
|
||||
| Type | Wraps | Invariant |
|
||||
|---|---|---|
|
||||
| `valuer.UUID` | `google/uuid.UUID` | Valid UUIDv7, generated via `GenerateUUID()` |
|
||||
| `valuer.Email` | `string` | Valid email format, lowercased and trimmed |
|
||||
| `valuer.String` | `string` | Lowercased and trimmed |
|
||||
| `valuer.TextDuration` | `time.Duration` | Valid duration, text-serializable |
|
||||
|
||||
### When to use a valuer type
|
||||
|
||||
Use a valuer type instead of a raw primitive when the value represents a domain concept with any of:
|
||||
|
||||
- **Enums**: All enums in the codebase must be backed by `valuer.String`. Do not use raw `string` constants or `iota`-based `int` enums. A struct embedding `valuer.String` with predefined variables gives you normalization, serialization, and an `Enum()` method for OpenAPI schema generation in one place.
|
||||
- **Validation**: emails must match a format, UUIDs must be parseable, durations must be valid.
|
||||
- **Normalization**: `valuer.String` lowercases and trims input, so comparisons are consistent throughout the system.
|
||||
- **Serialization boundary**: the value is stored in a database, sent over the wire, or bound from an HTTP parameter. Valuer types implement `Scan`, `Value`, `MarshalJSON`, `UnmarshalJSON`, and `UnmarshalParam` consistently.
|
||||
|
||||
```go
|
||||
// Wrong: raw string constant with no validation or normalization.
|
||||
const SignalTraces = "traces"
|
||||
|
||||
// Right: valuer-backed type that normalizes and serializes consistently.
|
||||
type Signal struct {
|
||||
valuer.String
|
||||
}
|
||||
|
||||
var SignalTraces = Signal{valuer.NewString("traces")}
|
||||
```
|
||||
|
||||
Only primitive domain types that serve as shared infrastructure belong in `pkg/valuer`. If you need a new base type (like `Email` or `TextDuration`) that multiple packages will embed for validation and serialization, add it there. Domain-specific types that build on top of a valuer (like `Signal` embedding `valuer.String`) belong in their own domain package, not in `pkg/valuer`.
|
||||
|
||||
### The `Valuer` interface
|
||||
|
||||
Every valuer type implements the `Valuer` interface, which gives you serialization for free:
|
||||
|
||||
```go
|
||||
type Valuer interface {
|
||||
IsZero() bool // check for zero value
|
||||
StringValue() string // raw string representation
|
||||
fmt.Stringer // String() for printing
|
||||
json.Marshaler / json.Unmarshaler // JSON
|
||||
sql.Scanner / driver.Valuer // database
|
||||
encoding.TextMarshaler / TextUnmarshaler // text
|
||||
ginbinding.BindUnmarshaler // HTTP query/path params
|
||||
}
|
||||
```
|
||||
|
||||
Use them in struct fields:
|
||||
|
||||
```go
|
||||
type User struct {
|
||||
Identifiable
|
||||
Email valuer.Email `bun:"email" json:"email"`
|
||||
OrgID valuer.UUID `bun:"org_id" json:"orgId"`
|
||||
}
|
||||
```
|
||||
|
||||
## Wrappers must add semantics, not just rename
|
||||
|
||||
A wrapper type is justified when it adds meaning, validation, or invariants that the underlying type does not carry. It is not justified when it merely renames fields or reorganizes the same data into a different shape.
|
||||
|
||||
```go
|
||||
// Justified: adds validation that the underlying string does not carry.
|
||||
type OrgID struct{ value string }
|
||||
func NewOrgID(s string) (OrgID, error) { /* validates format */ }
|
||||
|
||||
// Not justified: renames fields with no new invariant or behavior.
|
||||
type UserInfo struct {
|
||||
Name string // same as source.Name
|
||||
Email string // same as source.Email
|
||||
}
|
||||
```
|
||||
|
||||
Ask: what does the wrapper guarantee that the underlying type does not? If the answer is nothing, use the underlying type directly.
|
||||
|
||||
## When a new type IS warranted
|
||||
|
||||
A new type earns its place when it meets **at least one** of these criteria:
|
||||
|
||||
- **Serialization boundary**: It must be persisted, sent over the wire, or written to config. The source type is unsuitable (unexported fields, function pointers, cycles).
|
||||
- **Invariant enforcement**: The constructor or methods enforce constraints that raw data does not carry (e.g., non-empty, validated format, bounded range).
|
||||
- **Multiple distinct consumers**: Three or more call sites use the type in meaningfully different ways. The type is the shared vocabulary between them.
|
||||
- **Dependency firewall**: The type lives in a lightweight package so that consumers avoid importing a heavy dependency.
|
||||
|
||||
See [Abstractions](abstractions.md) for the full set of rules on when abstractions are and aren't justified.
|
||||
|
||||
## Store interfaces
|
||||
|
||||
Each domain type package defines a store interface for persistence. The store interface lives alongside the types it operates on:
|
||||
|
||||
```go
|
||||
// From pkg/types/ruletypes/maintenance.go
|
||||
type MaintenanceStore interface {
|
||||
CreatePlannedMaintenance(context.Context, GettablePlannedMaintenance) (valuer.UUID, error)
|
||||
DeletePlannedMaintenance(context.Context, valuer.UUID) error
|
||||
GetPlannedMaintenanceByID(context.Context, valuer.UUID) (*GettablePlannedMaintenance, error)
|
||||
EditPlannedMaintenance(context.Context, GettablePlannedMaintenance, valuer.UUID) error
|
||||
GetAllPlannedMaintenance(context.Context, string) ([]*GettablePlannedMaintenance, error)
|
||||
}
|
||||
```
|
||||
|
||||
Conventions:
|
||||
|
||||
- Name the interface `<Domain>Store` (e.g., `UserStore`, `MaintenanceStore`).
|
||||
- Accept `context.Context` as the first parameter.
|
||||
- Use typed values (`valuer.UUID`, `valuer.Email`) instead of raw strings for identifiers.
|
||||
- Implementations go in separate packages (e.g., `sqlstore/`), see [SQL](sql.md) for details.
|
||||
|
||||
## What should I remember?
|
||||
|
||||
- Shared types live in `pkg/types/`, domain types in `pkg/types/<domain>types/`.
|
||||
- No domain logic in type packages only data structures, constants, and simple methods.
|
||||
- Use `Storable`, `Gettable`, `Postable` prefixes when API or database representation differs from the domain type.
|
||||
- Embed `Identifiable`, `TimeAuditable`, and `UserAuditable` for standard fields instead of repeating them.
|
||||
- Constructors (`New<Type>`) validate, generate IDs, and set timestamps.
|
||||
- Use `pkg/valuer/` types instead of raw strings for domain identifiers like UUIDs and emails.
|
||||
- Store interfaces live alongside the types they persist and use `context.Context` as the first parameter.
|
||||
@@ -19,8 +19,6 @@ const config: Config.InitialOptions = {
|
||||
'^.*/useSafeNavigate$': USE_SAFE_NAVIGATE_MOCK_PATH,
|
||||
'^@signozhq/icons$':
|
||||
'<rootDir>/node_modules/@signozhq/icons/dist/index.esm.js',
|
||||
'^react-syntax-highlighter/dist/esm/(.*)$':
|
||||
'<rootDir>/node_modules/react-syntax-highlighter/dist/cjs/$1',
|
||||
},
|
||||
globals: {
|
||||
extensionsToTreatAsEsm: ['.ts'],
|
||||
|
||||
@@ -214,7 +214,7 @@
|
||||
"@types/react-redux": "^7.1.11",
|
||||
"@types/react-resizable": "3.0.3",
|
||||
"@types/react-router-dom": "^5.1.6",
|
||||
"@types/react-syntax-highlighter": "15.5.13",
|
||||
"@types/react-syntax-highlighter": "15.5.7",
|
||||
"@types/redux-mock-store": "1.0.4",
|
||||
"@types/styled-components": "^5.1.4",
|
||||
"@types/uuid": "^8.3.1",
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Extracts unique fenced code block language identifiers from all .md files under frontend/src/
|
||||
# Usage: bash frontend/scripts/extract-md-languages.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SRC_DIR="$SCRIPT_DIR/../src"
|
||||
|
||||
grep -roh '```[a-zA-Z0-9_+-]*' "$SRC_DIR" --include='*.md' \
|
||||
| sed 's/^```//' \
|
||||
| grep -v '^$' \
|
||||
| sort -u
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Validates that all fenced code block languages used in .md files are registered
|
||||
# in the syntax highlighter.
|
||||
# Usage: bash frontend/scripts/validate-md-languages.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
SYNTAX_HIGHLIGHTER="$SCRIPT_DIR/../src/components/MarkdownRenderer/syntaxHighlighter.ts"
|
||||
|
||||
# Get all languages used in .md files
|
||||
md_languages=$("$SCRIPT_DIR/extract-md-languages.sh")
|
||||
|
||||
# Get all registered languages from syntaxHighlighter.ts
|
||||
registered_languages=$(grep -oP "registerLanguage\('\K[^']+" "$SYNTAX_HIGHLIGHTER" | sort -u)
|
||||
|
||||
missing_languages=()
|
||||
|
||||
for lang in $md_languages; do
|
||||
if ! echo "$registered_languages" | grep -qx "$lang"; then
|
||||
missing_languages+=("$lang")
|
||||
fi
|
||||
done
|
||||
|
||||
if [ ${#missing_languages[@]} -gt 0 ]; then
|
||||
echo "Error: The following languages are used in .md files but not registered in syntaxHighlighter.ts:"
|
||||
for lang in "${missing_languages[@]}"; do
|
||||
echo " - $lang"
|
||||
done
|
||||
echo ""
|
||||
echo "Please add them to: frontend/src/components/MarkdownRenderer/syntaxHighlighter.ts"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "All markdown code block languages are registered in syntaxHighlighter.ts"
|
||||
@@ -20,8 +20,11 @@ import { useMutation, useQuery } from 'react-query';
|
||||
import type { BodyType, ErrorType } from '../../../generatedAPIInstance';
|
||||
import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
|
||||
import type {
|
||||
HandleExportRawDataGETParams,
|
||||
HandleExportRawDataPOSTParams,
|
||||
ListPromotedAndIndexedPaths200,
|
||||
PromotetypesPromotePathDTO,
|
||||
Querybuildertypesv5QueryRangeRequestDTO,
|
||||
RenderErrorResponseDTO,
|
||||
} from '../sigNoz.schemas';
|
||||
|
||||
@@ -29,6 +32,206 @@ type AwaitedInput<T> = PromiseLike<T> | T;
|
||||
|
||||
type Awaited<O> = O extends AwaitedInput<infer T> ? T : never;
|
||||
|
||||
/**
|
||||
* This endpoints allows simple query exporting raw data for traces and logs
|
||||
* @summary Export raw data
|
||||
*/
|
||||
export const handleExportRawDataGET = (
|
||||
params?: HandleExportRawDataGETParams,
|
||||
signal?: AbortSignal,
|
||||
) => {
|
||||
return GeneratedAPIInstance<string>({
|
||||
url: `/api/v1/export_raw_data`,
|
||||
method: 'GET',
|
||||
params,
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
export const getHandleExportRawDataGETQueryKey = (
|
||||
params?: HandleExportRawDataGETParams,
|
||||
) => {
|
||||
return [`/api/v1/export_raw_data`, ...(params ? [params] : [])] as const;
|
||||
};
|
||||
|
||||
export const getHandleExportRawDataGETQueryOptions = <
|
||||
TData = Awaited<ReturnType<typeof handleExportRawDataGET>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>
|
||||
>(
|
||||
params?: HandleExportRawDataGETParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof handleExportRawDataGET>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
) => {
|
||||
const { query: queryOptions } = options ?? {};
|
||||
|
||||
const queryKey =
|
||||
queryOptions?.queryKey ?? getHandleExportRawDataGETQueryKey(params);
|
||||
|
||||
const queryFn: QueryFunction<
|
||||
Awaited<ReturnType<typeof handleExportRawDataGET>>
|
||||
> = ({ signal }) => handleExportRawDataGET(params, signal);
|
||||
|
||||
return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
|
||||
Awaited<ReturnType<typeof handleExportRawDataGET>>,
|
||||
TError,
|
||||
TData
|
||||
> & { queryKey: QueryKey };
|
||||
};
|
||||
|
||||
export type HandleExportRawDataGETQueryResult = NonNullable<
|
||||
Awaited<ReturnType<typeof handleExportRawDataGET>>
|
||||
>;
|
||||
export type HandleExportRawDataGETQueryError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Export raw data
|
||||
*/
|
||||
|
||||
export function useHandleExportRawDataGET<
|
||||
TData = Awaited<ReturnType<typeof handleExportRawDataGET>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>
|
||||
>(
|
||||
params?: HandleExportRawDataGETParams,
|
||||
options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof handleExportRawDataGET>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
},
|
||||
): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
|
||||
const queryOptions = getHandleExportRawDataGETQueryOptions(params, options);
|
||||
|
||||
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
|
||||
queryKey: QueryKey;
|
||||
};
|
||||
|
||||
query.queryKey = queryOptions.queryKey;
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
/**
|
||||
* @summary Export raw data
|
||||
*/
|
||||
export const invalidateHandleExportRawDataGET = async (
|
||||
queryClient: QueryClient,
|
||||
params?: HandleExportRawDataGETParams,
|
||||
options?: InvalidateOptions,
|
||||
): Promise<QueryClient> => {
|
||||
await queryClient.invalidateQueries(
|
||||
{ queryKey: getHandleExportRawDataGETQueryKey(params) },
|
||||
options,
|
||||
);
|
||||
|
||||
return queryClient;
|
||||
};
|
||||
|
||||
/**
|
||||
* This endpoints allows complex query exporting raw data for traces and logs
|
||||
* @summary Export raw data
|
||||
*/
|
||||
export const handleExportRawDataPOST = (
|
||||
querybuildertypesv5QueryRangeRequestDTO: BodyType<Querybuildertypesv5QueryRangeRequestDTO>,
|
||||
params?: HandleExportRawDataPOSTParams,
|
||||
signal?: AbortSignal,
|
||||
) => {
|
||||
return GeneratedAPIInstance<string>({
|
||||
url: `/api/v1/export_raw_data`,
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
data: querybuildertypesv5QueryRangeRequestDTO,
|
||||
params,
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
export const getHandleExportRawDataPOSTMutationOptions = <
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
TContext = unknown
|
||||
>(options?: {
|
||||
mutation?: UseMutationOptions<
|
||||
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
|
||||
TError,
|
||||
{
|
||||
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
|
||||
params?: HandleExportRawDataPOSTParams;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
}): UseMutationOptions<
|
||||
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
|
||||
TError,
|
||||
{
|
||||
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
|
||||
params?: HandleExportRawDataPOSTParams;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
const mutationKey = ['handleExportRawDataPOST'];
|
||||
const { mutation: mutationOptions } = options
|
||||
? options.mutation &&
|
||||
'mutationKey' in options.mutation &&
|
||||
options.mutation.mutationKey
|
||||
? options
|
||||
: { ...options, mutation: { ...options.mutation, mutationKey } }
|
||||
: { mutation: { mutationKey } };
|
||||
|
||||
const mutationFn: MutationFunction<
|
||||
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
|
||||
{
|
||||
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
|
||||
params?: HandleExportRawDataPOSTParams;
|
||||
}
|
||||
> = (props) => {
|
||||
const { data, params } = props ?? {};
|
||||
|
||||
return handleExportRawDataPOST(data, params);
|
||||
};
|
||||
|
||||
return { mutationFn, ...mutationOptions };
|
||||
};
|
||||
|
||||
export type HandleExportRawDataPOSTMutationResult = NonNullable<
|
||||
Awaited<ReturnType<typeof handleExportRawDataPOST>>
|
||||
>;
|
||||
export type HandleExportRawDataPOSTMutationBody = BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
|
||||
export type HandleExportRawDataPOSTMutationError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Export raw data
|
||||
*/
|
||||
export const useHandleExportRawDataPOST = <
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
TContext = unknown
|
||||
>(options?: {
|
||||
mutation?: UseMutationOptions<
|
||||
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
|
||||
TError,
|
||||
{
|
||||
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
|
||||
params?: HandleExportRawDataPOSTParams;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
}): UseMutationResult<
|
||||
Awaited<ReturnType<typeof handleExportRawDataPOST>>,
|
||||
TError,
|
||||
{
|
||||
data: BodyType<Querybuildertypesv5QueryRangeRequestDTO>;
|
||||
params?: HandleExportRawDataPOSTParams;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
const mutationOptions = getHandleExportRawDataPOSTMutationOptions(options);
|
||||
|
||||
return useMutation(mutationOptions);
|
||||
};
|
||||
/**
|
||||
* This endpoints promotes and indexes paths
|
||||
* @summary Promote and index paths
|
||||
|
||||
@@ -2732,6 +2732,76 @@ export type DeleteAuthDomainPathParameters = {
|
||||
export type UpdateAuthDomainPathParameters = {
|
||||
id: string;
|
||||
};
|
||||
export type HandleExportRawDataGETParams = {
|
||||
/**
|
||||
* @enum csv,jsonl
|
||||
* @type string
|
||||
* @description undefined
|
||||
*/
|
||||
format?: HandleExportRawDataGETFormat;
|
||||
/**
|
||||
* @enum logs,traces
|
||||
* @type string
|
||||
* @description undefined
|
||||
*/
|
||||
source?: HandleExportRawDataGETSource;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
* @description undefined
|
||||
*/
|
||||
start?: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @minimum 0
|
||||
* @description undefined
|
||||
*/
|
||||
end?: number;
|
||||
/**
|
||||
* @type integer
|
||||
* @maximum 50000
|
||||
* @minimum 1
|
||||
* @description undefined
|
||||
*/
|
||||
limit?: number;
|
||||
/**
|
||||
* @type string
|
||||
* @description undefined
|
||||
*/
|
||||
filter?: string;
|
||||
/**
|
||||
* @type array
|
||||
* @description undefined
|
||||
*/
|
||||
columns?: string[];
|
||||
/**
|
||||
* @type string
|
||||
* @description undefined
|
||||
*/
|
||||
order_by?: string;
|
||||
};
|
||||
|
||||
export enum HandleExportRawDataGETFormat {
|
||||
csv = 'csv',
|
||||
jsonl = 'jsonl',
|
||||
}
|
||||
export enum HandleExportRawDataGETSource {
|
||||
logs = 'logs',
|
||||
traces = 'traces',
|
||||
}
|
||||
export type HandleExportRawDataPOSTParams = {
|
||||
/**
|
||||
* @enum csv,jsonl
|
||||
* @type string
|
||||
* @description undefined
|
||||
*/
|
||||
format?: HandleExportRawDataPOSTFormat;
|
||||
};
|
||||
|
||||
export enum HandleExportRawDataPOSTFormat {
|
||||
csv = 'csv',
|
||||
jsonl = 'jsonl',
|
||||
}
|
||||
export type GetFieldsKeysParams = {
|
||||
/**
|
||||
* @description undefined
|
||||
|
||||
@@ -2,12 +2,13 @@
|
||||
|
||||
import ReactMarkdown from 'react-markdown';
|
||||
import { CodeProps } from 'react-markdown/lib/ast-to-react';
|
||||
import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
|
||||
import { a11yDark } from 'react-syntax-highlighter/dist/cjs/styles/prism';
|
||||
import logEvent from 'api/common/logEvent';
|
||||
import { isEmpty } from 'lodash-es';
|
||||
import rehypeRaw from 'rehype-raw';
|
||||
|
||||
import CodeCopyBtn from './CodeCopyBtn/CodeCopyBtn';
|
||||
import SyntaxHighlighter, { a11yDark } from './syntaxHighlighter';
|
||||
|
||||
interface LinkProps {
|
||||
href: string;
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
import { PrismLight as SyntaxHighlighter } from 'react-syntax-highlighter';
|
||||
import bash from 'react-syntax-highlighter/dist/esm/languages/prism/bash';
|
||||
import docker from 'react-syntax-highlighter/dist/esm/languages/prism/docker';
|
||||
import elixir from 'react-syntax-highlighter/dist/esm/languages/prism/elixir';
|
||||
import go from 'react-syntax-highlighter/dist/esm/languages/prism/go';
|
||||
import javascript from 'react-syntax-highlighter/dist/esm/languages/prism/javascript';
|
||||
import json from 'react-syntax-highlighter/dist/esm/languages/prism/json';
|
||||
import jsx from 'react-syntax-highlighter/dist/esm/languages/prism/jsx';
|
||||
import rust from 'react-syntax-highlighter/dist/esm/languages/prism/rust';
|
||||
import swift from 'react-syntax-highlighter/dist/esm/languages/prism/swift';
|
||||
import tsx from 'react-syntax-highlighter/dist/esm/languages/prism/tsx';
|
||||
import typescript from 'react-syntax-highlighter/dist/esm/languages/prism/typescript';
|
||||
import yaml from 'react-syntax-highlighter/dist/esm/languages/prism/yaml';
|
||||
import a11yDark from 'react-syntax-highlighter/dist/esm/styles/prism/a11y-dark';
|
||||
|
||||
SyntaxHighlighter.registerLanguage('bash', bash);
|
||||
SyntaxHighlighter.registerLanguage('docker', docker);
|
||||
SyntaxHighlighter.registerLanguage('dockerfile', docker);
|
||||
SyntaxHighlighter.registerLanguage('elixir', elixir);
|
||||
SyntaxHighlighter.registerLanguage('go', go);
|
||||
SyntaxHighlighter.registerLanguage('javascript', javascript);
|
||||
SyntaxHighlighter.registerLanguage('js', javascript);
|
||||
SyntaxHighlighter.registerLanguage('json', json);
|
||||
SyntaxHighlighter.registerLanguage('jsx', jsx);
|
||||
SyntaxHighlighter.registerLanguage('rust', rust);
|
||||
SyntaxHighlighter.registerLanguage('swift', swift);
|
||||
SyntaxHighlighter.registerLanguage('ts', typescript);
|
||||
SyntaxHighlighter.registerLanguage('tsx', tsx);
|
||||
SyntaxHighlighter.registerLanguage('typescript', typescript);
|
||||
SyntaxHighlighter.registerLanguage('yaml', yaml);
|
||||
SyntaxHighlighter.registerLanguage('yml', yaml);
|
||||
|
||||
export default SyntaxHighlighter;
|
||||
export { a11yDark };
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useState } from 'react';
|
||||
import { CloudDownloadOutlined } from '@ant-design/icons';
|
||||
import { Button, Dropdown, MenuProps } from 'antd';
|
||||
import { Excel } from 'antd-table-saveas-excel';
|
||||
import { unparse } from 'papaparse';
|
||||
|
||||
import { DownloadProps } from './Download.types';
|
||||
@@ -8,36 +8,25 @@ import { DownloadProps } from './Download.types';
|
||||
import './Download.styles.scss';
|
||||
|
||||
function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
|
||||
const [isDownloading, setIsDownloading] = useState(false);
|
||||
|
||||
const downloadExcelFile = async (): Promise<void> => {
|
||||
setIsDownloading(true);
|
||||
|
||||
try {
|
||||
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
|
||||
const updatedTitle = item
|
||||
.split('_')
|
||||
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
||||
.join(' ');
|
||||
return {
|
||||
title: updatedTitle,
|
||||
dataIndex: item,
|
||||
};
|
||||
});
|
||||
|
||||
const excelLib = await import('antd-table-saveas-excel');
|
||||
|
||||
const excel = new excelLib.Excel();
|
||||
excel
|
||||
.addSheet(fileName)
|
||||
.addColumns(headers)
|
||||
.addDataSource(data, {
|
||||
str2Percent: true,
|
||||
})
|
||||
.saveAs(`${fileName}.xlsx`);
|
||||
} finally {
|
||||
setIsDownloading(false);
|
||||
}
|
||||
const downloadExcelFile = (): void => {
|
||||
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
|
||||
const updatedTitle = item
|
||||
.split('_')
|
||||
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
||||
.join(' ');
|
||||
return {
|
||||
title: updatedTitle,
|
||||
dataIndex: item,
|
||||
};
|
||||
});
|
||||
const excel = new Excel();
|
||||
excel
|
||||
.addSheet(fileName)
|
||||
.addColumns(headers)
|
||||
.addDataSource(data, {
|
||||
str2Percent: true,
|
||||
})
|
||||
.saveAs(`${fileName}.xlsx`);
|
||||
};
|
||||
|
||||
const downloadCsvFile = (): void => {
|
||||
@@ -70,7 +59,7 @@ function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
|
||||
<Dropdown menu={menu} trigger={['click']}>
|
||||
<Button
|
||||
className="download-button"
|
||||
loading={isLoading || isDownloading}
|
||||
loading={isLoading}
|
||||
size="small"
|
||||
type="link"
|
||||
>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState } from 'react';
|
||||
import { Button, Popover, Typography } from 'antd';
|
||||
import { Excel } from 'antd-table-saveas-excel';
|
||||
import { FileDigit, FileDown, Sheet } from 'lucide-react';
|
||||
import { unparse } from 'papaparse';
|
||||
|
||||
@@ -8,34 +8,25 @@ import { DownloadProps } from './DownloadV2.types';
|
||||
import './DownloadV2.styles.scss';
|
||||
|
||||
function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
|
||||
const [isDownloading, setIsDownloading] = useState(false);
|
||||
|
||||
const downloadExcelFile = async (): Promise<void> => {
|
||||
setIsDownloading(true);
|
||||
|
||||
try {
|
||||
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
|
||||
const updatedTitle = item
|
||||
.split('_')
|
||||
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
||||
.join(' ');
|
||||
return {
|
||||
title: updatedTitle,
|
||||
dataIndex: item,
|
||||
};
|
||||
});
|
||||
const excelLib = await import('antd-table-saveas-excel');
|
||||
const excel = new excelLib.Excel();
|
||||
excel
|
||||
.addSheet(fileName)
|
||||
.addColumns(headers)
|
||||
.addDataSource(data, {
|
||||
str2Percent: true,
|
||||
})
|
||||
.saveAs(`${fileName}.xlsx`);
|
||||
} finally {
|
||||
setIsDownloading(false);
|
||||
}
|
||||
const downloadExcelFile = (): void => {
|
||||
const headers = Object.keys(Object.assign({}, ...data)).map((item) => {
|
||||
const updatedTitle = item
|
||||
.split('_')
|
||||
.map((word) => word.charAt(0).toUpperCase() + word.slice(1))
|
||||
.join(' ');
|
||||
return {
|
||||
title: updatedTitle,
|
||||
dataIndex: item,
|
||||
};
|
||||
});
|
||||
const excel = new Excel();
|
||||
excel
|
||||
.addSheet(fileName)
|
||||
.addColumns(headers)
|
||||
.addDataSource(data, {
|
||||
str2Percent: true,
|
||||
})
|
||||
.saveAs(`${fileName}.xlsx`);
|
||||
};
|
||||
|
||||
const downloadCsvFile = (): void => {
|
||||
@@ -63,7 +54,6 @@ function Download({ data, isLoading, fileName }: DownloadProps): JSX.Element {
|
||||
type="text"
|
||||
onClick={downloadExcelFile}
|
||||
className="action-btns"
|
||||
loading={isDownloading}
|
||||
>
|
||||
Excel (.xlsx)
|
||||
</Button>
|
||||
|
||||
@@ -6205,10 +6205,10 @@
|
||||
"@types/history" "^4.7.11"
|
||||
"@types/react" "*"
|
||||
|
||||
"@types/react-syntax-highlighter@15.5.13":
|
||||
version "15.5.13"
|
||||
resolved "https://registry.yarnpkg.com/@types/react-syntax-highlighter/-/react-syntax-highlighter-15.5.13.tgz#c5baf62a3219b3bf28d39cfea55d0a49a263d1f2"
|
||||
integrity sha512-uLGJ87j6Sz8UaBAooU0T6lWJ0dBmjZgN1PZTrj05TNql2/XpC6+4HhMT5syIdFUUt+FASfCeLLv4kBygNU+8qA==
|
||||
"@types/react-syntax-highlighter@15.5.7":
|
||||
version "15.5.7"
|
||||
resolved "https://registry.yarnpkg.com/@types/react-syntax-highlighter/-/react-syntax-highlighter-15.5.7.tgz#bd29020ccb118543d88779848f99059b64b02d0f"
|
||||
integrity sha512-bo5fEO5toQeyCp0zVHBeggclqf5SQ/Z5blfFmjwO5dkMVGPgmiwZsJh9nu/Bo5L7IHTuGWrja6LxJVE2uB5ZrQ==
|
||||
dependencies:
|
||||
"@types/react" "*"
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/preference"
|
||||
"github.com/SigNoz/signoz/pkg/modules/promote"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
|
||||
"github.com/SigNoz/signoz/pkg/modules/session"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
@@ -46,6 +47,7 @@ type provider struct {
|
||||
gatewayHandler gateway.Handler
|
||||
fieldsHandler fields.Handler
|
||||
authzHandler authz.Handler
|
||||
rawDataExportHandler rawdataexport.Handler
|
||||
zeusHandler zeus.Handler
|
||||
querierHandler querier.Handler
|
||||
}
|
||||
@@ -67,6 +69,7 @@ func NewFactory(
|
||||
gatewayHandler gateway.Handler,
|
||||
fieldsHandler fields.Handler,
|
||||
authzHandler authz.Handler,
|
||||
rawDataExportHandler rawdataexport.Handler,
|
||||
zeusHandler zeus.Handler,
|
||||
querierHandler querier.Handler,
|
||||
) factory.ProviderFactory[apiserver.APIServer, apiserver.Config] {
|
||||
@@ -91,6 +94,7 @@ func NewFactory(
|
||||
gatewayHandler,
|
||||
fieldsHandler,
|
||||
authzHandler,
|
||||
rawDataExportHandler,
|
||||
zeusHandler,
|
||||
querierHandler,
|
||||
)
|
||||
@@ -117,6 +121,7 @@ func newProvider(
|
||||
gatewayHandler gateway.Handler,
|
||||
fieldsHandler fields.Handler,
|
||||
authzHandler authz.Handler,
|
||||
rawDataExportHandler rawdataexport.Handler,
|
||||
zeusHandler zeus.Handler,
|
||||
querierHandler querier.Handler,
|
||||
) (apiserver.APIServer, error) {
|
||||
@@ -141,6 +146,7 @@ func newProvider(
|
||||
gatewayHandler: gatewayHandler,
|
||||
fieldsHandler: fieldsHandler,
|
||||
authzHandler: authzHandler,
|
||||
rawDataExportHandler: rawDataExportHandler,
|
||||
zeusHandler: zeusHandler,
|
||||
querierHandler: querierHandler,
|
||||
}
|
||||
@@ -215,6 +221,10 @@ func (provider *provider) AddToRouter(router *mux.Router) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := provider.addRawDataExportRoutes(router); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := provider.addZeusRoutes(router); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
47
pkg/apiserver/signozapiserver/rawDataExport.go
Normal file
47
pkg/apiserver/signozapiserver/rawDataExport.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package signozapiserver
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/http/handler"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/exporttypes"
|
||||
v5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
func (provider *provider) addRawDataExportRoutes(router *mux.Router) error {
|
||||
if err := router.Handle("/api/v1/export_raw_data", handler.New(provider.authZ.ViewAccess(provider.rawDataExportHandler.ExportRawData), handler.OpenAPIDef{
|
||||
ID: "HandleExportRawDataGET",
|
||||
Tags: []string{"logs", "traces"},
|
||||
Summary: "Export raw data",
|
||||
Description: "This endpoints allows simple query exporting raw data for traces and logs",
|
||||
RequestQuery: new(exporttypes.ExportRawDataQueryParams),
|
||||
RequestContentType: "",
|
||||
Response: nil,
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusBadRequest},
|
||||
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
|
||||
})).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := router.Handle("/api/v1/export_raw_data", handler.New(provider.authZ.ViewAccess(provider.rawDataExportHandler.ExportRawData), handler.OpenAPIDef{
|
||||
ID: "HandleExportRawDataPOST",
|
||||
Tags: []string{"logs", "traces"},
|
||||
Summary: "Export raw data",
|
||||
Description: "This endpoints allows complex query exporting raw data for traces and logs",
|
||||
Request: new(v5.QueryRangeRequest),
|
||||
RequestQuery: new(exporttypes.ExportRawDataFormatQueryParam),
|
||||
RequestContentType: "application/json",
|
||||
Response: nil,
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusBadRequest},
|
||||
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
|
||||
})).Methods(http.MethodPost).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -6,18 +6,18 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/http/binding"
|
||||
"github.com/SigNoz/signoz/pkg/http/render"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/exporttypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -31,129 +31,52 @@ func NewHandler(module rawdataexport.Module) rawdataexport.Handler {
|
||||
return &handler{module: module}
|
||||
}
|
||||
|
||||
// ExportRawData handles data export requests.
|
||||
//
|
||||
// API Documentation:
|
||||
// Endpoint: GET /api/v1/export_raw_data
|
||||
//
|
||||
// Query Parameters:
|
||||
//
|
||||
// - source (optional): Type of data to export ["logs" (default), "metrics", "traces"]
|
||||
// Note: Currently only "logs" is fully supported
|
||||
//
|
||||
// - format (optional): Output format ["csv" (default), "jsonl"]
|
||||
//
|
||||
// - start (required): Start time for query (Unix timestamp in nanoseconds)
|
||||
//
|
||||
// - end (required): End time for query (Unix timestamp in nanoseconds)
|
||||
//
|
||||
// - limit (optional): Maximum number of rows to export
|
||||
// Constraints: Must be positive and cannot exceed MAX_EXPORT_ROW_COUNT_LIMIT
|
||||
//
|
||||
// - filter (optional): Filter expression to apply to the query
|
||||
//
|
||||
// - columns (optional): Specific columns to include in export
|
||||
// Default: all columns are returned
|
||||
// Format: ["context.field:type", "context.field", "field"]
|
||||
//
|
||||
// - order_by (optional): Sorting specification ["column:direction" or "context.field:type:direction"]
|
||||
// Direction: "asc" or "desc"
|
||||
// Default: ["timestamp:desc", "id:desc"]
|
||||
//
|
||||
// Response Headers:
|
||||
// - Content-Type: "text/csv" or "application/x-ndjson"
|
||||
// - Content-Encoding: "gzip" (handled by HTTP middleware)
|
||||
// - Content-Disposition: "attachment; filename=\"data_exported.[format]\""
|
||||
// - Cache-Control: "no-cache"
|
||||
// - Vary: "Accept-Encoding"
|
||||
// - Transfer-Encoding: "chunked"
|
||||
// - Trailers: X-Response-Complete
|
||||
//
|
||||
// Response Format:
|
||||
//
|
||||
// CSV: Headers in first row, data in subsequent rows
|
||||
// JSONL: One JSON object per line
|
||||
//
|
||||
// Example Usage:
|
||||
//
|
||||
// Basic CSV export:
|
||||
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000
|
||||
//
|
||||
// Export with columns and format:
|
||||
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000&format=jsonl
|
||||
// &columns=timestamp&columns=severity&columns=message
|
||||
//
|
||||
// Export with filter and ordering:
|
||||
// GET /api/v1/export_raw_data?start=1693612800000000000&end=1693699199000000000
|
||||
// &filter=severity="error"&order_by=timestamp:desc&limit=1000
|
||||
func (handler *handler) ExportRawData(rw http.ResponseWriter, r *http.Request) {
|
||||
source, err := getExportQuerySource(r.URL.Query())
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
var queryRangeRequest qbtypes.QueryRangeRequest
|
||||
var format string
|
||||
|
||||
switch source {
|
||||
case "logs":
|
||||
handler.exportLogs(rw, r)
|
||||
case "traces":
|
||||
handler.exportTraces(rw, r)
|
||||
case "metrics":
|
||||
handler.exportMetrics(rw, r)
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
var params exporttypes.ExportRawDataQueryParams
|
||||
if err := binding.Query.BindQuery(r.URL.Query(), ¶ms); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
params.Normalize()
|
||||
if err := params.Validate(); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
format = params.Format
|
||||
queryRangeRequest = buildQueryRangeRequest(¶ms)
|
||||
case http.MethodPost:
|
||||
var formatParam exporttypes.ExportRawDataFormatQueryParam
|
||||
if err := binding.Query.BindQuery(r.URL.Query(), &formatParam); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
format = formatParam.Format
|
||||
if err := json.NewDecoder(r.Body).Decode(&queryRangeRequest); err != nil {
|
||||
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid request body: %v", err))
|
||||
return
|
||||
}
|
||||
default:
|
||||
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid source: must be logs"))
|
||||
render.Error(rw, errors.Newf(errors.TypeMethodNotAllowed, errors.CodeMethodNotAllowed, "method not allowed, only GET/POST supported"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (handler *handler) exportMetrics(rw http.ResponseWriter, r *http.Request) {
|
||||
render.Error(rw, errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported, "metrics export is not yet supported"))
|
||||
}
|
||||
|
||||
func (handler *handler) exportTraces(rw http.ResponseWriter, r *http.Request) {
|
||||
render.Error(rw, errors.Newf(errors.TypeUnsupported, errors.CodeUnsupported, "traces export is not yet supported"))
|
||||
}
|
||||
|
||||
func (handler *handler) exportLogs(rw http.ResponseWriter, r *http.Request) {
|
||||
// Set up response headers
|
||||
rw.Header().Set("Cache-Control", "no-cache")
|
||||
rw.Header().Set("Vary", "Accept-Encoding") // Indicate that response varies based on Accept-Encoding
|
||||
rw.Header().Set("Access-Control-Expose-Headers", "Content-Disposition, X-Response-Complete")
|
||||
rw.Header().Set("Trailer", "X-Response-Complete")
|
||||
rw.Header().Set("Transfer-Encoding", "chunked")
|
||||
|
||||
queryParams := r.URL.Query()
|
||||
|
||||
startTime, endTime, err := getExportQueryTimeRange(queryParams)
|
||||
if err != nil {
|
||||
if err := validateSpecForExport(&queryRangeRequest); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
limit, err := getExportQueryLimit(queryParams)
|
||||
if err != nil {
|
||||
if err := validateAndApplyDefaultExportLimits(queryRangeRequest.CompositeQuery.Queries); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
format, err := getExportQueryFormat(queryParams)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Set appropriate content type and filename
|
||||
filename := fmt.Sprintf("data_exported_%s.%s", time.Now().Format("2006-01-02_150405"), format)
|
||||
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
|
||||
|
||||
filterExpression := queryParams.Get("filter")
|
||||
|
||||
orderByExpression, err := getExportQueryOrderBy(queryParams)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
columns := getExportQueryColumns(queryParams)
|
||||
// Use default OrderBy if not specified
|
||||
queryRangeRequest.UseDefaultOrderBy()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(r.Context())
|
||||
if err != nil {
|
||||
@@ -167,70 +90,134 @@ func (handler *handler) exportLogs(rw http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
queryRangeRequest := qbtypes.QueryRangeRequest{
|
||||
Start: startTime,
|
||||
End: endTime,
|
||||
RequestType: qbtypes.RequestTypeRaw,
|
||||
CompositeQuery: qbtypes.CompositeQuery{
|
||||
Queries: []qbtypes.QueryEnvelope{
|
||||
{
|
||||
Type: qbtypes.QueryTypeBuilder,
|
||||
Spec: nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
setExportResponseHeaders(rw, format)
|
||||
|
||||
spec := qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Name: "raw",
|
||||
Filter: &qbtypes.Filter{
|
||||
Expression: filterExpression,
|
||||
},
|
||||
Limit: limit,
|
||||
Order: orderByExpression,
|
||||
}
|
||||
|
||||
spec.SelectFields = columns
|
||||
|
||||
queryRangeRequest.CompositeQuery.Queries[0].Spec = spec
|
||||
|
||||
// This will signal Export module to stop sending data
|
||||
doneChan := make(chan any)
|
||||
defer close(doneChan)
|
||||
rowChan, errChan := handler.module.ExportRawData(r.Context(), orgID, &queryRangeRequest, doneChan)
|
||||
|
||||
var isComplete bool
|
||||
isComplete, err := handler.executeExport(rowChan, errChan, format, rw)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
rw.Header().Set("X-Response-Complete", strconv.FormatBool(isComplete))
|
||||
}
|
||||
|
||||
// validateSpecForExport validates query specs
|
||||
func validateSpecForExport(req *qbtypes.QueryRangeRequest) error {
|
||||
|
||||
queries := req.CompositeQuery.Queries
|
||||
|
||||
// If the trace operator query is not present, and there are multiple queries, return an error
|
||||
if req.TraceOperatorQueryIndex() == -1 && len(queries) > 1 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "multiple queries not allowed without a trace operator query")
|
||||
}
|
||||
|
||||
for idx := range queries {
|
||||
switch spec := queries[idx].Spec.(type) {
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
|
||||
qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
|
||||
qbtypes.QueryBuilderTraceOperator:
|
||||
// Supported spec types
|
||||
default:
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported query at index %d type: %T", idx, spec)
|
||||
}
|
||||
}
|
||||
|
||||
err := req.Validate(qbtypes.WithSkipLimitValidation())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAndApplyDefaultExportLimits(queries []qbtypes.QueryEnvelope) error {
|
||||
for idx := range queries {
|
||||
limit := queries[idx].GetLimit()
|
||||
if limit == 0 {
|
||||
limit = DefaultExportRowCountLimit
|
||||
} else if limit < 0 {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "limit must be positive")
|
||||
} else if limit > MaxExportRowCountLimit {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "limit cannot be more than %d", MaxExportRowCountLimit)
|
||||
}
|
||||
queries[idx].SetLimit(limit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildQueryEnvelope creates a QueryEnvelope with a QueryBuilderQuery for the given aggregation type.
|
||||
func buildQueryEnvelope[T any](signal telemetrytypes.Signal, filter *qbtypes.Filter, limit int, order []qbtypes.OrderBy, selectFields []telemetrytypes.TelemetryFieldKey) qbtypes.QueryEnvelope {
|
||||
return qbtypes.QueryEnvelope{
|
||||
Type: qbtypes.QueryTypeBuilder,
|
||||
Spec: qbtypes.QueryBuilderQuery[T]{
|
||||
Signal: signal,
|
||||
Filter: filter,
|
||||
Limit: limit,
|
||||
Order: order,
|
||||
SelectFields: selectFields,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// buildQueryRangeRequest builds a QueryRangeRequest from already-bound and validated GET query params.
|
||||
func buildQueryRangeRequest(params *exporttypes.ExportRawDataQueryParams) qbtypes.QueryRangeRequest {
|
||||
var query qbtypes.QueryEnvelope
|
||||
switch params.Signal {
|
||||
case telemetrytypes.SignalLogs:
|
||||
query = buildQueryEnvelope[qbtypes.LogAggregation](params.Signal, ¶ms.Filter, params.Limit, params.Order, params.SelectFields)
|
||||
case telemetrytypes.SignalTraces:
|
||||
query = buildQueryEnvelope[qbtypes.TraceAggregation](params.Signal, ¶ms.Filter, params.Limit, params.Order, params.SelectFields)
|
||||
}
|
||||
|
||||
return qbtypes.QueryRangeRequest{
|
||||
Start: params.Start,
|
||||
End: params.End,
|
||||
RequestType: qbtypes.RequestTypeRaw,
|
||||
CompositeQuery: qbtypes.CompositeQuery{
|
||||
Queries: []qbtypes.QueryEnvelope{query},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// setExportResponseHeaders sets common HTTP headers for export responses.
|
||||
func setExportResponseHeaders(rw http.ResponseWriter, format string) {
|
||||
rw.Header().Set("Cache-Control", "no-cache")
|
||||
rw.Header().Set("Vary", "Accept-Encoding")
|
||||
rw.Header().Set("Access-Control-Expose-Headers", "Content-Disposition, X-Response-Complete")
|
||||
rw.Header().Set("Trailer", "X-Response-Complete")
|
||||
rw.Header().Set("Transfer-Encoding", "chunked")
|
||||
filename := fmt.Sprintf("data_exported_%s.%s", time.Now().Format("2006-01-02_150405"), format)
|
||||
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename))
|
||||
}
|
||||
|
||||
// executeExport streams data from rowChan to the response writer in the specified format.
|
||||
func (handler *handler) executeExport(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, format string, rw http.ResponseWriter) (bool, error) {
|
||||
switch format {
|
||||
case "csv", "":
|
||||
rw.Header().Set("Content-Type", "text/csv")
|
||||
csvWriter := csv.NewWriter(rw)
|
||||
isComplete, err = handler.exportLogsCSV(rowChan, errChan, csvWriter)
|
||||
isComplete, err := handler.exportRawDataCSV(rowChan, errChan, csvWriter)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
return false, err
|
||||
}
|
||||
csvWriter.Flush()
|
||||
return isComplete, nil
|
||||
case "jsonl":
|
||||
rw.Header().Set("Content-Type", "application/x-ndjson")
|
||||
isComplete, err = handler.exportLogsJSONL(rowChan, errChan, rw)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
return handler.exportRawDataJSONL(rowChan, errChan, rw)
|
||||
default:
|
||||
render.Error(rw, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl"))
|
||||
return
|
||||
return false, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl")
|
||||
}
|
||||
|
||||
rw.Header().Set("X-Response-Complete", strconv.FormatBool(isComplete))
|
||||
}
|
||||
|
||||
func (handler *handler) exportLogsCSV(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, csvWriter *csv.Writer) (bool, error) {
|
||||
var header []string
|
||||
// exportRawDataCSV is a generic CSV export function that works with any raw data (logs, traces, etc.)
|
||||
func (handler *handler) exportRawDataCSV(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, csvWriter *csv.Writer) (bool, error) {
|
||||
|
||||
headerToIndexMapping := make(map[string]int, len(header))
|
||||
var header []string
|
||||
headerToIndexMapping := make(map[string]int)
|
||||
|
||||
totalBytes := uint64(0)
|
||||
for {
|
||||
@@ -268,8 +255,8 @@ func (handler *handler) exportLogsCSV(rowChan <-chan *qbtypes.RawRow, errChan <-
|
||||
}
|
||||
}
|
||||
|
||||
func (handler *handler) exportLogsJSONL(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, writer io.Writer) (bool, error) {
|
||||
|
||||
// exportRawDataJSONL is a generic JSONL export function that works with any raw data (logs, traces, etc.)
|
||||
func (handler *handler) exportRawDataJSONL(rowChan <-chan *qbtypes.RawRow, errChan <-chan error, writer io.Writer) (bool, error) {
|
||||
totalBytes := uint64(0)
|
||||
for {
|
||||
select {
|
||||
@@ -277,9 +264,11 @@ func (handler *handler) exportLogsJSONL(rowChan <-chan *qbtypes.RawRow, errChan
|
||||
if !ok {
|
||||
return true, nil
|
||||
}
|
||||
// Handle JSON format (JSONL - one object per line)
|
||||
jsonBytes, _ := json.Marshal(row.Data)
|
||||
totalBytes += uint64(len(jsonBytes)) + 1 // +1 for newline
|
||||
jsonBytes, err := json.Marshal(row.Data)
|
||||
if err != nil {
|
||||
return false, errors.NewUnexpectedf(errors.CodeInternal, "error marshaling JSON: %s", err)
|
||||
}
|
||||
totalBytes += uint64(len(jsonBytes)) + 1
|
||||
|
||||
if _, err := writer.Write(jsonBytes); err != nil {
|
||||
return false, errors.NewUnexpectedf(errors.CodeInternal, "error writing JSON: %s", err)
|
||||
@@ -299,74 +288,33 @@ func (handler *handler) exportLogsJSONL(rowChan <-chan *qbtypes.RawRow, errChan
|
||||
}
|
||||
}
|
||||
|
||||
func getExportQuerySource(queryParams url.Values) (string, error) {
|
||||
switch queryParams.Get("source") {
|
||||
case "logs", "":
|
||||
return "logs", nil
|
||||
case "metrics":
|
||||
return "metrics", errors.NewInvalidInputf(errors.CodeInvalidInput, "metrics export not yet supported")
|
||||
case "traces":
|
||||
return "traces", errors.NewInvalidInputf(errors.CodeInvalidInput, "traces export not yet supported")
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid source: must be logs, metrics or traces")
|
||||
}
|
||||
}
|
||||
|
||||
func getExportQueryFormat(queryParams url.Values) (string, error) {
|
||||
switch queryParams.Get("format") {
|
||||
case "csv", "":
|
||||
return "csv", nil
|
||||
case "jsonl":
|
||||
return "jsonl", nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid format: must be csv or jsonl")
|
||||
}
|
||||
}
|
||||
|
||||
func getExportQueryLimit(queryParams url.Values) (int, error) {
|
||||
|
||||
limitStr := queryParams.Get("limit")
|
||||
if limitStr == "" {
|
||||
return DefaultExportRowCountLimit, nil
|
||||
} else {
|
||||
limit, err := strconv.Atoi(limitStr)
|
||||
if err != nil {
|
||||
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid limit format: %s", err.Error())
|
||||
}
|
||||
if limit <= 0 {
|
||||
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "limit must be positive")
|
||||
}
|
||||
if limit > MaxExportRowCountLimit {
|
||||
return 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "limit cannot be more than %d", MaxExportRowCountLimit)
|
||||
}
|
||||
return limit, nil
|
||||
}
|
||||
}
|
||||
|
||||
func getExportQueryTimeRange(queryParams url.Values) (uint64, uint64, error) {
|
||||
|
||||
startTimeStr := queryParams.Get("start")
|
||||
endTimeStr := queryParams.Get("end")
|
||||
|
||||
if startTimeStr == "" || endTimeStr == "" {
|
||||
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "start and end time are required")
|
||||
}
|
||||
startTime, err := strconv.ParseUint(startTimeStr, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid start time format: %s", err.Error())
|
||||
}
|
||||
endTime, err := strconv.ParseUint(endTimeStr, 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid end time format: %s", err.Error())
|
||||
}
|
||||
return startTime, endTime, nil
|
||||
}
|
||||
// priorityColumns defines the columns that should appear first in the CSV output, in order.
|
||||
var priorityColumns = []string{"timestamp", "id"}
|
||||
|
||||
func constructCSVHeaderFromQueryResponse(data map[string]any) []string {
|
||||
header := make([]string, 0, len(data))
|
||||
for key := range data {
|
||||
header = append(header, key)
|
||||
}
|
||||
// This is to ensure CSV output is consistent across multiple queries
|
||||
slices.SortFunc(header, func(a, b string) int {
|
||||
ai, bi := slices.Index(priorityColumns, a), slices.Index(priorityColumns, b)
|
||||
switch {
|
||||
case ai != -1 && bi != -1:
|
||||
return ai - bi
|
||||
case ai != -1:
|
||||
return -1
|
||||
case bi != -1:
|
||||
return 1
|
||||
default:
|
||||
if a < b {
|
||||
return -1
|
||||
} else if a > b {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
})
|
||||
return header
|
||||
}
|
||||
|
||||
@@ -427,9 +375,12 @@ func constructCSVRecordFromQueryResponse(data map[string]any, headerToIndexMappi
|
||||
valueStr = v.String()
|
||||
|
||||
default:
|
||||
// For all other complex types (maps, structs, etc.)
|
||||
jsonBytes, _ := json.Marshal(v)
|
||||
valueStr = string(jsonBytes)
|
||||
jsonBytes, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
valueStr = fmt.Sprintf("%v", v)
|
||||
} else {
|
||||
valueStr = string(jsonBytes)
|
||||
}
|
||||
}
|
||||
|
||||
record[index] = sanitizeForCSV(valueStr)
|
||||
@@ -438,26 +389,6 @@ func constructCSVRecordFromQueryResponse(data map[string]any, headerToIndexMappi
|
||||
return record
|
||||
}
|
||||
|
||||
// getExportQueryColumns parses the "columns" query parameters and returns a slice of TelemetryFieldKey structs.
|
||||
// Each column should be a valid telemetry field key in the format "context.field:type" or "context.field" or "field"
|
||||
func getExportQueryColumns(queryParams url.Values) []telemetrytypes.TelemetryFieldKey {
|
||||
columnParams := queryParams["columns"]
|
||||
|
||||
columns := make([]telemetrytypes.TelemetryFieldKey, 0, len(columnParams))
|
||||
|
||||
for _, columnStr := range columnParams {
|
||||
// Skip empty strings
|
||||
columnStr = strings.TrimSpace(columnStr)
|
||||
if columnStr == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
columns = append(columns, telemetrytypes.GetFieldKeyFromKeyText(columnStr))
|
||||
}
|
||||
|
||||
return columns
|
||||
}
|
||||
|
||||
func getsizeOfStringSlice(slice []string) uint64 {
|
||||
var totalBytes uint64
|
||||
for _, str := range slice {
|
||||
@@ -465,52 +396,3 @@ func getsizeOfStringSlice(slice []string) uint64 {
|
||||
}
|
||||
return totalBytes
|
||||
}
|
||||
|
||||
// getExportQueryOrderBy parses the "order_by" query parameters and returns a slice of OrderBy structs.
|
||||
// Each "order_by" parameter should be in the format "column:direction"
|
||||
// Each "column" should be a valid telemetry field key in the format "context.field:type" or "context.field" or "field"
|
||||
func getExportQueryOrderBy(queryParams url.Values) ([]qbtypes.OrderBy, error) {
|
||||
orderByParam := queryParams.Get("order_by")
|
||||
|
||||
orderByParam = strings.TrimSpace(orderByParam)
|
||||
if orderByParam == "" {
|
||||
return telemetrylogs.DefaultLogsV2SortingOrder, nil
|
||||
}
|
||||
|
||||
parts := strings.Split(orderByParam, ":")
|
||||
if len(parts) != 2 && len(parts) != 3 {
|
||||
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order_by format: %s, should be <column>:<direction>", orderByParam)
|
||||
}
|
||||
|
||||
column := strings.Join(parts[:len(parts)-1], ":")
|
||||
direction := parts[len(parts)-1]
|
||||
|
||||
orderDirection, ok := qbtypes.OrderDirectionMap[direction]
|
||||
if !ok {
|
||||
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order_by direction: %s, should be one of %s, %s", direction, qbtypes.OrderDirectionAsc, qbtypes.OrderDirectionDesc)
|
||||
}
|
||||
|
||||
orderByKey := telemetrytypes.GetFieldKeyFromKeyText(column)
|
||||
|
||||
orderBy := []qbtypes.OrderBy{
|
||||
{
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: orderByKey,
|
||||
},
|
||||
Direction: orderDirection,
|
||||
},
|
||||
}
|
||||
|
||||
// If we are ordering by the timestamp column, also order by the ID column
|
||||
if orderByKey.Name == telemetrylogs.LogsV2TimestampColumn {
|
||||
orderBy = append(orderBy, qbtypes.OrderBy{
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2IDColumn,
|
||||
},
|
||||
},
|
||||
Direction: orderDirection,
|
||||
})
|
||||
}
|
||||
return orderBy, nil
|
||||
}
|
||||
|
||||
@@ -2,162 +2,85 @@ package implrawdataexport
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/http/binding"
|
||||
"github.com/SigNoz/signoz/pkg/types/exporttypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetExportQuerySource(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
queryParams url.Values
|
||||
expectedSource string
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
name: "default logs source",
|
||||
queryParams: url.Values{},
|
||||
expectedSource: "logs",
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "explicit logs source",
|
||||
queryParams: url.Values{"source": {"logs"}},
|
||||
expectedSource: "logs",
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "metrics source - not supported",
|
||||
queryParams: url.Values{"source": {"metrics"}},
|
||||
expectedSource: "metrics",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "traces source - not supported",
|
||||
queryParams: url.Values{"source": {"traces"}},
|
||||
expectedSource: "traces",
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid source",
|
||||
queryParams: url.Values{"source": {"invalid"}},
|
||||
expectedSource: "",
|
||||
expectedError: true,
|
||||
},
|
||||
}
|
||||
func TestExportRawDataQueryParams_BindingDefaults(t *testing.T) {
|
||||
var params exporttypes.ExportRawDataQueryParams
|
||||
err := binding.Query.BindQuery(url.Values{}, ¶ms)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "csv", params.Format)
|
||||
assert.Equal(t, DefaultExportRowCountLimit, params.Limit)
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
source, err := getExportQuerySource(tt.queryParams)
|
||||
assert.Equal(t, tt.expectedSource, source)
|
||||
if tt.expectedError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
func logQuery(limit int) qbtypes.QueryEnvelope {
|
||||
return qbtypes.QueryEnvelope{
|
||||
Type: qbtypes.QueryTypeBuilder,
|
||||
Spec: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{Limit: limit},
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetExportQueryFormat(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
queryParams url.Values
|
||||
expectedFormat string
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
name: "default csv format",
|
||||
queryParams: url.Values{},
|
||||
expectedFormat: "csv",
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "explicit csv format",
|
||||
queryParams: url.Values{"format": {"csv"}},
|
||||
expectedFormat: "csv",
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "jsonl format",
|
||||
queryParams: url.Values{"format": {"jsonl"}},
|
||||
expectedFormat: "jsonl",
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
queryParams: url.Values{"format": {"xml"}},
|
||||
expectedFormat: "",
|
||||
expectedError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
format, err := getExportQueryFormat(tt.queryParams)
|
||||
assert.Equal(t, tt.expectedFormat, format)
|
||||
if tt.expectedError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
func traceQuery(limit int) qbtypes.QueryEnvelope {
|
||||
return qbtypes.QueryEnvelope{
|
||||
Type: qbtypes.QueryTypeBuilder,
|
||||
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{Limit: limit},
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetExportQueryLimit(t *testing.T) {
|
||||
func traceOperatorQuery(limit int) qbtypes.QueryEnvelope {
|
||||
return qbtypes.QueryEnvelope{
|
||||
Type: qbtypes.QueryTypeTraceOperator,
|
||||
Spec: qbtypes.QueryBuilderTraceOperator{Limit: limit, Expression: "A"},
|
||||
}
|
||||
}
|
||||
|
||||
func makeRequest(queries ...qbtypes.QueryEnvelope) qbtypes.QueryRangeRequest {
|
||||
return qbtypes.QueryRangeRequest{
|
||||
Start: 1000000000000,
|
||||
End: 1000003600000,
|
||||
RequestType: qbtypes.RequestTypeRaw,
|
||||
CompositeQuery: qbtypes.CompositeQuery{Queries: queries},
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSpecForExport(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
queryParams url.Values
|
||||
expectedLimit int
|
||||
req qbtypes.QueryRangeRequest
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
name: "default limit",
|
||||
queryParams: url.Values{},
|
||||
expectedLimit: DefaultExportRowCountLimit,
|
||||
expectedError: false,
|
||||
name: "single log query",
|
||||
req: makeRequest(logQuery(0)),
|
||||
},
|
||||
{
|
||||
name: "valid limit",
|
||||
queryParams: url.Values{"limit": {"5000"}},
|
||||
expectedLimit: 5000,
|
||||
expectedError: false,
|
||||
name: "single trace query",
|
||||
req: makeRequest(traceQuery(0)),
|
||||
},
|
||||
{
|
||||
name: "maximum limit",
|
||||
queryParams: url.Values{"limit": {strconv.Itoa(MaxExportRowCountLimit)}},
|
||||
expectedLimit: MaxExportRowCountLimit,
|
||||
expectedError: false,
|
||||
name: "trace operator alone",
|
||||
req: makeRequest(traceOperatorQuery(0)),
|
||||
},
|
||||
{
|
||||
name: "limit exceeds maximum",
|
||||
queryParams: url.Values{"limit": {"100000"}},
|
||||
expectedLimit: 0,
|
||||
name: "multiple queries without trace operator",
|
||||
req: makeRequest(logQuery(0), traceQuery(0)),
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid limit format",
|
||||
queryParams: url.Values{"limit": {"invalid"}},
|
||||
expectedLimit: 0,
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "negative limit",
|
||||
queryParams: url.Values{"limit": {"-100"}},
|
||||
expectedLimit: 0,
|
||||
name: "unsupported query type",
|
||||
req: makeRequest(qbtypes.QueryEnvelope{Type: qbtypes.QueryTypeBuilder, Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{}}),
|
||||
expectedError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
limit, err := getExportQueryLimit(tt.queryParams)
|
||||
assert.Equal(t, tt.expectedLimit, limit)
|
||||
err := validateSpecForExport(&tt.req)
|
||||
if tt.expectedError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
@@ -167,352 +90,69 @@ func TestGetExportQueryLimit(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetExportQueryTimeRange(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
queryParams url.Values
|
||||
expectedStartTime uint64
|
||||
expectedEndTime uint64
|
||||
expectedError bool
|
||||
}{
|
||||
{
|
||||
name: "valid time range",
|
||||
queryParams: url.Values{
|
||||
"start": {"1640995200"},
|
||||
"end": {"1641081600"},
|
||||
},
|
||||
expectedStartTime: 1640995200,
|
||||
expectedEndTime: 1641081600,
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "missing start time",
|
||||
queryParams: url.Values{"end": {"1641081600"}},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "missing end time",
|
||||
queryParams: url.Values{"start": {"1640995200"}},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "missing both times",
|
||||
queryParams: url.Values{},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid start time format",
|
||||
queryParams: url.Values{
|
||||
"start": {"invalid"},
|
||||
"end": {"1641081600"},
|
||||
},
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "invalid end time format",
|
||||
queryParams: url.Values{
|
||||
"start": {"1640995200"},
|
||||
"end": {"invalid"},
|
||||
},
|
||||
expectedError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
startTime, endTime, err := getExportQueryTimeRange(tt.queryParams)
|
||||
if tt.expectedError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, tt.expectedStartTime, startTime)
|
||||
assert.Equal(t, tt.expectedEndTime, endTime)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetExportQueryColumns(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
queryParams url.Values
|
||||
expectedColumns []telemetrytypes.TelemetryFieldKey
|
||||
}{
|
||||
{
|
||||
name: "no columns specified",
|
||||
queryParams: url.Values{},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{},
|
||||
},
|
||||
{
|
||||
name: "single column",
|
||||
queryParams: url.Values{
|
||||
"columns": {"timestamp"},
|
||||
},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple columns",
|
||||
queryParams: url.Values{
|
||||
"columns": {"timestamp", "message", "level"},
|
||||
},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "message"},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty column name (should be skipped)",
|
||||
queryParams: url.Values{
|
||||
"columns": {"timestamp", "", "level"},
|
||||
},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "whitespace column name (should be skipped)",
|
||||
queryParams: url.Values{
|
||||
"columns": {"timestamp", " ", "level"},
|
||||
},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid column name with data type",
|
||||
queryParams: url.Values{
|
||||
"columns": {"timestamp", "attribute.user:string", "level"},
|
||||
},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "user", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid column name with dot notation",
|
||||
queryParams: url.Values{
|
||||
"columns": {"timestamp", "attribute.user.string", "level"},
|
||||
},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "user.string", FieldContext: telemetrytypes.FieldContextAttribute},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
columns := getExportQueryColumns(tt.queryParams)
|
||||
assert.Equal(t, len(tt.expectedColumns), len(columns))
|
||||
for i, expectedCol := range tt.expectedColumns {
|
||||
assert.Equal(t, expectedCol, columns[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetExportQueryOrderBy(t *testing.T) {
|
||||
func TestValidateAndApplyDefaultExportLimits(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
queryParams url.Values
|
||||
expectedOrder []qbtypes.OrderBy
|
||||
queries []qbtypes.QueryEnvelope
|
||||
expectedError bool
|
||||
checkQueries func(t *testing.T, queries []qbtypes.QueryEnvelope)
|
||||
}{
|
||||
{
|
||||
name: "no order specified",
|
||||
queryParams: url.Values{},
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2TimestampColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2IDColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
name: "single log query, zero limit gets default",
|
||||
queries: makeRequest(logQuery(0)).CompositeQuery.Queries,
|
||||
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
|
||||
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
|
||||
},
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "single order error, direction not specified",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"timestamp"},
|
||||
name: "single log query, valid limit kept",
|
||||
queries: makeRequest(logQuery(1000)).CompositeQuery.Queries,
|
||||
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
|
||||
assert.Equal(t, 1000, q[0].GetLimit())
|
||||
},
|
||||
expectedOrder: nil,
|
||||
},
|
||||
{
|
||||
name: "single log query, max limit kept",
|
||||
queries: makeRequest(logQuery(MaxExportRowCountLimit)).CompositeQuery.Queries,
|
||||
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
|
||||
assert.Equal(t, MaxExportRowCountLimit, q[0].GetLimit())
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single log query, limit exceeds max",
|
||||
queries: makeRequest(logQuery(MaxExportRowCountLimit + 1)).CompositeQuery.Queries,
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "single order no error",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"timestamp:asc"},
|
||||
},
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2TimestampColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2IDColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "multiple orders",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"timestamp:asc", "body:desc", "id:asc"},
|
||||
},
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2TimestampColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2IDColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "empty order name (should be skipped)",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"timestamp:asc", "", "id:asc"},
|
||||
},
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2TimestampColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2IDColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "whitespace order name (should be skipped)",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"timestamp:asc", " ", "id:asc"},
|
||||
},
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2TimestampColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: telemetrylogs.LogsV2IDColumn,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "invalid order name (should error out)",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"attributes.user:", "id:asc"},
|
||||
},
|
||||
expectedOrder: nil,
|
||||
name: "single log query, negative limit",
|
||||
queries: makeRequest(logQuery(-1)).CompositeQuery.Queries,
|
||||
expectedError: true,
|
||||
},
|
||||
{
|
||||
name: "valid order name (should be included)",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"attribute.user:string:desc", "id:asc"},
|
||||
name: "single trace query, zero limit gets default",
|
||||
queries: makeRequest(traceQuery(0)).CompositeQuery.Queries,
|
||||
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
|
||||
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
|
||||
},
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "user",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
},
|
||||
{
|
||||
name: "valid order name (should be included)",
|
||||
queryParams: url.Values{
|
||||
"order_by": {"attribute.user.string:desc", "id:asc"},
|
||||
name: "trace operator alone, zero limit gets default",
|
||||
queries: makeRequest(traceOperatorQuery(0)).CompositeQuery.Queries,
|
||||
checkQueries: func(t *testing.T, q []qbtypes.QueryEnvelope) {
|
||||
assert.Equal(t, DefaultExportRowCountLimit, q[0].GetLimit())
|
||||
},
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "user.string",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
order, err := getExportQueryOrderBy(tt.queryParams)
|
||||
err := validateAndApplyDefaultExportLimits(tt.queries)
|
||||
if tt.expectedError {
|
||||
assert.Error(t, err)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, len(tt.expectedOrder), len(order))
|
||||
for i, expectedOrd := range tt.expectedOrder {
|
||||
assert.Equal(t, expectedOrd, order[i])
|
||||
if tt.checkQueries != nil {
|
||||
tt.checkQueries(t, tt.queries)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -529,13 +169,8 @@ func TestConstructCSVHeaderFromQueryResponse(t *testing.T) {
|
||||
|
||||
header := constructCSVHeaderFromQueryResponse(data)
|
||||
|
||||
// Since map iteration order is not guaranteed, check that all expected keys are present
|
||||
expectedKeys := []string{"timestamp", "message", "level", "id"}
|
||||
assert.Equal(t, len(expectedKeys), len(header))
|
||||
|
||||
for _, key := range expectedKeys {
|
||||
assert.Contains(t, header, key)
|
||||
}
|
||||
// Priority columns come first in order, then the rest alphabetically.
|
||||
assert.Equal(t, []string{"timestamp", "id", "level", "message"}, header)
|
||||
}
|
||||
|
||||
func TestConstructCSVRecordFromQueryResponse(t *testing.T) {
|
||||
|
||||
@@ -23,8 +23,18 @@ func NewModule(querier querier.Querier) rawdataexport.Module {
|
||||
|
||||
func (m *Module) ExportRawData(ctx context.Context, orgID valuer.UUID, rangeRequest *qbtypes.QueryRangeRequest, doneChan chan any) (chan *qbtypes.RawRow, chan error) {
|
||||
|
||||
spec := rangeRequest.CompositeQuery.Queries[0].Spec.(qbtypes.QueryBuilderQuery[qbtypes.LogAggregation])
|
||||
rowCountLimit := spec.Limit
|
||||
traceOperatorQueryIndex := rangeRequest.TraceOperatorQueryIndex()
|
||||
|
||||
queries := rangeRequest.CompositeQuery.Queries
|
||||
|
||||
// If the trace operator query is present, mark the queries other than trace operator as disabled
|
||||
if traceOperatorQueryIndex > -1 {
|
||||
for idx := range len(queries) {
|
||||
if idx != traceOperatorQueryIndex {
|
||||
queries[idx].SetDisabled(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rowChan := make(chan *qbtypes.RawRow, 1)
|
||||
errChan := make(chan error, 1)
|
||||
@@ -38,52 +48,62 @@ func (m *Module) ExportRawData(ctx context.Context, orgID valuer.UUID, rangeRequ
|
||||
defer close(errChan)
|
||||
defer close(rowChan)
|
||||
|
||||
rowCount := 0
|
||||
|
||||
for rowCount < rowCountLimit {
|
||||
spec.Limit = min(ChunkSize, rowCountLimit-rowCount)
|
||||
spec.Offset = rowCount
|
||||
|
||||
rangeRequest.CompositeQuery.Queries[0].Spec = spec
|
||||
|
||||
response, err := m.querier.QueryRange(contextWithTimeout, orgID, rangeRequest)
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
|
||||
newRowsCount := 0
|
||||
for _, result := range response.Data.Results {
|
||||
resultData, ok := result.(*qbtypes.RawData)
|
||||
if !ok {
|
||||
errChan <- errors.NewInternalf(errors.CodeInternal, "expected RawData, got %T", result)
|
||||
return
|
||||
}
|
||||
|
||||
newRowsCount += len(resultData.Rows)
|
||||
for _, row := range resultData.Rows {
|
||||
select {
|
||||
case rowChan <- row:
|
||||
case <-doneChan:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
errChan <- ctx.Err()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Break if we did not receive any new rows
|
||||
if newRowsCount == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
rowCount += newRowsCount
|
||||
|
||||
if traceOperatorQueryIndex > -1 {
|
||||
// If the trace operator query is present, we need to export the data for the trace operator query only
|
||||
exportRawDataForSingleQuery(m.querier, contextWithTimeout, orgID, rangeRequest, rowChan, errChan, doneChan, traceOperatorQueryIndex)
|
||||
} else {
|
||||
// If the trace operator query is not present, we need to export the data for the first query only
|
||||
exportRawDataForSingleQuery(m.querier, contextWithTimeout, orgID, rangeRequest, rowChan, errChan, doneChan, 0)
|
||||
}
|
||||
}()
|
||||
|
||||
return rowChan, errChan
|
||||
|
||||
}
|
||||
|
||||
func exportRawDataForSingleQuery(querier querier.Querier, ctx context.Context, orgID valuer.UUID, rangeRequest *qbtypes.QueryRangeRequest, rowChan chan *qbtypes.RawRow, errChan chan error, doneChan chan any, queryIndex int) {
|
||||
|
||||
queries := rangeRequest.CompositeQuery.Queries
|
||||
rowCountLimit := queries[queryIndex].GetLimit()
|
||||
rowCount := 0
|
||||
|
||||
for rowCount < rowCountLimit {
|
||||
chunkSize := min(ChunkSize, rowCountLimit-rowCount)
|
||||
queries[queryIndex].SetLimit(chunkSize)
|
||||
queries[queryIndex].SetOffset(rowCount)
|
||||
|
||||
response, err := querier.QueryRange(ctx, orgID, rangeRequest)
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
|
||||
newRowsCount := 0
|
||||
for _, result := range response.Data.Results {
|
||||
resultData, ok := result.(*qbtypes.RawData)
|
||||
if !ok {
|
||||
errChan <- errors.NewInternalf(errors.CodeInternal, "expected RawData, got %T", result)
|
||||
return
|
||||
}
|
||||
|
||||
newRowsCount += len(resultData.Rows)
|
||||
for _, row := range resultData.Rows {
|
||||
select {
|
||||
case rowChan <- row:
|
||||
case <-doneChan:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
errChan <- ctx.Err()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rowCount += newRowsCount
|
||||
|
||||
// Stop if we received fewer rows than requested — no more data available
|
||||
if newRowsCount < chunkSize {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/savedviewtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
@@ -25,7 +24,7 @@ func NewModule(sqlstore sqlstore.SQLStore) savedview.Module {
|
||||
}
|
||||
|
||||
func (module *module) GetViewsForFilters(ctx context.Context, orgID string, sourcePage string, name string, category string) ([]*v3.SavedView, error) {
|
||||
var views []savedviewtypes.SavedView
|
||||
var views []types.SavedView
|
||||
var err error
|
||||
if len(category) == 0 {
|
||||
err = module.sqlstore.BunDB().NewSelect().Model(&views).Where("org_id = ? AND source_page = ? AND name LIKE ?", orgID, sourcePage, "%"+name+"%").Scan(ctx)
|
||||
@@ -77,7 +76,7 @@ func (module *module) CreateView(ctx context.Context, orgID string, view v3.Save
|
||||
createBy := claims.Email
|
||||
updatedBy := claims.Email
|
||||
|
||||
dbView := savedviewtypes.SavedView{
|
||||
dbView := types.SavedView{
|
||||
TimeAuditable: types.TimeAuditable{
|
||||
CreatedAt: createdAt,
|
||||
UpdatedAt: updatedAt,
|
||||
@@ -106,7 +105,7 @@ func (module *module) CreateView(ctx context.Context, orgID string, view v3.Save
|
||||
}
|
||||
|
||||
func (module *module) GetView(ctx context.Context, orgID string, uuid valuer.UUID) (*v3.SavedView, error) {
|
||||
var view savedviewtypes.SavedView
|
||||
var view types.SavedView
|
||||
err := module.sqlstore.BunDB().NewSelect().Model(&view).Where("org_id = ? AND id = ?", orgID, uuid.StringValue()).Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error in getting saved view")
|
||||
@@ -147,7 +146,7 @@ func (module *module) UpdateView(ctx context.Context, orgID string, uuid valuer.
|
||||
updatedBy := claims.Email
|
||||
|
||||
_, err = module.sqlstore.BunDB().NewUpdate().
|
||||
Model(&savedviewtypes.SavedView{}).
|
||||
Model(&types.SavedView{}).
|
||||
Set("updated_at = ?, updated_by = ?, name = ?, category = ?, source_page = ?, tags = ?, data = ?, extra_data = ?",
|
||||
updatedAt, updatedBy, view.Name, view.Category, view.SourcePage, strings.Join(view.Tags, ","), data, view.ExtraData).
|
||||
Where("id = ?", uuid.StringValue()).
|
||||
@@ -161,7 +160,7 @@ func (module *module) UpdateView(ctx context.Context, orgID string, uuid valuer.
|
||||
|
||||
func (module *module) DeleteView(ctx context.Context, orgID string, uuid valuer.UUID) error {
|
||||
_, err := module.sqlstore.BunDB().NewDelete().
|
||||
Model(&savedviewtypes.SavedView{}).
|
||||
Model(&types.SavedView{}).
|
||||
Where("id = ?", uuid.StringValue()).
|
||||
Where("org_id = ?", orgID).
|
||||
Exec(ctx)
|
||||
@@ -172,7 +171,7 @@ func (module *module) DeleteView(ctx context.Context, orgID string, uuid valuer.
|
||||
}
|
||||
|
||||
func (module *module) Collect(ctx context.Context, orgID valuer.UUID) (map[string]any, error) {
|
||||
savedViews := []*savedviewtypes.SavedView{}
|
||||
savedViews := []*types.SavedView{}
|
||||
|
||||
err := module.
|
||||
sqlstore.
|
||||
@@ -185,5 +184,5 @@ func (module *module) Collect(ctx context.Context, orgID valuer.UUID) (map[strin
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return savedviewtypes.NewStatsFromSavedViews(savedViews), nil
|
||||
return types.NewStatsFromSavedViews(savedViews), nil
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
root "github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
@@ -463,7 +462,7 @@ func (h *handler) UpdateAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if slices.Contains(integrationtypes.AllIntegrationUserEmails, integrationtypes.IntegrationUserEmail(createdByUser.Email.String())) {
|
||||
if slices.Contains(types.AllIntegrationUserEmails, types.IntegrationUserEmail(createdByUser.Email.String())) {
|
||||
render.Error(w, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "API Keys for integration users cannot be revoked"))
|
||||
return
|
||||
}
|
||||
@@ -508,7 +507,7 @@ func (h *handler) RevokeAPIKey(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if slices.Contains(integrationtypes.AllIntegrationUserEmails, integrationtypes.IntegrationUserEmail(createdByUser.Email.String())) {
|
||||
if slices.Contains(types.AllIntegrationUserEmails, types.IntegrationUserEmail(createdByUser.Email.String())) {
|
||||
render.Error(w, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "API Keys for integration users cannot be revoked"))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/emailtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/roletypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/dustin/go-humanize"
|
||||
@@ -280,7 +279,7 @@ func (module *Module) DeleteUser(ctx context.Context, orgID valuer.UUID, id stri
|
||||
return errors.WithAdditionalf(err, "cannot delete root user")
|
||||
}
|
||||
|
||||
if slices.Contains(integrationtypes.AllIntegrationUserEmails, integrationtypes.IntegrationUserEmail(user.Email.String())) {
|
||||
if slices.Contains(types.AllIntegrationUserEmails, types.IntegrationUserEmail(user.Email.String())) {
|
||||
return errors.New(errors.TypeForbidden, errors.CodeForbidden, "integration user cannot be deleted")
|
||||
}
|
||||
|
||||
|
||||
@@ -10,16 +10,15 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
type cloudProviderAccountsRepository interface {
|
||||
listConnected(ctx context.Context, orgId string, provider string) ([]integrationtypes.CloudIntegration, *model.ApiError)
|
||||
listConnected(ctx context.Context, orgId string, provider string) ([]types.CloudIntegration, *model.ApiError)
|
||||
|
||||
get(ctx context.Context, orgId string, provider string, id string) (*integrationtypes.CloudIntegration, *model.ApiError)
|
||||
get(ctx context.Context, orgId string, provider string, id string) (*types.CloudIntegration, *model.ApiError)
|
||||
|
||||
getConnectedCloudAccount(ctx context.Context, orgId string, provider string, accountID string) (*integrationtypes.CloudIntegration, *model.ApiError)
|
||||
getConnectedCloudAccount(ctx context.Context, orgId string, provider string, accountID string) (*types.CloudIntegration, *model.ApiError)
|
||||
|
||||
// Insert an account or update it by (cloudProvider, id)
|
||||
// for specified non-empty fields
|
||||
@@ -28,11 +27,11 @@ type cloudProviderAccountsRepository interface {
|
||||
orgId string,
|
||||
provider string,
|
||||
id *string,
|
||||
config *integrationtypes.AccountConfig,
|
||||
config *types.AccountConfig,
|
||||
accountId *string,
|
||||
agentReport *integrationtypes.AgentReport,
|
||||
agentReport *types.AgentReport,
|
||||
removedAt *time.Time,
|
||||
) (*integrationtypes.CloudIntegration, *model.ApiError)
|
||||
) (*types.CloudIntegration, *model.ApiError)
|
||||
}
|
||||
|
||||
func newCloudProviderAccountsRepository(store sqlstore.SQLStore) (
|
||||
@@ -49,8 +48,8 @@ type cloudProviderAccountsSQLRepository struct {
|
||||
|
||||
func (r *cloudProviderAccountsSQLRepository) listConnected(
|
||||
ctx context.Context, orgId string, cloudProvider string,
|
||||
) ([]integrationtypes.CloudIntegration, *model.ApiError) {
|
||||
accounts := []integrationtypes.CloudIntegration{}
|
||||
) ([]types.CloudIntegration, *model.ApiError) {
|
||||
accounts := []types.CloudIntegration{}
|
||||
|
||||
err := r.store.BunDB().NewSelect().
|
||||
Model(&accounts).
|
||||
@@ -73,8 +72,8 @@ func (r *cloudProviderAccountsSQLRepository) listConnected(
|
||||
|
||||
func (r *cloudProviderAccountsSQLRepository) get(
|
||||
ctx context.Context, orgId string, provider string, id string,
|
||||
) (*integrationtypes.CloudIntegration, *model.ApiError) {
|
||||
var result integrationtypes.CloudIntegration
|
||||
) (*types.CloudIntegration, *model.ApiError) {
|
||||
var result types.CloudIntegration
|
||||
|
||||
err := r.store.BunDB().NewSelect().
|
||||
Model(&result).
|
||||
@@ -98,8 +97,8 @@ func (r *cloudProviderAccountsSQLRepository) get(
|
||||
|
||||
func (r *cloudProviderAccountsSQLRepository) getConnectedCloudAccount(
|
||||
ctx context.Context, orgId string, provider string, accountId string,
|
||||
) (*integrationtypes.CloudIntegration, *model.ApiError) {
|
||||
var result integrationtypes.CloudIntegration
|
||||
) (*types.CloudIntegration, *model.ApiError) {
|
||||
var result types.CloudIntegration
|
||||
|
||||
err := r.store.BunDB().NewSelect().
|
||||
Model(&result).
|
||||
@@ -128,11 +127,11 @@ func (r *cloudProviderAccountsSQLRepository) upsert(
|
||||
orgId string,
|
||||
provider string,
|
||||
id *string,
|
||||
config *integrationtypes.AccountConfig,
|
||||
config *types.AccountConfig,
|
||||
accountId *string,
|
||||
agentReport *integrationtypes.AgentReport,
|
||||
agentReport *types.AgentReport,
|
||||
removedAt *time.Time,
|
||||
) (*integrationtypes.CloudIntegration, *model.ApiError) {
|
||||
) (*types.CloudIntegration, *model.ApiError) {
|
||||
// Insert
|
||||
if id == nil {
|
||||
temp := valuer.GenerateUUID().StringValue()
|
||||
@@ -182,7 +181,7 @@ func (r *cloudProviderAccountsSQLRepository) upsert(
|
||||
)
|
||||
}
|
||||
|
||||
integration := integrationtypes.CloudIntegration{
|
||||
integration := types.CloudIntegration{
|
||||
OrgID: orgId,
|
||||
Provider: provider,
|
||||
Identifiable: types.Identifiable{ID: valuer.MustNewUUID(*id)},
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"golang.org/x/exp/maps"
|
||||
)
|
||||
@@ -53,7 +52,7 @@ func NewController(sqlStore sqlstore.SQLStore) (*Controller, error) {
|
||||
}
|
||||
|
||||
type ConnectedAccountsListResponse struct {
|
||||
Accounts []integrationtypes.Account `json:"accounts"`
|
||||
Accounts []types.Account `json:"accounts"`
|
||||
}
|
||||
|
||||
func (c *Controller) ListConnectedAccounts(ctx context.Context, orgId string, cloudProvider string) (
|
||||
@@ -68,7 +67,7 @@ func (c *Controller) ListConnectedAccounts(ctx context.Context, orgId string, cl
|
||||
return nil, model.WrapApiError(apiErr, "couldn't list cloud accounts")
|
||||
}
|
||||
|
||||
connectedAccounts := []integrationtypes.Account{}
|
||||
connectedAccounts := []types.Account{}
|
||||
for _, a := range accountRecords {
|
||||
connectedAccounts = append(connectedAccounts, a.Account())
|
||||
}
|
||||
@@ -82,7 +81,7 @@ type GenerateConnectionUrlRequest struct {
|
||||
// Optional. To be specified for updates.
|
||||
AccountId *string `json:"account_id,omitempty"`
|
||||
|
||||
AccountConfig integrationtypes.AccountConfig `json:"account_config"`
|
||||
AccountConfig types.AccountConfig `json:"account_config"`
|
||||
|
||||
AgentConfig SigNozAgentConfig `json:"agent_config"`
|
||||
}
|
||||
@@ -150,9 +149,9 @@ func (c *Controller) GenerateConnectionUrl(ctx context.Context, orgId string, cl
|
||||
}
|
||||
|
||||
type AccountStatusResponse struct {
|
||||
Id string `json:"id"`
|
||||
CloudAccountId *string `json:"cloud_account_id,omitempty"`
|
||||
Status integrationtypes.AccountStatus `json:"status"`
|
||||
Id string `json:"id"`
|
||||
CloudAccountId *string `json:"cloud_account_id,omitempty"`
|
||||
Status types.AccountStatus `json:"status"`
|
||||
}
|
||||
|
||||
func (c *Controller) GetAccountStatus(ctx context.Context, orgId string, cloudProvider string, accountId string) (
|
||||
@@ -218,7 +217,7 @@ func (c *Controller) CheckInAsAgent(ctx context.Context, orgId string, cloudProv
|
||||
))
|
||||
}
|
||||
|
||||
agentReport := integrationtypes.AgentReport{
|
||||
agentReport := types.AgentReport{
|
||||
TimestampMillis: time.Now().UnixMilli(),
|
||||
Data: req.Data,
|
||||
}
|
||||
@@ -287,10 +286,10 @@ func (c *Controller) CheckInAsAgent(ctx context.Context, orgId string, cloudProv
|
||||
}
|
||||
|
||||
type UpdateAccountConfigRequest struct {
|
||||
Config integrationtypes.AccountConfig `json:"config"`
|
||||
Config types.AccountConfig `json:"config"`
|
||||
}
|
||||
|
||||
func (c *Controller) UpdateAccountConfig(ctx context.Context, orgId string, cloudProvider string, accountId string, req UpdateAccountConfigRequest) (*integrationtypes.Account, *model.ApiError) {
|
||||
func (c *Controller) UpdateAccountConfig(ctx context.Context, orgId string, cloudProvider string, accountId string, req UpdateAccountConfigRequest) (*types.Account, *model.ApiError) {
|
||||
if apiErr := validateCloudProviderName(cloudProvider); apiErr != nil {
|
||||
return nil, apiErr
|
||||
}
|
||||
@@ -307,7 +306,7 @@ func (c *Controller) UpdateAccountConfig(ctx context.Context, orgId string, clou
|
||||
return &account, nil
|
||||
}
|
||||
|
||||
func (c *Controller) DisconnectAccount(ctx context.Context, orgId string, cloudProvider string, accountId string) (*integrationtypes.CloudIntegration, *model.ApiError) {
|
||||
func (c *Controller) DisconnectAccount(ctx context.Context, orgId string, cloudProvider string, accountId string) (*types.CloudIntegration, *model.ApiError) {
|
||||
if apiErr := validateCloudProviderName(cloudProvider); apiErr != nil {
|
||||
return nil, apiErr
|
||||
}
|
||||
@@ -347,7 +346,7 @@ func (c *Controller) ListServices(
|
||||
return nil, model.WrapApiError(apiErr, "couldn't list cloud services")
|
||||
}
|
||||
|
||||
svcConfigs := map[string]*integrationtypes.CloudServiceConfig{}
|
||||
svcConfigs := map[string]*types.CloudServiceConfig{}
|
||||
if cloudAccountId != nil {
|
||||
activeAccount, apiErr := c.accountsRepo.getConnectedCloudAccount(
|
||||
ctx, orgID, cloudProvider, *cloudAccountId,
|
||||
@@ -442,8 +441,8 @@ func (c *Controller) GetServiceDetails(
|
||||
}
|
||||
|
||||
type UpdateServiceConfigRequest struct {
|
||||
CloudAccountId string `json:"cloud_account_id"`
|
||||
Config integrationtypes.CloudServiceConfig `json:"config"`
|
||||
CloudAccountId string `json:"cloud_account_id"`
|
||||
Config types.CloudServiceConfig `json:"config"`
|
||||
}
|
||||
|
||||
func (u *UpdateServiceConfigRequest) Validate(def *services.Definition) error {
|
||||
@@ -461,8 +460,8 @@ func (u *UpdateServiceConfigRequest) Validate(def *services.Definition) error {
|
||||
}
|
||||
|
||||
type UpdateServiceConfigResponse struct {
|
||||
Id string `json:"id"`
|
||||
Config integrationtypes.CloudServiceConfig `json:"config"`
|
||||
Id string `json:"id"`
|
||||
Config types.CloudServiceConfig `json:"config"`
|
||||
}
|
||||
|
||||
func (c *Controller) UpdateServiceConfig(
|
||||
|
||||
@@ -3,20 +3,20 @@ package cloudintegrations
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/cloudintegrations/services"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
)
|
||||
|
||||
type ServiceSummary struct {
|
||||
services.Metadata
|
||||
|
||||
Config *integrationtypes.CloudServiceConfig `json:"config"`
|
||||
Config *types.CloudServiceConfig `json:"config"`
|
||||
}
|
||||
|
||||
type ServiceDetails struct {
|
||||
services.Definition
|
||||
|
||||
Config *integrationtypes.CloudServiceConfig `json:"config"`
|
||||
ConnectionStatus *ServiceConnectionStatus `json:"status,omitempty"`
|
||||
Config *types.CloudServiceConfig `json:"config"`
|
||||
ConnectionStatus *ServiceConnectionStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
type AccountStatus struct {
|
||||
@@ -61,7 +61,7 @@ func NewCompiledCollectionStrategy(provider string) (*CompiledCollectionStrategy
|
||||
|
||||
// Helper for accumulating strategies for enabled services.
|
||||
func AddServiceStrategy(serviceType string, cs *CompiledCollectionStrategy,
|
||||
definitionStrat *services.CollectionStrategy, config *integrationtypes.CloudServiceConfig) error {
|
||||
definitionStrat *services.CollectionStrategy, config *types.CloudServiceConfig) error {
|
||||
if definitionStrat.Provider != cs.Provider {
|
||||
return errors.NewInternalf(CodeMismatchCloudProvider, "can't add %s service strategy to compiled strategy for %s",
|
||||
definitionStrat.Provider, cs.Provider)
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
@@ -19,7 +18,7 @@ type ServiceConfigDatabase interface {
|
||||
orgID string,
|
||||
cloudAccountId string,
|
||||
serviceType string,
|
||||
) (*integrationtypes.CloudServiceConfig, *model.ApiError)
|
||||
) (*types.CloudServiceConfig, *model.ApiError)
|
||||
|
||||
upsert(
|
||||
ctx context.Context,
|
||||
@@ -27,15 +26,15 @@ type ServiceConfigDatabase interface {
|
||||
cloudProvider string,
|
||||
cloudAccountId string,
|
||||
serviceId string,
|
||||
config integrationtypes.CloudServiceConfig,
|
||||
) (*integrationtypes.CloudServiceConfig, *model.ApiError)
|
||||
config types.CloudServiceConfig,
|
||||
) (*types.CloudServiceConfig, *model.ApiError)
|
||||
|
||||
getAllForAccount(
|
||||
ctx context.Context,
|
||||
orgID string,
|
||||
cloudAccountId string,
|
||||
) (
|
||||
configsBySvcId map[string]*integrationtypes.CloudServiceConfig,
|
||||
configsBySvcId map[string]*types.CloudServiceConfig,
|
||||
apiErr *model.ApiError,
|
||||
)
|
||||
}
|
||||
@@ -57,9 +56,9 @@ func (r *serviceConfigSQLRepository) get(
|
||||
orgID string,
|
||||
cloudAccountId string,
|
||||
serviceType string,
|
||||
) (*integrationtypes.CloudServiceConfig, *model.ApiError) {
|
||||
) (*types.CloudServiceConfig, *model.ApiError) {
|
||||
|
||||
var result integrationtypes.CloudIntegrationService
|
||||
var result types.CloudIntegrationService
|
||||
|
||||
err := r.store.BunDB().NewSelect().
|
||||
Model(&result).
|
||||
@@ -90,14 +89,14 @@ func (r *serviceConfigSQLRepository) upsert(
|
||||
cloudProvider string,
|
||||
cloudAccountId string,
|
||||
serviceId string,
|
||||
config integrationtypes.CloudServiceConfig,
|
||||
) (*integrationtypes.CloudServiceConfig, *model.ApiError) {
|
||||
config types.CloudServiceConfig,
|
||||
) (*types.CloudServiceConfig, *model.ApiError) {
|
||||
|
||||
// get cloud integration id from account id
|
||||
// if the account is not connected, we don't need to upsert the config
|
||||
var cloudIntegrationId string
|
||||
err := r.store.BunDB().NewSelect().
|
||||
Model((*integrationtypes.CloudIntegration)(nil)).
|
||||
Model((*types.CloudIntegration)(nil)).
|
||||
Column("id").
|
||||
Where("provider = ?", cloudProvider).
|
||||
Where("account_id = ?", cloudAccountId).
|
||||
@@ -112,7 +111,7 @@ func (r *serviceConfigSQLRepository) upsert(
|
||||
))
|
||||
}
|
||||
|
||||
serviceConfig := integrationtypes.CloudIntegrationService{
|
||||
serviceConfig := types.CloudIntegrationService{
|
||||
Identifiable: types.Identifiable{ID: valuer.GenerateUUID()},
|
||||
TimeAuditable: types.TimeAuditable{
|
||||
CreatedAt: time.Now(),
|
||||
@@ -140,8 +139,8 @@ func (r *serviceConfigSQLRepository) getAllForAccount(
|
||||
ctx context.Context,
|
||||
orgID string,
|
||||
cloudAccountId string,
|
||||
) (map[string]*integrationtypes.CloudServiceConfig, *model.ApiError) {
|
||||
serviceConfigs := []integrationtypes.CloudIntegrationService{}
|
||||
) (map[string]*types.CloudServiceConfig, *model.ApiError) {
|
||||
serviceConfigs := []types.CloudIntegrationService{}
|
||||
|
||||
err := r.store.BunDB().NewSelect().
|
||||
Model(&serviceConfigs).
|
||||
@@ -155,7 +154,7 @@ func (r *serviceConfigSQLRepository) getAllForAccount(
|
||||
))
|
||||
}
|
||||
|
||||
result := map[string]*integrationtypes.CloudServiceConfig{}
|
||||
result := map[string]*types.CloudServiceConfig{}
|
||||
|
||||
for _, r := range serviceConfigs {
|
||||
result[r.Type] = &r.Config
|
||||
|
||||
@@ -572,9 +572,6 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *middleware.AuthZ) {
|
||||
aH.LicensingAPI.Activate(rw, req)
|
||||
})).Methods(http.MethodGet)
|
||||
|
||||
// Export
|
||||
router.HandleFunc("/api/v1/export_raw_data", am.ViewAccess(aH.Signoz.Handlers.RawDataExport.ExportRawData)).Methods(http.MethodGet)
|
||||
|
||||
router.HandleFunc("/api/v1/span_percentile", am.ViewAccess(aH.Signoz.Handlers.SpanPercentile.GetSpanPercentileDetails)).Methods(http.MethodPost)
|
||||
|
||||
// Query Filter Analyzer api used to extract metric names and grouping columns from a query
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
|
||||
ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -108,7 +107,7 @@ type IntegrationsListItem struct {
|
||||
|
||||
type Integration struct {
|
||||
IntegrationDetails
|
||||
Installation *integrationtypes.InstalledIntegration `json:"installation"`
|
||||
Installation *types.InstalledIntegration `json:"installation"`
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
@@ -224,7 +223,7 @@ func (m *Manager) InstallIntegration(
|
||||
ctx context.Context,
|
||||
orgId string,
|
||||
integrationId string,
|
||||
config integrationtypes.InstalledIntegrationConfig,
|
||||
config types.InstalledIntegrationConfig,
|
||||
) (*IntegrationsListItem, *model.ApiError) {
|
||||
integrationDetails, apiErr := m.getIntegrationDetails(ctx, integrationId)
|
||||
if apiErr != nil {
|
||||
@@ -430,7 +429,7 @@ func (m *Manager) getInstalledIntegration(
|
||||
ctx context.Context,
|
||||
orgId string,
|
||||
integrationId string,
|
||||
) (*integrationtypes.InstalledIntegration, *model.ApiError) {
|
||||
) (*types.InstalledIntegration, *model.ApiError) {
|
||||
iis, apiErr := m.installedIntegrationsRepo.get(
|
||||
ctx, orgId, []string{integrationId},
|
||||
)
|
||||
@@ -458,7 +457,7 @@ func (m *Manager) getInstalledIntegrations(
|
||||
return nil, apiErr
|
||||
}
|
||||
|
||||
installedTypes := utils.MapSlice(installations, func(i integrationtypes.InstalledIntegration) string {
|
||||
installedTypes := utils.MapSlice(installations, func(i types.InstalledIntegration) string {
|
||||
return i.Type
|
||||
})
|
||||
integrationDetails, apiErr := m.availableIntegrationsRepo.get(ctx, installedTypes)
|
||||
|
||||
@@ -4,22 +4,22 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
)
|
||||
|
||||
type InstalledIntegrationsRepo interface {
|
||||
list(ctx context.Context, orgId string) ([]integrationtypes.InstalledIntegration, *model.ApiError)
|
||||
list(ctx context.Context, orgId string) ([]types.InstalledIntegration, *model.ApiError)
|
||||
|
||||
get(
|
||||
ctx context.Context, orgId string, integrationTypes []string,
|
||||
) (map[string]integrationtypes.InstalledIntegration, *model.ApiError)
|
||||
) (map[string]types.InstalledIntegration, *model.ApiError)
|
||||
|
||||
upsert(
|
||||
ctx context.Context,
|
||||
orgId string,
|
||||
integrationType string,
|
||||
config integrationtypes.InstalledIntegrationConfig,
|
||||
) (*integrationtypes.InstalledIntegration, *model.ApiError)
|
||||
config types.InstalledIntegrationConfig,
|
||||
) (*types.InstalledIntegration, *model.ApiError)
|
||||
|
||||
delete(ctx context.Context, orgId string, integrationType string) *model.ApiError
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
@@ -27,8 +26,8 @@ func NewInstalledIntegrationsSqliteRepo(store sqlstore.SQLStore) (
|
||||
func (r *InstalledIntegrationsSqliteRepo) list(
|
||||
ctx context.Context,
|
||||
orgId string,
|
||||
) ([]integrationtypes.InstalledIntegration, *model.ApiError) {
|
||||
integrations := []integrationtypes.InstalledIntegration{}
|
||||
) ([]types.InstalledIntegration, *model.ApiError) {
|
||||
integrations := []types.InstalledIntegration{}
|
||||
|
||||
err := r.store.BunDB().NewSelect().
|
||||
Model(&integrations).
|
||||
@@ -45,8 +44,8 @@ func (r *InstalledIntegrationsSqliteRepo) list(
|
||||
|
||||
func (r *InstalledIntegrationsSqliteRepo) get(
|
||||
ctx context.Context, orgId string, integrationTypes []string,
|
||||
) (map[string]integrationtypes.InstalledIntegration, *model.ApiError) {
|
||||
integrations := []integrationtypes.InstalledIntegration{}
|
||||
) (map[string]types.InstalledIntegration, *model.ApiError) {
|
||||
integrations := []types.InstalledIntegration{}
|
||||
|
||||
typeValues := []interface{}{}
|
||||
for _, integrationType := range integrationTypes {
|
||||
@@ -63,7 +62,7 @@ func (r *InstalledIntegrationsSqliteRepo) get(
|
||||
))
|
||||
}
|
||||
|
||||
result := map[string]integrationtypes.InstalledIntegration{}
|
||||
result := map[string]types.InstalledIntegration{}
|
||||
for _, ii := range integrations {
|
||||
result[ii.Type] = ii
|
||||
}
|
||||
@@ -75,10 +74,10 @@ func (r *InstalledIntegrationsSqliteRepo) upsert(
|
||||
ctx context.Context,
|
||||
orgId string,
|
||||
integrationType string,
|
||||
config integrationtypes.InstalledIntegrationConfig,
|
||||
) (*integrationtypes.InstalledIntegration, *model.ApiError) {
|
||||
config types.InstalledIntegrationConfig,
|
||||
) (*types.InstalledIntegration, *model.ApiError) {
|
||||
|
||||
integration := integrationtypes.InstalledIntegration{
|
||||
integration := types.InstalledIntegration{
|
||||
Identifiable: types.Identifiable{
|
||||
ID: valuer.GenerateUUID(),
|
||||
},
|
||||
@@ -115,7 +114,7 @@ func (r *InstalledIntegrationsSqliteRepo) delete(
|
||||
ctx context.Context, orgId string, integrationType string,
|
||||
) *model.ApiError {
|
||||
_, dbErr := r.store.BunDB().NewDelete().
|
||||
Model(&integrationtypes.InstalledIntegration{}).
|
||||
Model(&types.InstalledIntegration{}).
|
||||
Where("type = ?", integrationType).
|
||||
Where("org_id = ?", orgId).
|
||||
Exec(ctx)
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/preference"
|
||||
"github.com/SigNoz/signoz/pkg/modules/promote"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
|
||||
"github.com/SigNoz/signoz/pkg/modules/session"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
@@ -57,6 +58,7 @@ func NewOpenAPI(ctx context.Context, instrumentation instrumentation.Instrumenta
|
||||
struct{ gateway.Handler }{},
|
||||
struct{ fields.Handler }{},
|
||||
struct{ authz.Handler }{},
|
||||
struct{ rawdataexport.Handler }{},
|
||||
struct{ zeus.Handler }{},
|
||||
struct{ querier.Handler }{},
|
||||
).New(ctx, instrumentation.ToProviderSettings(), apiserver.Config{})
|
||||
|
||||
@@ -253,6 +253,7 @@ func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.Au
|
||||
handlers.GatewayHandler,
|
||||
handlers.Fields,
|
||||
handlers.AuthzHandler,
|
||||
handlers.RawDataExport,
|
||||
handlers.ZeusHandler,
|
||||
handlers.QuerierHandler,
|
||||
),
|
||||
|
||||
120
pkg/types/exporttypes/rawdataexport.go
Normal file
120
pkg/types/exporttypes/rawdataexport.go
Normal file
@@ -0,0 +1,120 @@
|
||||
package exporttypes
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// ExportRawDataQueryParams represents the query parameters for the export raw data endpoint
|
||||
type ExportRawDataQueryParams struct {
|
||||
ExportRawDataFormatQueryParam
|
||||
|
||||
// Signal specifies the type of data to export: "logs" or "traces"
|
||||
Signal telemetrytypes.Signal `query:"signal" enum:"logs,traces" required:"true"`
|
||||
|
||||
// Source specifies the type of data to export: "logs" or "traces"
|
||||
// Deprecated: Use Signal instead.
|
||||
Source string `query:"source" deprecated:"true"`
|
||||
|
||||
// Start is the start time for the query (Unix timestamp in nanoseconds)
|
||||
Start uint64 `query:"start"`
|
||||
|
||||
// End is the end time for the query (Unix timestamp in nanoseconds)
|
||||
End uint64 `query:"end"`
|
||||
|
||||
// Limit specifies the maximum number of rows to export
|
||||
Limit int `query:"limit,default=10000" default:"10000" minimum:"1" maximum:"50000"`
|
||||
|
||||
// Filter is a filter expression to apply to the query
|
||||
FilterString string `query:"filter" deprecated:"true"`
|
||||
Filter qbtypes.Filter `query:"filterExpression"`
|
||||
|
||||
// Columns specifies the columns to include in the export
|
||||
// Format: ["context.field:type", "context.field", "field"]
|
||||
Columns []string `query:"columns" deprecated:"true"`
|
||||
|
||||
// SelectFields specifies the columns to include in the export
|
||||
SelectFields []telemetrytypes.TelemetryFieldKey `query:"selectFields"`
|
||||
|
||||
// OrderBy specifies the sorting order
|
||||
// Format: "column:direction" or "context.field:type:direction"
|
||||
// Direction can be "asc" or "desc"
|
||||
// ** Deprecated **
|
||||
OrderBy string `query:"order_by" deprecated:"true"`
|
||||
|
||||
// order by keys and directions
|
||||
Order []qbtypes.OrderBy `query:"order"`
|
||||
}
|
||||
|
||||
type ExportRawDataFormatQueryParam struct {
|
||||
// Format specifies the output format: "csv" or "jsonl"
|
||||
Format string `query:"format,default=csv" default:"csv" enum:"csv,jsonl"`
|
||||
}
|
||||
|
||||
func (p *ExportRawDataQueryParams) Normalize() {
|
||||
if len(p.Order) == 0 && len(p.OrderBy) > 0 {
|
||||
p.Order = parseExportQueryOrderBy(p.OrderBy)
|
||||
}
|
||||
|
||||
if len(p.SelectFields) == 0 && len(p.Columns) != 0 {
|
||||
p.SelectFields = parseExportQueryColumns(p.Columns)
|
||||
|
||||
}
|
||||
|
||||
if len(p.Filter.Expression) == 0 && len(p.FilterString) > 0 {
|
||||
p.Filter = qbtypes.Filter{Expression: p.FilterString}
|
||||
}
|
||||
|
||||
if p.Signal == telemetrytypes.SignalUnspecified && p.Source != "" {
|
||||
p.Signal = telemetrytypes.Signal{String: valuer.NewString(p.Source)}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ExportRawDataQueryParams) Validate() error {
|
||||
|
||||
if p.Signal != telemetrytypes.SignalLogs && p.Signal != telemetrytypes.SignalTraces {
|
||||
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid signal %s", p.Signal).WithAdditional("Allowed values: [logs, traces]")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseExportQueryColumns converts bound column strings to TelemetryFieldKey structs.
|
||||
// Each column should be in the format "context.field:type" or "context.field" or "field"
|
||||
func parseExportQueryColumns(columnParams []string) []telemetrytypes.TelemetryFieldKey {
|
||||
columns := make([]telemetrytypes.TelemetryFieldKey, 0, len(columnParams))
|
||||
for _, columnStr := range columnParams {
|
||||
columnStr = strings.TrimSpace(columnStr)
|
||||
if columnStr == "" {
|
||||
continue
|
||||
}
|
||||
columns = append(columns, telemetrytypes.GetFieldKeyFromKeyText(columnStr))
|
||||
}
|
||||
return columns
|
||||
}
|
||||
|
||||
// parseExportQueryOrderBy converts a bound order_by string to an OrderBy slice.
|
||||
// The string should be in the format "column:direction" and is assumed already validated.
|
||||
func parseExportQueryOrderBy(orderByParam string) []qbtypes.OrderBy {
|
||||
orderByParam = strings.TrimSpace(orderByParam)
|
||||
if orderByParam == "" {
|
||||
return []qbtypes.OrderBy{}
|
||||
}
|
||||
|
||||
parts := strings.Split(orderByParam, ":")
|
||||
column := strings.Join(parts[:len(parts)-1], ":")
|
||||
direction := parts[len(parts)-1]
|
||||
|
||||
return []qbtypes.OrderBy{
|
||||
{
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.GetFieldKeyFromKeyText(column),
|
||||
},
|
||||
Direction: qbtypes.OrderDirectionMap[direction],
|
||||
},
|
||||
}
|
||||
}
|
||||
174
pkg/types/exporttypes/rawdataexport_test.go
Normal file
174
pkg/types/exporttypes/rawdataexport_test.go
Normal file
@@ -0,0 +1,174 @@
|
||||
package exporttypes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestParseExportQueryColumns(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input []string
|
||||
expectedColumns []telemetrytypes.TelemetryFieldKey
|
||||
}{
|
||||
{
|
||||
name: "empty input",
|
||||
input: []string{},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{},
|
||||
},
|
||||
{
|
||||
name: "single column",
|
||||
input: []string{"timestamp"},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple columns",
|
||||
input: []string{"timestamp", "message", "level"},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "message"},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty entry is skipped",
|
||||
input: []string{"timestamp", "", "level"},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "whitespace-only entry is skipped",
|
||||
input: []string{"timestamp", " ", "level"},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "timestamp"},
|
||||
{Name: "level"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "column with context and type",
|
||||
input: []string{"attribute.user:string"},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "user", FieldContext: telemetrytypes.FieldContextAttribute, FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "column with context, dot-notation name",
|
||||
input: []string{"attribute.user.string"},
|
||||
expectedColumns: []telemetrytypes.TelemetryFieldKey{
|
||||
{Name: "user.string", FieldContext: telemetrytypes.FieldContextAttribute},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
columns := parseExportQueryColumns(tt.input)
|
||||
assert.Equal(t, len(tt.expectedColumns), len(columns))
|
||||
for i, expected := range tt.expectedColumns {
|
||||
assert.Equal(t, expected, columns[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseExportQueryOrderBy(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expectedOrder []qbtypes.OrderBy
|
||||
}{
|
||||
{
|
||||
name: "empty string returns empty slice",
|
||||
input: "",
|
||||
expectedOrder: []qbtypes.OrderBy{},
|
||||
},
|
||||
{
|
||||
name: "simple column asc",
|
||||
input: "timestamp:asc",
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "timestamp"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "simple column desc",
|
||||
input: "timestamp:desc",
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "timestamp"},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "column with context and type qualifier",
|
||||
input: "attribute.user:string:desc",
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "user",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "column with context, dot-notation name",
|
||||
input: "attribute.user.string:desc",
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionDesc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "user.string",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "resource with context and type",
|
||||
input: "resource.service.name:string:asc",
|
||||
expectedOrder: []qbtypes.OrderBy{
|
||||
{
|
||||
Direction: qbtypes.OrderDirectionAsc,
|
||||
Key: qbtypes.OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
order := parseExportQueryOrderBy(tt.input)
|
||||
assert.Equal(t, len(tt.expectedOrder), len(order))
|
||||
for i, expected := range tt.expectedOrder {
|
||||
assert.Equal(t, expected, order[i])
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package integrationtypes
|
||||
package types
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
@@ -27,17 +26,17 @@ var AllIntegrationUserEmails = []IntegrationUserEmail{
|
||||
type InstalledIntegration struct {
|
||||
bun.BaseModel `bun:"table:installed_integration"`
|
||||
|
||||
types.Identifiable
|
||||
Identifiable
|
||||
Type string `json:"type" bun:"type,type:text,unique:org_id_type"`
|
||||
Config InstalledIntegrationConfig `json:"config" bun:"config,type:text"`
|
||||
InstalledAt time.Time `json:"installed_at" bun:"installed_at,default:current_timestamp"`
|
||||
OrgID string `json:"org_id" bun:"org_id,type:text,unique:org_id_type,references:organizations(id),on_delete:cascade"`
|
||||
}
|
||||
|
||||
type InstalledIntegrationConfig map[string]any
|
||||
type InstalledIntegrationConfig map[string]interface{}
|
||||
|
||||
// For serializing from db
|
||||
func (c *InstalledIntegrationConfig) Scan(src any) error {
|
||||
func (c *InstalledIntegrationConfig) Scan(src interface{}) error {
|
||||
var data []byte
|
||||
switch v := src.(type) {
|
||||
case []byte:
|
||||
@@ -68,8 +67,8 @@ func (c *InstalledIntegrationConfig) Value() (driver.Value, error) {
|
||||
type CloudIntegration struct {
|
||||
bun.BaseModel `bun:"table:cloud_integration"`
|
||||
|
||||
types.Identifiable
|
||||
types.TimeAuditable
|
||||
Identifiable
|
||||
TimeAuditable
|
||||
Provider string `json:"provider" bun:"provider,type:text,unique:provider_id"`
|
||||
Config *AccountConfig `json:"config" bun:"config,type:text"`
|
||||
AccountID *string `json:"account_id" bun:"account_id,type:text"`
|
||||
@@ -195,8 +194,8 @@ func (r *AgentReport) Value() (driver.Value, error) {
|
||||
type CloudIntegrationService struct {
|
||||
bun.BaseModel `bun:"table:cloud_integration_service,alias:cis"`
|
||||
|
||||
types.Identifiable
|
||||
types.TimeAuditable
|
||||
Identifiable
|
||||
TimeAuditable
|
||||
Type string `bun:"type,type:text,notnull,unique:cloud_integration_id_type"`
|
||||
Config CloudServiceConfig `bun:"config,type:text"`
|
||||
CloudIntegrationID string `bun:"cloud_integration_id,type:text,notnull,unique:cloud_integration_id_type,references:cloud_integrations(id),on_delete:cascade"`
|
||||
@@ -393,6 +393,59 @@ func (r *QueryRangeRequest) HasOrderSpecified() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// UseDefaultOrderBy applies a default order unless query has an explicit order provided.
|
||||
func (r *QueryRangeRequest) UseDefaultOrderBy() {
|
||||
queries := r.CompositeQuery.Queries
|
||||
for idx := range queries {
|
||||
switch queries[idx].Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation],
|
||||
QueryBuilderTraceOperator:
|
||||
if len(queries[idx].GetOrder()) == 0 {
|
||||
queries[idx].SetOrder(
|
||||
[]OrderBy{
|
||||
{
|
||||
Key: OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "timestamp",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeNumber,
|
||||
},
|
||||
},
|
||||
Direction: OrderDirectionDesc,
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
if len(queries[idx].GetOrder()) == 0 {
|
||||
queries[idx].SetOrder(
|
||||
[]OrderBy{
|
||||
{
|
||||
Key: OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "timestamp",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeNumber},
|
||||
},
|
||||
Direction: OrderDirectionDesc,
|
||||
},
|
||||
{
|
||||
Key: OrderByKey{
|
||||
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "id",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString},
|
||||
},
|
||||
Direction: OrderDirectionDesc,
|
||||
},
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *QueryRangeRequest) FuncsForQuery(name string) []Function {
|
||||
funcs := []Function{}
|
||||
for _, query := range r.CompositeQuery.Queries {
|
||||
@@ -437,6 +490,16 @@ func (r *QueryRangeRequest) IsAnomalyRequest() (*QueryBuilderQuery[MetricAggrega
|
||||
return &q, hasAnomaly
|
||||
}
|
||||
|
||||
func (r *QueryRangeRequest) TraceOperatorQueryIndex() int {
|
||||
for idx, query := range r.CompositeQuery.Queries {
|
||||
switch query.Spec.(type) {
|
||||
case TraceOperatorType:
|
||||
return idx
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// We do not support fill gaps for these queries. Maybe support in future?
|
||||
func (r *QueryRangeRequest) SkipFillGaps(name string) bool {
|
||||
for _, query := range r.CompositeQuery.Queries {
|
||||
|
||||
379
pkg/types/querybuildertypes/querybuildertypesv5/req_getters.go
Normal file
379
pkg/types/querybuildertypes/querybuildertypesv5/req_getters.go
Normal file
@@ -0,0 +1,379 @@
|
||||
package querybuildertypesv5
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
// GetExpression returns the expression string.
|
||||
func (q *QueryEnvelope) GetExpression() string {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Expression
|
||||
case QueryBuilderFormula:
|
||||
return spec.Expression
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetReturnSpansFrom returns the return-spans-from value.
|
||||
func (q *QueryEnvelope) GetReturnSpansFrom() string {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.ReturnSpansFrom
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetSignal returns the signal.
|
||||
func (q *QueryEnvelope) GetSignal() telemetrytypes.Signal {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Signal
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Signal
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Signal
|
||||
}
|
||||
return telemetrytypes.SignalUnspecified
|
||||
}
|
||||
|
||||
// GetSource returns the source.
|
||||
func (q *QueryEnvelope) GetSource() telemetrytypes.Source {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Source
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Source
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Source
|
||||
}
|
||||
return telemetrytypes.SourceUnspecified
|
||||
}
|
||||
|
||||
// GetQuery returns the raw query string.
|
||||
func (q *QueryEnvelope) GetQuery() string {
|
||||
switch spec := q.Spec.(type) {
|
||||
case PromQuery:
|
||||
return spec.Query
|
||||
case ClickHouseQuery:
|
||||
return spec.Query
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetStats returns the PromQL stats flag.
|
||||
func (q *QueryEnvelope) GetStats() bool {
|
||||
switch spec := q.Spec.(type) {
|
||||
case PromQuery:
|
||||
return spec.Stats
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetLeft returns the left query reference of a join.
|
||||
func (q *QueryEnvelope) GetLeft() QueryRef {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
return spec.Left
|
||||
}
|
||||
return QueryRef{}
|
||||
}
|
||||
|
||||
// GetRight returns the right query reference of a join.
|
||||
func (q *QueryEnvelope) GetRight() QueryRef {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
return spec.Right
|
||||
}
|
||||
return QueryRef{}
|
||||
}
|
||||
|
||||
// GetJoinType returns the join type.
|
||||
func (q *QueryEnvelope) GetJoinType() JoinType {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
return spec.Type
|
||||
}
|
||||
return JoinType{}
|
||||
}
|
||||
|
||||
// GetOn returns the join ON condition.
|
||||
func (q *QueryEnvelope) GetOn() string {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
return spec.On
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetQueryName returns the name of the spec.
|
||||
func (q *QueryEnvelope) GetQueryName() string {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Name
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Name
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Name
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Name
|
||||
case QueryBuilderFormula:
|
||||
return spec.Name
|
||||
case QueryBuilderJoin:
|
||||
return spec.Name
|
||||
case PromQuery:
|
||||
return spec.Name
|
||||
case ClickHouseQuery:
|
||||
return spec.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// IsDisabled returns whether the spec is disabled.
|
||||
func (q *QueryEnvelope) IsDisabled() bool {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Disabled
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Disabled
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Disabled
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Disabled
|
||||
case QueryBuilderFormula:
|
||||
return spec.Disabled
|
||||
case QueryBuilderJoin:
|
||||
return spec.Disabled
|
||||
case PromQuery:
|
||||
return spec.Disabled
|
||||
case ClickHouseQuery:
|
||||
return spec.Disabled
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetLimit returns the row limit.
|
||||
func (q *QueryEnvelope) GetLimit() int {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Limit
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Limit
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Limit
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Limit
|
||||
case QueryBuilderFormula:
|
||||
return spec.Limit
|
||||
case QueryBuilderJoin:
|
||||
return spec.Limit
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// GetOffset returns the row offset.
|
||||
func (q *QueryEnvelope) GetOffset() int {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Offset
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Offset
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Offset
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Offset
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// GetType returns the QueryType of the envelope.
|
||||
func (q *QueryEnvelope) GetType() QueryType {
|
||||
return q.Type
|
||||
}
|
||||
|
||||
// GetOrder returns the order-by clauses.
|
||||
func (q *QueryEnvelope) GetOrder() []OrderBy {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Order
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Order
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Order
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Order
|
||||
case QueryBuilderFormula:
|
||||
return spec.Order
|
||||
case QueryBuilderJoin:
|
||||
return spec.Order
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetGroupBy returns the group-by keys.
|
||||
func (q *QueryEnvelope) GetGroupBy() []GroupByKey {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.GroupBy
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.GroupBy
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.GroupBy
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.GroupBy
|
||||
case QueryBuilderJoin:
|
||||
return spec.GroupBy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFilter returns the filter.
|
||||
func (q *QueryEnvelope) GetFilter() *Filter {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Filter
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Filter
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Filter
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Filter
|
||||
case QueryBuilderJoin:
|
||||
return spec.Filter
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetHaving returns the having clause.
|
||||
func (q *QueryEnvelope) GetHaving() *Having {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Having
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Having
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Having
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Having
|
||||
case QueryBuilderFormula:
|
||||
return spec.Having
|
||||
case QueryBuilderJoin:
|
||||
return spec.Having
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFunctions returns the post-processing functions.
|
||||
func (q *QueryEnvelope) GetFunctions() []Function {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Functions
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Functions
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Functions
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Functions
|
||||
case QueryBuilderFormula:
|
||||
return spec.Functions
|
||||
case QueryBuilderJoin:
|
||||
return spec.Functions
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSelectFields returns the selected fields.
|
||||
func (q *QueryEnvelope) GetSelectFields() []telemetrytypes.TelemetryFieldKey {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.SelectFields
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.SelectFields
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.SelectFields
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.SelectFields
|
||||
case QueryBuilderJoin:
|
||||
return spec.SelectFields
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetLegend returns the legend label.
|
||||
func (q *QueryEnvelope) GetLegend() string {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Legend
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Legend
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Legend
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Legend
|
||||
case QueryBuilderFormula:
|
||||
return spec.Legend
|
||||
case PromQuery:
|
||||
return spec.Legend
|
||||
case ClickHouseQuery:
|
||||
return spec.Legend
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetCursor returns the pagination cursor.
|
||||
func (q *QueryEnvelope) GetCursor() string {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Cursor
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Cursor
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Cursor
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Cursor
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetStepInterval returns the step interval.
|
||||
func (q *QueryEnvelope) GetStepInterval() Step {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.StepInterval
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.StepInterval
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.StepInterval
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.StepInterval
|
||||
case PromQuery:
|
||||
return spec.Step
|
||||
}
|
||||
return Step{}
|
||||
}
|
||||
|
||||
// GetSecondaryAggregations returns the secondary aggregations.
|
||||
func (q *QueryEnvelope) GetSecondaryAggregations() []SecondaryAggregation {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.SecondaryAggregations
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.SecondaryAggregations
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.SecondaryAggregations
|
||||
case QueryBuilderJoin:
|
||||
return spec.SecondaryAggregations
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetLimitBy returns the limit-by configuration.
|
||||
func (q *QueryEnvelope) GetLimitBy() *LimitBy {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.LimitBy
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.LimitBy
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.LimitBy
|
||||
}
|
||||
return nil
|
||||
}
|
||||
452
pkg/types/querybuildertypes/querybuildertypesv5/req_setters.go
Normal file
452
pkg/types/querybuildertypes/querybuildertypesv5/req_setters.go
Normal file
@@ -0,0 +1,452 @@
|
||||
package querybuildertypesv5
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
// SetExpression sets the expression string of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetExpression(expression string) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Expression = expression
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Expression = expression
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetReturnSpansFrom sets the return-spans-from value, if applicable.
|
||||
func (q *QueryEnvelope) SetReturnSpansFrom(returnSpansFrom string) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.ReturnSpansFrom = returnSpansFrom
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetSignal sets the signal of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetSignal(signal telemetrytypes.Signal) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Signal = signal
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Signal = signal
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Signal = signal
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetSource sets the source of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetSource(source telemetrytypes.Source) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Source = source
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Source = source
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Source = source
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetQuery sets the raw query string of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetQuery(query string) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case PromQuery:
|
||||
spec.Query = query
|
||||
q.Spec = spec
|
||||
case ClickHouseQuery:
|
||||
spec.Query = query
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetStats sets the PromQL stats flag, if applicable.
|
||||
func (q *QueryEnvelope) SetStats(stats bool) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case PromQuery:
|
||||
spec.Stats = stats
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetLeft sets the left query reference of a join, if applicable.
|
||||
func (q *QueryEnvelope) SetLeft(left QueryRef) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
spec.Left = left
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetRight sets the right query reference of a join, if applicable.
|
||||
func (q *QueryEnvelope) SetRight(right QueryRef) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
spec.Right = right
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetJoinType sets the join type, if applicable.
|
||||
func (q *QueryEnvelope) SetJoinType(joinType JoinType) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
spec.Type = joinType
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetOn sets the join ON condition, if applicable.
|
||||
func (q *QueryEnvelope) SetOn(on string) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderJoin:
|
||||
spec.On = on
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetQueryName sets the name of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetQueryName(name string) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
case PromQuery:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
case ClickHouseQuery:
|
||||
spec.Name = name
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetDisabled sets the disabled flag of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetDisabled(disabled bool) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
case PromQuery:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
case ClickHouseQuery:
|
||||
spec.Disabled = disabled
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetLimit sets the row limit of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetLimit(limit int) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Limit = limit
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Limit = limit
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Limit = limit
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Limit = limit
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Limit = limit
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.Limit = limit
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetOffset sets the row offset of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetOffset(offset int) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Offset = offset
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Offset = offset
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Offset = offset
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Offset = offset
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetType sets the QueryType of the envelope.
|
||||
func (q *QueryEnvelope) SetType(t QueryType) {
|
||||
q.Type = t
|
||||
}
|
||||
|
||||
// SetOrder sets the order-by clauses of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetOrder(order []OrderBy) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Order = order
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Order = order
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Order = order
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Order = order
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Order = order
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.Order = order
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetGroupBy sets the group-by keys of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetGroupBy(groupBy []GroupByKey) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.GroupBy = groupBy
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.GroupBy = groupBy
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.GroupBy = groupBy
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.GroupBy = groupBy
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.GroupBy = groupBy
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetFilter sets the filter of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetFilter(filter *Filter) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Filter = filter
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Filter = filter
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Filter = filter
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Filter = filter
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.Filter = filter
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetHaving sets the having clause of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetHaving(having *Having) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Having = having
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Having = having
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Having = having
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Having = having
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Having = having
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.Having = having
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetFunctions sets the post-processing functions of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetFunctions(functions []Function) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Functions = functions
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Functions = functions
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Functions = functions
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Functions = functions
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Functions = functions
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.Functions = functions
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetSelectFields sets the selected fields of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetSelectFields(fields []telemetrytypes.TelemetryFieldKey) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.SelectFields = fields
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.SelectFields = fields
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.SelectFields = fields
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.SelectFields = fields
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.SelectFields = fields
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetLegend sets the legend label of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetLegend(legend string) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Legend = legend
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Legend = legend
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Legend = legend
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Legend = legend
|
||||
q.Spec = spec
|
||||
case QueryBuilderFormula:
|
||||
spec.Legend = legend
|
||||
q.Spec = spec
|
||||
case PromQuery:
|
||||
spec.Legend = legend
|
||||
q.Spec = spec
|
||||
case ClickHouseQuery:
|
||||
spec.Legend = legend
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetCursor sets the pagination cursor of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetCursor(cursor string) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.Cursor = cursor
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.Cursor = cursor
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.Cursor = cursor
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.Cursor = cursor
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetStepInterval sets the step interval of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetStepInterval(step Step) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderTraceOperator:
|
||||
spec.StepInterval = step
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.StepInterval = step
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.StepInterval = step
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.StepInterval = step
|
||||
q.Spec = spec
|
||||
case PromQuery:
|
||||
spec.Step = step
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetSecondaryAggregations sets the secondary aggregations of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetSecondaryAggregations(secondaryAggregations []SecondaryAggregation) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.SecondaryAggregations = secondaryAggregations
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.SecondaryAggregations = secondaryAggregations
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.SecondaryAggregations = secondaryAggregations
|
||||
q.Spec = spec
|
||||
case QueryBuilderJoin:
|
||||
spec.SecondaryAggregations = secondaryAggregations
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
|
||||
// SetLimitBy sets the limit-by configuration of the spec, if applicable.
|
||||
func (q *QueryEnvelope) SetLimitBy(limitBy *LimitBy) {
|
||||
switch spec := q.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
spec.LimitBy = limitBy
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
spec.LimitBy = limitBy
|
||||
q.Spec = spec
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
spec.LimitBy = limitBy
|
||||
q.Spec = spec
|
||||
}
|
||||
}
|
||||
@@ -10,55 +10,9 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// queryName returns the name from any query envelope spec type.
|
||||
func (e QueryEnvelope) queryName() string {
|
||||
switch spec := e.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Name
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Name
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Name
|
||||
case QueryBuilderFormula:
|
||||
return spec.Name
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Name
|
||||
case QueryBuilderJoin:
|
||||
return spec.Name
|
||||
case PromQuery:
|
||||
return spec.Name
|
||||
case ClickHouseQuery:
|
||||
return spec.Name
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// isDisabled returns the disabled status from any query envelope spec type.
|
||||
func (e QueryEnvelope) isDisabled() bool {
|
||||
switch spec := e.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Disabled
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Disabled
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Disabled
|
||||
case QueryBuilderFormula:
|
||||
return spec.Disabled
|
||||
case QueryBuilderTraceOperator:
|
||||
return spec.Disabled
|
||||
case QueryBuilderJoin:
|
||||
return spec.Disabled
|
||||
case PromQuery:
|
||||
return spec.Disabled
|
||||
case ClickHouseQuery:
|
||||
return spec.Disabled
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// getQueryIdentifier returns a friendly identifier for a query based on its type and name/content
|
||||
func getQueryIdentifier(envelope QueryEnvelope, index int) string {
|
||||
name := envelope.queryName()
|
||||
name := envelope.GetQueryName()
|
||||
|
||||
var typeLabel string
|
||||
switch envelope.Type {
|
||||
@@ -89,50 +43,115 @@ const (
|
||||
MaxQueryLimit = 10000
|
||||
)
|
||||
|
||||
// Validate performs preliminary validation on QueryBuilderQuery
|
||||
func (q *QueryBuilderQuery[T]) Validate(requestType RequestType) error {
|
||||
// Validate signal
|
||||
// ValidationOption is a functional option for configuring validation behaviour.
|
||||
type ValidationOption func(*validationConfig)
|
||||
|
||||
type validationConfig struct {
|
||||
skipLimitValidation bool
|
||||
skipAggregationValidation bool
|
||||
skipHavingValidation bool
|
||||
skipAggregationOrderBy bool
|
||||
skipSelectFieldValidation bool
|
||||
skipGroupByValidation bool
|
||||
}
|
||||
|
||||
func applyValidationOptions(opts []ValidationOption) validationConfig {
|
||||
cfg := validationConfig{}
|
||||
for _, opt := range opts {
|
||||
opt(&cfg)
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SkipLimitValidation returns a ValidationOption that skips the limit range check.
|
||||
// Use this when the caller has already validated limits with different constraints.
|
||||
func WithSkipLimitValidation() ValidationOption {
|
||||
return func(cfg *validationConfig) {
|
||||
cfg.skipLimitValidation = true
|
||||
}
|
||||
}
|
||||
|
||||
// SkipAggregationValidation skips aggregation validation.
|
||||
// Used for raw/trace request types where aggregations are not required.
|
||||
func WithSkipAggregationValidation() ValidationOption {
|
||||
return func(cfg *validationConfig) {
|
||||
cfg.skipAggregationValidation = true
|
||||
}
|
||||
}
|
||||
|
||||
// SkipHavingValidation skips having-clause validation.
|
||||
// Used for raw/trace request types where having clauses do not apply.
|
||||
func WithSkipHavingValidation() ValidationOption {
|
||||
return func(cfg *validationConfig) {
|
||||
cfg.skipHavingValidation = true
|
||||
}
|
||||
}
|
||||
|
||||
// SkipAggregationOrderBy skips the aggregation-specific order-by key validation.
|
||||
// Used for raw/trace request types where order-by keys are not restricted to group-by or aggregation keys.
|
||||
func WithSkipAggregationOrderBy() ValidationOption {
|
||||
return func(cfg *validationConfig) {
|
||||
cfg.skipAggregationOrderBy = true
|
||||
}
|
||||
}
|
||||
|
||||
// SkipSelectFieldValidation skips select-field validation.
|
||||
// Used for aggregation request types where select fields do not apply.
|
||||
func WithSkipSelectFieldValidation() ValidationOption {
|
||||
return func(cfg *validationConfig) {
|
||||
cfg.skipSelectFieldValidation = true
|
||||
}
|
||||
}
|
||||
|
||||
// SkipGroupByValidation skips group-by validation.
|
||||
// Used for raw/trace request types where group-by does not apply.
|
||||
func WithSkipGroupByValidation() ValidationOption {
|
||||
return func(cfg *validationConfig) {
|
||||
cfg.skipGroupByValidation = true
|
||||
}
|
||||
}
|
||||
|
||||
// Validate performs preliminary validation on QueryBuilderQuery.
|
||||
func (q *QueryBuilderQuery[T]) Validate(opts ...ValidationOption) error {
|
||||
cfg := applyValidationOptions(opts)
|
||||
|
||||
if err := q.validateSignal(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.validateAggregations(requestType); err != nil {
|
||||
if err := q.validateAggregations(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.validateGroupBy(requestType); err != nil {
|
||||
if err := q.validateGroupBy(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate limit and pagination
|
||||
if err := q.validateLimitAndPagination(); err != nil {
|
||||
if err := q.validateLimitAndPagination(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate functions
|
||||
if err := q.validateFunctions(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Validate secondary aggregations
|
||||
if err := q.validateSecondaryAggregations(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.validateOrderBy(requestType); err != nil {
|
||||
if err := q.validateOrderBy(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.validateSelectFields(requestType); err != nil {
|
||||
if err := q.validateSelectFields(cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *QueryBuilderQuery[T]) validateSelectFields(requestType RequestType) error {
|
||||
// selectFields don't apply to aggregation queries, skip validation
|
||||
if requestType.IsAggregation() {
|
||||
func (q *QueryBuilderQuery[T]) validateSelectFields(cfg validationConfig) error {
|
||||
if cfg.skipSelectFieldValidation {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -148,9 +167,8 @@ func (q *QueryBuilderQuery[T]) validateSelectFields(requestType RequestType) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *QueryBuilderQuery[T]) validateGroupBy(requestType RequestType) error {
|
||||
// groupBy doesn't apply to non-aggregation queries, skip validation
|
||||
if !requestType.IsAggregation() {
|
||||
func (q *QueryBuilderQuery[T]) validateGroupBy(cfg validationConfig) error {
|
||||
if cfg.skipGroupByValidation {
|
||||
return nil
|
||||
}
|
||||
for idx, item := range q.GroupBy {
|
||||
@@ -183,9 +201,8 @@ func (q *QueryBuilderQuery[T]) validateSignal() error {
|
||||
}
|
||||
}
|
||||
|
||||
func (q *QueryBuilderQuery[T]) validateAggregations(requestType RequestType) error {
|
||||
// aggregations don't apply to non-aggregation queries, skip validation
|
||||
if !requestType.IsAggregation() {
|
||||
func (q *QueryBuilderQuery[T]) validateAggregations(cfg validationConfig) error {
|
||||
if cfg.skipAggregationValidation {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -265,24 +282,25 @@ func (q *QueryBuilderQuery[T]) validateAggregations(requestType RequestType) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *QueryBuilderQuery[T]) validateLimitAndPagination() error {
|
||||
// Validate limit
|
||||
if q.Limit < 0 {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"limit must be non-negative, got %d",
|
||||
q.Limit,
|
||||
)
|
||||
}
|
||||
func (q *QueryBuilderQuery[T]) validateLimitAndPagination(cfg validationConfig) error {
|
||||
if !cfg.skipLimitValidation {
|
||||
if q.Limit < 0 {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"limit must be non-negative, got %d",
|
||||
q.Limit,
|
||||
)
|
||||
}
|
||||
|
||||
if q.Limit > MaxQueryLimit {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"limit exceeds maximum allowed value of %d",
|
||||
MaxQueryLimit,
|
||||
).WithAdditional(
|
||||
fmt.Sprintf("Provided limit: %d", q.Limit),
|
||||
)
|
||||
if q.Limit > MaxQueryLimit {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
"limit exceeds maximum allowed value of %d",
|
||||
MaxQueryLimit,
|
||||
).WithAdditional(
|
||||
fmt.Sprintf("Provided limit: %d", q.Limit),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate offset
|
||||
@@ -329,7 +347,7 @@ func (q *QueryBuilderQuery[T]) validateSecondaryAggregations() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q *QueryBuilderQuery[T]) validateOrderBy(requestType RequestType) error {
|
||||
func (q *QueryBuilderQuery[T]) validateOrderBy(cfg validationConfig) error {
|
||||
for i, order := range q.Order {
|
||||
// Direction validation is handled by the OrderDirection type
|
||||
if order.Direction != OrderDirectionAsc && order.Direction != OrderDirectionDesc {
|
||||
@@ -348,8 +366,7 @@ func (q *QueryBuilderQuery[T]) validateOrderBy(requestType RequestType) error {
|
||||
}
|
||||
}
|
||||
|
||||
// aggregation-specific order key validation only applies to aggregation queries
|
||||
if requestType.IsAggregation() {
|
||||
if !cfg.skipAggregationOrderBy {
|
||||
return q.validateOrderByForAggregation()
|
||||
}
|
||||
|
||||
@@ -431,8 +448,8 @@ func (q *QueryBuilderQuery[T]) validateOrderByForAggregation() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateQueryRangeRequest validates the entire query range request
|
||||
func (r *QueryRangeRequest) Validate() error {
|
||||
// Validate validates the entire query range request.
|
||||
func (r *QueryRangeRequest) Validate(opts ...ValidationOption) error {
|
||||
// Validate time range
|
||||
if r.RequestType != RequestTypeRawStream && r.Start >= r.End {
|
||||
return errors.NewInvalidInputf(
|
||||
@@ -443,8 +460,10 @@ func (r *QueryRangeRequest) Validate() error {
|
||||
|
||||
// Validate request type
|
||||
switch r.RequestType {
|
||||
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTimeSeries, RequestTypeScalar, RequestTypeTrace:
|
||||
// Valid request types
|
||||
case RequestTypeRaw, RequestTypeRawStream, RequestTypeTrace:
|
||||
opts = append(opts, getValidationOptions(false)...)
|
||||
case RequestTypeTimeSeries, RequestTypeScalar:
|
||||
opts = append(opts, getValidationOptions(true)...)
|
||||
default:
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
@@ -456,7 +475,7 @@ func (r *QueryRangeRequest) Validate() error {
|
||||
}
|
||||
|
||||
// Validate composite query
|
||||
if err := r.validateCompositeQuery(); err != nil {
|
||||
if err := r.CompositeQuery.Validate(opts...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -471,7 +490,7 @@ func (r *QueryRangeRequest) Validate() error {
|
||||
// validateAllQueriesNotDisabled validates that at least one query in the composite query is enabled
|
||||
func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error {
|
||||
for _, envelope := range r.CompositeQuery.Queries {
|
||||
if !envelope.isDisabled() {
|
||||
if !envelope.IsDisabled() {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -482,12 +501,8 @@ func (r *QueryRangeRequest) validateAllQueriesNotDisabled() error {
|
||||
)
|
||||
}
|
||||
|
||||
func (r *QueryRangeRequest) validateCompositeQuery() error {
|
||||
return r.CompositeQuery.Validate(r.RequestType)
|
||||
}
|
||||
|
||||
// Validate performs validation on CompositeQuery
|
||||
func (c *CompositeQuery) Validate(requestType RequestType) error {
|
||||
func (c *CompositeQuery) Validate(opts ...ValidationOption) error {
|
||||
if len(c.Queries) == 0 {
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
@@ -499,14 +514,14 @@ func (c *CompositeQuery) Validate(requestType RequestType) error {
|
||||
queryNames := make(map[string]bool)
|
||||
|
||||
for i, envelope := range c.Queries {
|
||||
if err := validateQueryEnvelope(envelope, requestType); err != nil {
|
||||
if err := validateQueryEnvelope(envelope, opts...); err != nil {
|
||||
queryId := getQueryIdentifier(envelope, i)
|
||||
return wrapValidationError(err, queryId, "invalid %s: %s")
|
||||
}
|
||||
|
||||
// Check name uniqueness for builder queries
|
||||
if envelope.Type == QueryTypeBuilder || envelope.Type == QueryTypeSubQuery {
|
||||
name := envelope.queryName()
|
||||
name := envelope.GetQueryName()
|
||||
if name != "" {
|
||||
if queryNames[name] {
|
||||
return errors.NewInvalidInputf(
|
||||
@@ -523,16 +538,16 @@ func (c *CompositeQuery) Validate(requestType RequestType) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateQueryEnvelope(envelope QueryEnvelope, requestType RequestType) error {
|
||||
func validateQueryEnvelope(envelope QueryEnvelope, opts ...ValidationOption) error {
|
||||
switch envelope.Type {
|
||||
case QueryTypeBuilder, QueryTypeSubQuery:
|
||||
switch spec := envelope.Spec.(type) {
|
||||
case QueryBuilderQuery[TraceAggregation]:
|
||||
return spec.Validate(requestType)
|
||||
return spec.Validate(opts...)
|
||||
case QueryBuilderQuery[LogAggregation]:
|
||||
return spec.Validate(requestType)
|
||||
return spec.Validate(opts...)
|
||||
case QueryBuilderQuery[MetricAggregation]:
|
||||
return spec.Validate(requestType)
|
||||
return spec.Validate(opts...)
|
||||
default:
|
||||
return errors.NewInvalidInputf(
|
||||
errors.CodeInvalidInput,
|
||||
@@ -618,3 +633,11 @@ func validateQueryEnvelope(envelope QueryEnvelope, requestType RequestType) erro
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func getValidationOptions(isAggregationQuery bool) []ValidationOption {
|
||||
if isAggregationQuery {
|
||||
return []ValidationOption{WithSkipSelectFieldValidation()}
|
||||
}
|
||||
return []ValidationOption{WithSkipAggregationValidation(), WithSkipHavingValidation(), WithSkipAggregationOrderBy(), WithSkipGroupByValidation()}
|
||||
|
||||
}
|
||||
|
||||
@@ -743,7 +743,7 @@ func TestValidateQueryEnvelope(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateQueryEnvelope(tt.envelope, tt.requestType)
|
||||
err := validateQueryEnvelope(tt.envelope)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("validateQueryEnvelope() expected error but got none")
|
||||
@@ -816,7 +816,7 @@ func TestQueryEnvelope_Helpers(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.envelope.queryName()
|
||||
got := tt.envelope.GetQueryName()
|
||||
if got != tt.want {
|
||||
t.Errorf("queryName() = %q, want %q", got, tt.want)
|
||||
}
|
||||
@@ -868,7 +868,7 @@ func TestQueryEnvelope_Helpers(t *testing.T) {
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := tt.envelope.isDisabled()
|
||||
got := tt.envelope.IsDisabled()
|
||||
if got != tt.want {
|
||||
t.Errorf("isDisabled() = %v, want %v", got, tt.want)
|
||||
}
|
||||
@@ -1107,7 +1107,7 @@ func TestQueryRangeRequest_ValidateOrderByForAggregation(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := tt.query.Validate(RequestTypeTimeSeries)
|
||||
err := tt.query.Validate(getValidationOptions(true)...)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Errorf("validateOrderByForAggregation() expected error but got none")
|
||||
@@ -1161,7 +1161,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
|
||||
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "service.name"}},
|
||||
},
|
||||
}
|
||||
err := query.Validate(RequestTypeRaw)
|
||||
err := query.Validate(getValidationOptions(false)...)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error for groupBy with raw request type, got: %v", err)
|
||||
}
|
||||
@@ -1178,7 +1178,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
|
||||
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: ""}},
|
||||
},
|
||||
}
|
||||
err := query.Validate(RequestTypeTimeSeries)
|
||||
err := query.Validate(getValidationOptions(true)...)
|
||||
if err == nil {
|
||||
t.Errorf("expected error for empty groupBy key with timeseries request type")
|
||||
}
|
||||
@@ -1190,7 +1190,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
Having: &Having{Expression: "count() > 10"},
|
||||
}
|
||||
err := query.Validate(RequestTypeRaw)
|
||||
err := query.Validate(getValidationOptions(false)...)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error for having with raw request type, got: %v", err)
|
||||
}
|
||||
@@ -1202,7 +1202,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Having: &Having{Expression: "count() > 10"},
|
||||
}
|
||||
err := query.Validate(RequestTypeTrace)
|
||||
err := query.Validate(getValidationOptions(false)...)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error for having with trace request type, got: %v", err)
|
||||
}
|
||||
@@ -1216,7 +1216,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
|
||||
{Expression: "count()"},
|
||||
},
|
||||
}
|
||||
err := query.Validate(RequestTypeRaw)
|
||||
err := query.Validate(getValidationOptions(false)...)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error for aggregations with raw request type, got: %v", err)
|
||||
}
|
||||
@@ -1230,7 +1230,7 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
|
||||
{Expression: "count()"},
|
||||
},
|
||||
}
|
||||
err := query.Validate(RequestTypeRawStream)
|
||||
err := query.Validate(getValidationOptions(false)...)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error for aggregations with raw_stream request type, got: %v", err)
|
||||
}
|
||||
@@ -1248,12 +1248,12 @@ func TestNonAggregationFieldsSkipped(t *testing.T) {
|
||||
},
|
||||
}
|
||||
// Should error for raw (selectFields are validated)
|
||||
err := query.Validate(RequestTypeRaw)
|
||||
err := query.Validate(getValidationOptions(false)...)
|
||||
if err == nil {
|
||||
t.Errorf("expected error for isRoot in selectFields with raw request type")
|
||||
}
|
||||
// Should pass for timeseries (selectFields skipped)
|
||||
err = query.Validate(RequestTypeTimeSeries)
|
||||
err = query.Validate(getValidationOptions(true)...)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error for isRoot in selectFields with timeseries request type, got: %v", err)
|
||||
}
|
||||
|
||||
@@ -1,18 +1,17 @@
|
||||
package savedviewtypes
|
||||
package types
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/uptrace/bun"
|
||||
)
|
||||
|
||||
type SavedView struct {
|
||||
bun.BaseModel `bun:"table:saved_views"`
|
||||
|
||||
types.Identifiable
|
||||
types.TimeAuditable
|
||||
types.UserAuditable
|
||||
Identifiable
|
||||
TimeAuditable
|
||||
UserAuditable
|
||||
OrgID string `json:"orgId" bun:"org_id,notnull"`
|
||||
Name string `json:"name" bun:"name,type:text,notnull"`
|
||||
Category string `json:"category" bun:"category,type:text,notnull"`
|
||||
634
tests/integration/src/rawexportdata/01_logs.py
Normal file
634
tests/integration/src/rawexportdata/01_logs.py
Normal file
@@ -0,0 +1,634 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from http import HTTPStatus
|
||||
from typing import Callable, List
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
|
||||
|
||||
def test_export_logs_csv(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 3 logs with different severity levels and attributes.
|
||||
|
||||
Tests:
|
||||
1. Export logs as CSV format
|
||||
2. Verify CSV structure and content
|
||||
3. Validate headers are present
|
||||
4. Check log data is correctly formatted
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
body="Application started successfully",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "api-service",
|
||||
"deployment.environment": "production",
|
||||
"host.name": "server-01",
|
||||
},
|
||||
attributes={
|
||||
"http.method": "GET",
|
||||
"http.status_code": 200,
|
||||
"user.id": "user123",
|
||||
},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=8),
|
||||
body="Connection to database failed",
|
||||
severity_text="ERROR",
|
||||
resources={
|
||||
"service.name": "api-service",
|
||||
"deployment.environment": "production",
|
||||
"host.name": "server-01",
|
||||
},
|
||||
attributes={
|
||||
"error.type": "ConnectionError",
|
||||
"db.name": "production_db",
|
||||
},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
body="Request processed",
|
||||
severity_text="DEBUG",
|
||||
resources={
|
||||
"service.name": "worker-service",
|
||||
"deployment.environment": "production",
|
||||
"host.name": "server-02",
|
||||
},
|
||||
attributes={
|
||||
"request.id": "req-456",
|
||||
"duration_ms": 150.5,
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
}
|
||||
|
||||
# Export logs as CSV (default format, no source needed)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=30,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "text/csv"
|
||||
assert "attachment" in response.headers.get("Content-Disposition", "")
|
||||
|
||||
# Parse CSV content
|
||||
csv_content = response.text
|
||||
csv_reader = csv.DictReader(io.StringIO(csv_content))
|
||||
|
||||
rows = list(csv_reader)
|
||||
assert len(rows) == 3, f"Expected 3 rows, got {len(rows)}"
|
||||
|
||||
# Verify log bodies are present in the exported data
|
||||
bodies = [row.get("body") for row in rows]
|
||||
assert "Application started successfully" in bodies
|
||||
assert "Connection to database failed" in bodies
|
||||
assert "Request processed" in bodies
|
||||
|
||||
# Verify severity levels
|
||||
severities = [row.get("severity_text") for row in rows]
|
||||
assert "INFO" in severities
|
||||
assert "ERROR" in severities
|
||||
assert "DEBUG" in severities
|
||||
|
||||
|
||||
def test_export_logs_jsonl(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 2 logs with different attributes.
|
||||
|
||||
Tests:
|
||||
1. Export logs as JSONL format
|
||||
2. Verify JSONL structure and content
|
||||
3. Check each line is valid JSON
|
||||
4. Validate log data is correctly formatted
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
body="User logged in",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "auth-service",
|
||||
"deployment.environment": "staging",
|
||||
},
|
||||
attributes={
|
||||
"user.email": "test@example.com",
|
||||
"session.id": "sess-789",
|
||||
},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
body="Payment processed successfully",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "payment-service",
|
||||
"deployment.environment": "staging",
|
||||
},
|
||||
attributes={
|
||||
"transaction.id": "txn-123",
|
||||
"amount": 99.99,
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "jsonl",
|
||||
"source": "logs",
|
||||
}
|
||||
|
||||
# Export logs as JSONL
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
assert "attachment" in response.headers.get("Content-Disposition", "")
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert len(jsonl_lines) == 2, f"Expected 2 lines, got {len(jsonl_lines)}"
|
||||
|
||||
# Verify each line is valid JSON
|
||||
json_objects = []
|
||||
for line in jsonl_lines:
|
||||
obj = json.loads(line)
|
||||
json_objects.append(obj)
|
||||
assert "id" in obj
|
||||
assert "timestamp" in obj
|
||||
assert "body" in obj
|
||||
assert "severity_text" in obj
|
||||
|
||||
# Verify log bodies
|
||||
bodies = [obj.get("body") for obj in json_objects]
|
||||
assert "User logged in" in bodies
|
||||
assert "Payment processed successfully" in bodies
|
||||
|
||||
|
||||
def test_export_logs_with_filter(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert logs with different severity levels.
|
||||
|
||||
Tests:
|
||||
1. Export logs with filter applied
|
||||
2. Verify only filtered logs are returned
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
body="Info message",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=8),
|
||||
body="Error message",
|
||||
severity_text="ERROR",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
body="Another error message",
|
||||
severity_text="ERROR",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "jsonl",
|
||||
"source": "logs",
|
||||
"filter": "severity_text = 'ERROR'",
|
||||
}
|
||||
|
||||
# Export logs with filter
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert len(jsonl_lines) == 2, f"Expected 2 lines (filtered), got {len(jsonl_lines)}"
|
||||
|
||||
# Verify only ERROR logs are returned
|
||||
for line in jsonl_lines:
|
||||
obj = json.loads(line)
|
||||
assert obj["severity_text"] == "ERROR"
|
||||
assert "error message" in obj["body"].lower()
|
||||
|
||||
|
||||
def test_export_logs_with_limit(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 5 logs.
|
||||
|
||||
Tests:
|
||||
1. Export logs with limit applied
|
||||
2. Verify only limited number of logs are returned
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
logs = []
|
||||
for i in range(5):
|
||||
logs.append(
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=i),
|
||||
body=f"Log message {i}",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={
|
||||
"index": i,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
insert_logs(logs)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "csv",
|
||||
"source": "logs",
|
||||
"limit": 3,
|
||||
}
|
||||
|
||||
# Export logs with limit
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "text/csv"
|
||||
|
||||
# Parse CSV content
|
||||
csv_content = response.text
|
||||
csv_reader = csv.DictReader(io.StringIO(csv_content))
|
||||
|
||||
rows = list(csv_reader)
|
||||
assert len(rows) == 3, f"Expected 3 rows (limited), got {len(rows)}"
|
||||
|
||||
|
||||
def test_export_logs_with_columns(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert logs with various attributes.
|
||||
|
||||
Tests:
|
||||
1. Export logs with specific columns
|
||||
2. Verify only specified columns are returned
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
body="Test log message",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
"deployment.environment": "production",
|
||||
},
|
||||
attributes={
|
||||
"http.method": "GET",
|
||||
"http.status_code": 200,
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
# Request only specific columns
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "csv",
|
||||
"source": "logs",
|
||||
"columns": ["timestamp", "severity_text", "body"],
|
||||
}
|
||||
|
||||
# Export logs with specific columns
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params, doseq=True)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "text/csv"
|
||||
|
||||
# Parse CSV content
|
||||
csv_content = response.text
|
||||
csv_reader = csv.DictReader(io.StringIO(csv_content))
|
||||
|
||||
rows = list(csv_reader)
|
||||
assert len(rows) == 1
|
||||
|
||||
# Verify the specified columns are present
|
||||
row = rows[0]
|
||||
assert "timestamp" in row
|
||||
assert "severity_text" in row
|
||||
assert "body" in row
|
||||
assert row["severity_text"] == "INFO"
|
||||
assert row["body"] == "Test log message"
|
||||
|
||||
|
||||
def test_export_logs_with_order_by(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert logs at different timestamps.
|
||||
|
||||
Tests:
|
||||
1. Export logs with ascending timestamp order
|
||||
2. Verify logs are returned in correct order
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
body="First log",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
body="Second log",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
body="Third log",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "jsonl",
|
||||
"source": "logs",
|
||||
"order_by": "timestamp:asc",
|
||||
}
|
||||
|
||||
# Export logs with ascending order
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert len(jsonl_lines) == 3
|
||||
|
||||
# Verify order - first log should be "First log" (oldest)
|
||||
json_objects = [json.loads(line) for line in jsonl_lines]
|
||||
assert json_objects[0]["body"] == "First log"
|
||||
assert json_objects[1]["body"] == "Second log"
|
||||
assert json_objects[2]["body"] == "Third log"
|
||||
|
||||
|
||||
def test_export_logs_with_complex_filter(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[List[Logs]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert logs with various service names and severity levels.
|
||||
|
||||
Tests:
|
||||
1. Export logs with complex filter (multiple conditions)
|
||||
2. Verify only logs matching all conditions are returned
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
body="API error occurred",
|
||||
severity_text="ERROR",
|
||||
resources={
|
||||
"service.name": "api-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=8),
|
||||
body="Worker info message",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "worker-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
Logs(
|
||||
timestamp=now - timedelta(seconds=5),
|
||||
body="API info message",
|
||||
severity_text="INFO",
|
||||
resources={
|
||||
"service.name": "api-service",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
# Filter for api-service AND ERROR severity
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "jsonl",
|
||||
"source": "logs",
|
||||
"filter": "service.name = 'api-service' AND severity_text = 'ERROR'",
|
||||
}
|
||||
|
||||
# Export logs with complex filter
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert (
|
||||
len(jsonl_lines) == 1
|
||||
), f"Expected 1 line (complex filter), got {len(jsonl_lines)}"
|
||||
|
||||
# Verify the filtered log
|
||||
filtered_obj = json.loads(jsonl_lines[0])
|
||||
assert filtered_obj["body"] == "API error occurred"
|
||||
assert filtered_obj["severity_text"] == "ERROR"
|
||||
782
tests/integration/src/rawexportdata/02_traces.py
Normal file
782
tests/integration/src/rawexportdata/02_traces.py
Normal file
@@ -0,0 +1,782 @@
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from http import HTTPStatus
|
||||
from typing import Callable, List
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.traces import TraceIdGenerator, Traces, TracesKind, TracesStatusCode
|
||||
|
||||
|
||||
def test_export_traces_csv(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[List[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 3 traces with different attributes.
|
||||
|
||||
Tests:
|
||||
1. Export traces as CSV format
|
||||
2. Verify CSV structure and content
|
||||
3. Validate headers are present
|
||||
4. Check trace data is correctly formatted
|
||||
"""
|
||||
http_service_trace_id = TraceIdGenerator.trace_id()
|
||||
http_service_span_id = TraceIdGenerator.span_id()
|
||||
http_service_db_span_id = TraceIdGenerator.span_id()
|
||||
topic_service_trace_id = TraceIdGenerator.trace_id()
|
||||
topic_service_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=3),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_span_id,
|
||||
parent_span_id="",
|
||||
name="POST /integration",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
},
|
||||
attributes={
|
||||
"net.transport": "IP.TCP",
|
||||
"http.scheme": "http",
|
||||
"http.user_agent": "Integration Test",
|
||||
"http.request.method": "POST",
|
||||
"http.response.status_code": "200",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=3.5),
|
||||
duration=timedelta(seconds=0.5),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_db_span_id,
|
||||
parent_span_id=http_service_span_id,
|
||||
name="SELECT",
|
||||
kind=TracesKind.SPAN_KIND_CLIENT,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "http-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-000",
|
||||
},
|
||||
attributes={
|
||||
"db.name": "integration",
|
||||
"db.operation": "SELECT",
|
||||
"db.statement": "SELECT * FROM integration",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
duration=timedelta(seconds=2),
|
||||
trace_id=topic_service_trace_id,
|
||||
span_id=topic_service_span_id,
|
||||
parent_span_id="",
|
||||
name="topic publish",
|
||||
kind=TracesKind.SPAN_KIND_PRODUCER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"deployment.environment": "production",
|
||||
"service.name": "topic-service",
|
||||
"os.type": "linux",
|
||||
"host.name": "linux-001",
|
||||
},
|
||||
attributes={
|
||||
"message.type": "SENT",
|
||||
"messaging.operation": "publish",
|
||||
"messaging.message.id": "001",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"source": "traces",
|
||||
"limit": 1000,
|
||||
}
|
||||
|
||||
# Export traces as CSV (GET for simple queries)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=30,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "text/csv"
|
||||
assert "attachment" in response.headers.get("Content-Disposition", "")
|
||||
|
||||
# Parse CSV content
|
||||
csv_content = response.text
|
||||
csv_reader = csv.DictReader(io.StringIO(csv_content))
|
||||
|
||||
rows = list(csv_reader)
|
||||
assert len(rows) == 3, f"Expected 3 rows, got {len(rows)}"
|
||||
|
||||
# Verify trace IDs are present in the exported data
|
||||
trace_ids = [row.get("trace_id") for row in rows]
|
||||
assert http_service_trace_id in trace_ids
|
||||
assert topic_service_trace_id in trace_ids
|
||||
|
||||
# Verify span names are present
|
||||
span_names = [row.get("name") for row in rows]
|
||||
assert "POST /integration" in span_names
|
||||
assert "SELECT" in span_names
|
||||
assert "topic publish" in span_names
|
||||
|
||||
|
||||
def test_export_traces_jsonl(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[List[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 2 traces with different attributes.
|
||||
|
||||
Tests:
|
||||
1. Export traces as JSONL format
|
||||
2. Verify JSONL structure and content
|
||||
3. Check each line is valid JSON
|
||||
4. Validate trace data is correctly formatted
|
||||
"""
|
||||
http_service_trace_id = TraceIdGenerator.trace_id()
|
||||
http_service_span_id = TraceIdGenerator.span_id()
|
||||
topic_service_trace_id = TraceIdGenerator.trace_id()
|
||||
topic_service_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=3),
|
||||
trace_id=http_service_trace_id,
|
||||
span_id=http_service_span_id,
|
||||
parent_span_id="",
|
||||
name="POST /api/test",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "api-service",
|
||||
"deployment.environment": "staging",
|
||||
},
|
||||
attributes={
|
||||
"http.request.method": "POST",
|
||||
"http.response.status_code": "201",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=topic_service_trace_id,
|
||||
span_id=topic_service_span_id,
|
||||
parent_span_id="",
|
||||
name="queue.process",
|
||||
kind=TracesKind.SPAN_KIND_CONSUMER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "queue-service",
|
||||
"deployment.environment": "staging",
|
||||
},
|
||||
attributes={
|
||||
"messaging.operation": "process",
|
||||
"messaging.system": "rabbitmq",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "jsonl",
|
||||
"source": "traces",
|
||||
"limit": 1000,
|
||||
}
|
||||
|
||||
# Export traces as JSONL (GET for simple queries)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
assert "attachment" in response.headers.get("Content-Disposition", "")
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert len(jsonl_lines) == 2, f"Expected 2 lines, got {len(jsonl_lines)}"
|
||||
|
||||
# Verify each line is valid JSON
|
||||
json_objects = []
|
||||
for line in jsonl_lines:
|
||||
obj = json.loads(line)
|
||||
json_objects.append(obj)
|
||||
assert "trace_id" in obj
|
||||
assert "span_id" in obj
|
||||
assert "name" in obj
|
||||
|
||||
# Verify trace IDs are present
|
||||
trace_ids = [obj.get("trace_id") for obj in json_objects]
|
||||
assert http_service_trace_id in trace_ids
|
||||
assert topic_service_trace_id in trace_ids
|
||||
|
||||
# Verify span names are present
|
||||
span_names = [obj.get("name") for obj in json_objects]
|
||||
assert "POST /api/test" in span_names
|
||||
assert "queue.process" in span_names
|
||||
|
||||
|
||||
def test_export_traces_with_filter(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[List[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert traces with different service names.
|
||||
|
||||
Tests:
|
||||
1. Export traces with filter applied
|
||||
2. Verify only filtered traces are returned
|
||||
"""
|
||||
service_a_trace_id = TraceIdGenerator.trace_id()
|
||||
service_a_span_id = TraceIdGenerator.span_id()
|
||||
service_b_trace_id = TraceIdGenerator.trace_id()
|
||||
service_b_span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=4),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=service_a_trace_id,
|
||||
span_id=service_a_span_id,
|
||||
parent_span_id="",
|
||||
name="operation-a",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "service-a",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=service_b_trace_id,
|
||||
span_id=service_b_span_id,
|
||||
parent_span_id="",
|
||||
name="operation-b",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "service-b",
|
||||
},
|
||||
attributes={},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "jsonl",
|
||||
"source": "traces",
|
||||
"limit": 1000,
|
||||
"filter": "service.name = 'service-a'",
|
||||
}
|
||||
|
||||
# Export traces with filter (GET supports filter param)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert len(jsonl_lines) == 1, f"Expected 1 line (filtered), got {len(jsonl_lines)}"
|
||||
|
||||
# Verify the filtered trace
|
||||
filtered_obj = json.loads(jsonl_lines[0])
|
||||
assert filtered_obj["trace_id"] == service_a_trace_id
|
||||
assert filtered_obj["name"] == "operation-a"
|
||||
|
||||
|
||||
def test_export_traces_with_limit(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[List[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert 5 traces.
|
||||
|
||||
Tests:
|
||||
1. Export traces with limit applied
|
||||
2. Verify only limited number of traces are returned
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
traces = []
|
||||
for i in range(5):
|
||||
traces.append(
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=i),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=TraceIdGenerator.trace_id(),
|
||||
span_id=TraceIdGenerator.span_id(),
|
||||
parent_span_id="",
|
||||
name=f"operation-{i}",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
},
|
||||
attributes={},
|
||||
)
|
||||
)
|
||||
|
||||
insert_traces(traces)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
params = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"format": "csv",
|
||||
"source": "traces",
|
||||
"limit": 3,
|
||||
}
|
||||
|
||||
# Export traces with limit (GET supports limit param)
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get(
|
||||
f"/api/v1/export_raw_data?{urlencode(params)}"
|
||||
),
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "text/csv"
|
||||
|
||||
# Parse CSV content
|
||||
csv_content = response.text
|
||||
csv_reader = csv.DictReader(io.StringIO(csv_content))
|
||||
|
||||
rows = list(csv_reader)
|
||||
assert len(rows) == 3, f"Expected 3 rows (limited), got {len(rows)}"
|
||||
|
||||
|
||||
def test_export_traces_multiple_queries_rejected(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
) -> None:
|
||||
"""
|
||||
Tests:
|
||||
1. POST with multiple builder queries but no trace operator is rejected
|
||||
2. Verify 400 error is returned
|
||||
"""
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
body = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"signal": "traces",
|
||||
"name": "A",
|
||||
"limit": 1000,
|
||||
"filter": {"expression": "service.name = 'service-a'"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"signal": "traces",
|
||||
"name": "B",
|
||||
"limit": 1000,
|
||||
"filter": {"expression": "service.name = 'service-b'"},
|
||||
},
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
|
||||
response = requests.post(
|
||||
url,
|
||||
json=body,
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
def test_export_traces_with_composite_query_trace_operator(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[List[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert multiple traces with parent-child relationships.
|
||||
|
||||
Tests:
|
||||
1. Export traces using trace operator in composite query (POST)
|
||||
2. Verify trace operator query works correctly
|
||||
"""
|
||||
parent_trace_id = TraceIdGenerator.trace_id()
|
||||
parent_span_id = TraceIdGenerator.span_id()
|
||||
child_span_id_1 = TraceIdGenerator.span_id()
|
||||
child_span_id_2 = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
duration=timedelta(seconds=5),
|
||||
trace_id=parent_trace_id,
|
||||
span_id=parent_span_id,
|
||||
parent_span_id="",
|
||||
name="parent-operation",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "parent-service",
|
||||
},
|
||||
attributes={
|
||||
"operation.type": "parent",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=9),
|
||||
duration=timedelta(seconds=2),
|
||||
trace_id=parent_trace_id,
|
||||
span_id=child_span_id_1,
|
||||
parent_span_id=parent_span_id,
|
||||
name="child-operation-1",
|
||||
kind=TracesKind.SPAN_KIND_INTERNAL,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "parent-service",
|
||||
},
|
||||
attributes={
|
||||
"operation.type": "child",
|
||||
},
|
||||
),
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=7),
|
||||
duration=timedelta(seconds=1),
|
||||
trace_id=parent_trace_id,
|
||||
span_id=child_span_id_2,
|
||||
parent_span_id=parent_span_id,
|
||||
name="child-operation-2",
|
||||
kind=TracesKind.SPAN_KIND_INTERNAL,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "parent-service",
|
||||
},
|
||||
attributes={
|
||||
"operation.type": "child",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
# A: spans with operation.type = 'parent'
|
||||
query_a = {
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"signal": "traces",
|
||||
"name": "A",
|
||||
"limit": 1000,
|
||||
"filter": {"expression": "operation.type = 'parent'"},
|
||||
},
|
||||
}
|
||||
|
||||
# B: spans with operation.type = 'child'
|
||||
query_b = {
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"signal": "traces",
|
||||
"name": "B",
|
||||
"limit": 1000,
|
||||
"filter": {"expression": "operation.type = 'child'"},
|
||||
},
|
||||
}
|
||||
|
||||
# Trace operator: find traces where A has a direct descendant B
|
||||
query_c = {
|
||||
"type": "builder_trace_operator",
|
||||
"spec": {
|
||||
"name": "C",
|
||||
"expression": "A => B",
|
||||
"returnSpansFrom": "A",
|
||||
"limit": 1000,
|
||||
"order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
|
||||
},
|
||||
}
|
||||
|
||||
body = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [query_a, query_b, query_c],
|
||||
},
|
||||
}
|
||||
|
||||
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
|
||||
response = requests.post(
|
||||
url,
|
||||
json=body,
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
print(response.text)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert len(jsonl_lines) == 1, f"Expected at least 1 line, got {len(jsonl_lines)}"
|
||||
|
||||
# Verify all returned spans belong to the matched trace
|
||||
json_objects = [json.loads(line) for line in jsonl_lines]
|
||||
trace_ids = [obj.get("trace_id") for obj in json_objects]
|
||||
assert all(tid == parent_trace_id for tid in trace_ids)
|
||||
|
||||
# Verify the parent span (returnSpansFrom = "A") is present
|
||||
span_names = [obj.get("name") for obj in json_objects]
|
||||
assert "parent-operation" in span_names
|
||||
|
||||
|
||||
def test_export_traces_with_select_fields(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_traces: Callable[[List[Traces]], None],
|
||||
) -> None:
|
||||
"""
|
||||
Setup:
|
||||
Insert traces with various attributes.
|
||||
|
||||
Tests:
|
||||
1. Export traces with specific select fields via POST
|
||||
2. Verify only specified fields are returned in the output
|
||||
"""
|
||||
trace_id = TraceIdGenerator.trace_id()
|
||||
span_id = TraceIdGenerator.span_id()
|
||||
|
||||
now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
|
||||
|
||||
insert_traces(
|
||||
[
|
||||
Traces(
|
||||
timestamp=now - timedelta(seconds=10),
|
||||
duration=timedelta(seconds=2),
|
||||
trace_id=trace_id,
|
||||
span_id=span_id,
|
||||
parent_span_id="",
|
||||
name="test-operation",
|
||||
kind=TracesKind.SPAN_KIND_SERVER,
|
||||
status_code=TracesStatusCode.STATUS_CODE_OK,
|
||||
status_message="",
|
||||
resources={
|
||||
"service.name": "test-service",
|
||||
"deployment.environment": "production",
|
||||
"host.name": "server-01",
|
||||
},
|
||||
attributes={
|
||||
"http.method": "POST",
|
||||
"http.status_code": "201",
|
||||
"user.id": "user123",
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# Calculate timestamps in nanoseconds
|
||||
start_ns = int((now - timedelta(minutes=5)).timestamp() * 1e9)
|
||||
end_ns = int(now.timestamp() * 1e9)
|
||||
|
||||
body = {
|
||||
"start": start_ns,
|
||||
"end": end_ns,
|
||||
"requestType": "raw",
|
||||
"compositeQuery": {
|
||||
"queries": [
|
||||
{
|
||||
"type": "builder_query",
|
||||
"spec": {
|
||||
"signal": "traces",
|
||||
"name": "A",
|
||||
"limit": 1000,
|
||||
"selectFields": [
|
||||
{
|
||||
"name": "trace_id",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "span_id",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "span",
|
||||
"signal": "traces",
|
||||
},
|
||||
{
|
||||
"name": "service.name",
|
||||
"fieldDataType": "string",
|
||||
"fieldContext": "resource",
|
||||
"signal": "traces",
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
url = signoz.self.host_configs["8080"].get("/api/v1/export_raw_data?format=jsonl")
|
||||
response = requests.post(
|
||||
url,
|
||||
json=body,
|
||||
timeout=10,
|
||||
headers={
|
||||
"authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.headers["Content-Type"] == "application/x-ndjson"
|
||||
|
||||
# Parse JSONL content
|
||||
jsonl_lines = response.text.strip().split("\n")
|
||||
assert len(jsonl_lines) == 1
|
||||
|
||||
# Verify the selected fields are present
|
||||
result = json.loads(jsonl_lines[0])
|
||||
assert "trace_id" in result
|
||||
assert "span_id" in result
|
||||
assert "name" in result
|
||||
|
||||
# Verify values
|
||||
assert result["trace_id"] == trace_id
|
||||
assert result["span_id"] == span_id
|
||||
assert result["name"] == "test-operation"
|
||||
Reference in New Issue
Block a user