mirror of
https://github.com/SigNoz/signoz.git
synced 2026-03-30 09:00:23 +01:00
Compare commits
90 Commits
v0.117.1
...
refactor/c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
66aaaea918 | ||
|
|
900c489d91 | ||
|
|
743fe56523 | ||
|
|
3a9e93ebdf | ||
|
|
e4f0d026c1 | ||
|
|
754dbc7f38 | ||
|
|
07dbf1e69f | ||
|
|
abe5454d11 | ||
|
|
749884ce70 | ||
|
|
70fdc88112 | ||
|
|
234787f2c1 | ||
|
|
e7d0dd8850 | ||
|
|
c23c6c8c27 | ||
|
|
58dabf9a6c | ||
|
|
91edc2a895 | ||
|
|
2e70857554 | ||
|
|
f3e6892d5b | ||
|
|
cdbb78a93d | ||
|
|
c11186f7bf | ||
|
|
51dbb0b5b9 | ||
|
|
23a4960e74 | ||
|
|
5d0c55d682 | ||
|
|
15704e0433 | ||
|
|
2545d7df61 | ||
|
|
3f91821825 | ||
|
|
ee5d182539 | ||
|
|
0bc12f02bc | ||
|
|
e5f00421fe | ||
|
|
539252e10c | ||
|
|
d65f426254 | ||
|
|
6e52f2c8f0 | ||
|
|
d9f8a4ae5a | ||
|
|
eefe3edffd | ||
|
|
2051861a03 | ||
|
|
4b01a40fb9 | ||
|
|
2d8a00bf18 | ||
|
|
f1b26b310f | ||
|
|
2c438b6c32 | ||
|
|
1814c2d13c | ||
|
|
e6cd771f11 | ||
|
|
6b94f87ca0 | ||
|
|
bf315253ae | ||
|
|
668ff7bc39 | ||
|
|
07f2aa52fd | ||
|
|
3416b3ad55 | ||
|
|
d6caa4f2c7 | ||
|
|
f86371566d | ||
|
|
9115803084 | ||
|
|
0c14d8f966 | ||
|
|
7afb461af8 | ||
|
|
a21fbb4ee0 | ||
|
|
0369842f3d | ||
|
|
59cd96562a | ||
|
|
cc4475cab7 | ||
|
|
ac8c648420 | ||
|
|
bede6be4b8 | ||
|
|
dd3d60e6df | ||
|
|
538ab686d2 | ||
|
|
936a325cb9 | ||
|
|
c6cdcd0143 | ||
|
|
cd9211d718 | ||
|
|
0601c28782 | ||
|
|
580610dbfa | ||
|
|
2d2aa02a81 | ||
|
|
dd9723ad13 | ||
|
|
3651469416 | ||
|
|
febce75734 | ||
|
|
e1616f3487 | ||
|
|
4b94287ac7 | ||
|
|
1575c7c54c | ||
|
|
8def3f835b | ||
|
|
11ed15f4c5 | ||
|
|
f47877cca9 | ||
|
|
bb2b9215ba | ||
|
|
3111904223 | ||
|
|
003e2c30d8 | ||
|
|
00fe516d10 | ||
|
|
0305f4f7db | ||
|
|
c60019a6dc | ||
|
|
acde2a37fa | ||
|
|
945241a52a | ||
|
|
e967f80c86 | ||
|
|
a09dc325de | ||
|
|
379b4f7fc4 | ||
|
|
5e536ae077 | ||
|
|
234585e642 | ||
|
|
2cc14f1ad4 | ||
|
|
dc4ed4d239 | ||
|
|
7281c36873 | ||
|
|
40288776e8 |
@@ -18,9 +18,12 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/gateway/noopgateway"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/licensing/nooplicensing"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
@@ -28,6 +31,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
"github.com/SigNoz/signoz/pkg/zeus"
|
||||
"github.com/SigNoz/signoz/pkg/zeus/noopzeus"
|
||||
@@ -90,6 +94,9 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
|
||||
return querier.NewHandler(ps, q, a)
|
||||
},
|
||||
func(_ cloudintegrationtypes.Store, _ zeus.Zeus, _ gateway.Gateway, _ licensing.Licensing, _ user.Getter, _ user.Setter) (cloudintegration.Module, error) {
|
||||
return implcloudintegration.NewModule(), nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
logger.ErrorContext(ctx, "failed to create signoz", errors.Attr(err))
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/SigNoz/signoz/ee/gateway/httpgateway"
|
||||
enterpriselicensing "github.com/SigNoz/signoz/ee/licensing"
|
||||
"github.com/SigNoz/signoz/ee/licensing/httplicensing"
|
||||
"github.com/SigNoz/signoz/ee/modules/cloudintegration/implcloudintegration"
|
||||
"github.com/SigNoz/signoz/ee/modules/dashboard/impldashboard"
|
||||
eequerier "github.com/SigNoz/signoz/ee/querier"
|
||||
enterpriseapp "github.com/SigNoz/signoz/ee/query-service/app"
|
||||
@@ -29,9 +30,11 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/gateway"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
pkgimpldashboard "github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
"github.com/SigNoz/signoz/pkg/signoz"
|
||||
@@ -39,6 +42,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstorehook"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
"github.com/SigNoz/signoz/pkg/zeus"
|
||||
)
|
||||
@@ -131,8 +135,10 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
communityHandler := querier.NewHandler(ps, q, a)
|
||||
return eequerier.NewHandler(ps, q, communityHandler)
|
||||
},
|
||||
func(store cloudintegrationtypes.Store, zeus zeus.Zeus, gateway gateway.Gateway, licensing licensing.Licensing, userGetter user.Getter, userSetter user.Setter) (cloudintegration.Module, error) {
|
||||
return implcloudintegration.NewModule(store, config.Global, zeus, gateway, licensing, userGetter, userSetter)
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
logger.ErrorContext(ctx, "failed to create signoz", errors.Attr(err))
|
||||
return err
|
||||
|
||||
@@ -421,11 +421,11 @@ components:
|
||||
type: object
|
||||
CloudintegrationtypesAWSCollectionStrategy:
|
||||
properties:
|
||||
aws_logs:
|
||||
logs:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesAWSLogsStrategy'
|
||||
aws_metrics:
|
||||
metrics:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesAWSMetricsStrategy'
|
||||
s3_buckets:
|
||||
s3Buckets:
|
||||
additionalProperties:
|
||||
items:
|
||||
type: string
|
||||
@@ -465,12 +465,12 @@ components:
|
||||
type: object
|
||||
CloudintegrationtypesAWSLogsStrategy:
|
||||
properties:
|
||||
cloudwatch_logs_subscriptions:
|
||||
cloudwatchLogsSubscriptions:
|
||||
items:
|
||||
properties:
|
||||
filter_pattern:
|
||||
filterPattern:
|
||||
type: string
|
||||
log_group_name_prefix:
|
||||
logGroupNamePrefix:
|
||||
type: string
|
||||
type: object
|
||||
nullable: true
|
||||
@@ -478,7 +478,7 @@ components:
|
||||
type: object
|
||||
CloudintegrationtypesAWSMetricsStrategy:
|
||||
properties:
|
||||
cloudwatch_metric_stream_filters:
|
||||
cloudwatchMetricStreamFilters:
|
||||
items:
|
||||
properties:
|
||||
MetricNames:
|
||||
@@ -577,6 +577,26 @@ components:
|
||||
nullable: true
|
||||
type: array
|
||||
type: object
|
||||
CloudintegrationtypesCloudIntegrationService:
|
||||
nullable: true
|
||||
properties:
|
||||
cloudIntegrationId:
|
||||
type: string
|
||||
config:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesServiceConfig'
|
||||
createdAt:
|
||||
format: date-time
|
||||
type: string
|
||||
id:
|
||||
type: string
|
||||
type:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesServiceID'
|
||||
updatedAt:
|
||||
format: date-time
|
||||
type: string
|
||||
required:
|
||||
- id
|
||||
type: object
|
||||
CloudintegrationtypesCollectedLogAttribute:
|
||||
properties:
|
||||
name:
|
||||
@@ -710,11 +730,54 @@ components:
|
||||
type: string
|
||||
type: array
|
||||
telemetry:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesAWSCollectionStrategy'
|
||||
$ref: '#/components/schemas/CloudintegrationtypesOldAWSCollectionStrategy'
|
||||
required:
|
||||
- enabled_regions
|
||||
- telemetry
|
||||
type: object
|
||||
CloudintegrationtypesOldAWSCollectionStrategy:
|
||||
properties:
|
||||
aws_logs:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesOldAWSLogsStrategy'
|
||||
aws_metrics:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesOldAWSMetricsStrategy'
|
||||
provider:
|
||||
type: string
|
||||
s3_buckets:
|
||||
additionalProperties:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
type: object
|
||||
type: object
|
||||
CloudintegrationtypesOldAWSLogsStrategy:
|
||||
properties:
|
||||
cloudwatch_logs_subscriptions:
|
||||
items:
|
||||
properties:
|
||||
filter_pattern:
|
||||
type: string
|
||||
log_group_name_prefix:
|
||||
type: string
|
||||
type: object
|
||||
nullable: true
|
||||
type: array
|
||||
type: object
|
||||
CloudintegrationtypesOldAWSMetricsStrategy:
|
||||
properties:
|
||||
cloudwatch_metric_stream_filters:
|
||||
items:
|
||||
properties:
|
||||
MetricNames:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
Namespace:
|
||||
type: string
|
||||
type: object
|
||||
nullable: true
|
||||
type: array
|
||||
type: object
|
||||
CloudintegrationtypesPostableAgentCheckInRequest:
|
||||
properties:
|
||||
account_id:
|
||||
@@ -743,6 +806,8 @@ components:
|
||||
properties:
|
||||
assets:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesAssets'
|
||||
cloudIntegrationService:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesCloudIntegrationService'
|
||||
dataCollected:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesDataCollected'
|
||||
icon:
|
||||
@@ -751,9 +816,7 @@ components:
|
||||
type: string
|
||||
overview:
|
||||
type: string
|
||||
serviceConfig:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesServiceConfig'
|
||||
supported_signals:
|
||||
supportedSignals:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesSupportedSignals'
|
||||
telemetryCollectionStrategy:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesCollectionStrategy'
|
||||
@@ -765,9 +828,10 @@ components:
|
||||
- icon
|
||||
- overview
|
||||
- assets
|
||||
- supported_signals
|
||||
- supportedSignals
|
||||
- dataCollected
|
||||
- telemetryCollectionStrategy
|
||||
- cloudIntegrationService
|
||||
type: object
|
||||
CloudintegrationtypesServiceConfig:
|
||||
properties:
|
||||
@@ -776,6 +840,22 @@ components:
|
||||
required:
|
||||
- aws
|
||||
type: object
|
||||
CloudintegrationtypesServiceID:
|
||||
enum:
|
||||
- alb
|
||||
- api-gateway
|
||||
- dynamodb
|
||||
- ec2
|
||||
- ecs
|
||||
- eks
|
||||
- elasticache
|
||||
- lambda
|
||||
- msk
|
||||
- rds
|
||||
- s3sync
|
||||
- sns
|
||||
- sqs
|
||||
type: string
|
||||
CloudintegrationtypesServiceMetadata:
|
||||
properties:
|
||||
enabled:
|
||||
@@ -3298,6 +3378,61 @@ paths:
|
||||
summary: Update account
|
||||
tags:
|
||||
- cloudintegration
|
||||
/api/v1/cloud_integrations/{cloud_provider}/accounts/{id}/services/{service_id}:
|
||||
put:
|
||||
deprecated: false
|
||||
description: This endpoint updates a service for the specified cloud provider
|
||||
operationId: UpdateService
|
||||
parameters:
|
||||
- in: path
|
||||
name: cloud_provider
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- in: path
|
||||
name: id
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- in: path
|
||||
name: service_id
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesUpdatableService'
|
||||
responses:
|
||||
"204":
|
||||
description: No Content
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- ADMIN
|
||||
- tokenizer:
|
||||
- ADMIN
|
||||
summary: Update service
|
||||
tags:
|
||||
- cloudintegration
|
||||
/api/v1/cloud_integrations/{cloud_provider}/accounts/check_in:
|
||||
post:
|
||||
deprecated: false
|
||||
@@ -3465,55 +3600,6 @@ paths:
|
||||
summary: Get service
|
||||
tags:
|
||||
- cloudintegration
|
||||
put:
|
||||
deprecated: false
|
||||
description: This endpoint updates a service for the specified cloud provider
|
||||
operationId: UpdateService
|
||||
parameters:
|
||||
- in: path
|
||||
name: cloud_provider
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- in: path
|
||||
name: service_id
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
requestBody:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/CloudintegrationtypesUpdatableService'
|
||||
responses:
|
||||
"204":
|
||||
description: No Content
|
||||
"401":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Unauthorized
|
||||
"403":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Forbidden
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
security:
|
||||
- api_key:
|
||||
- ADMIN
|
||||
- tokenizer:
|
||||
- ADMIN
|
||||
summary: Update service
|
||||
tags:
|
||||
- cloudintegration
|
||||
/api/v1/complete/google:
|
||||
get:
|
||||
deprecated: false
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package implcloudprovider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sort"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes/definitions"
|
||||
)
|
||||
|
||||
type awscloudprovider struct {
|
||||
serviceDefinitions definitions.ServiceDefinitionLoader
|
||||
}
|
||||
|
||||
func NewAWSCloudProvider() (cloudintegration.CloudProviderModule, error) {
|
||||
loader, err := definitions.NewServiceDefinitionLoader(cloudintegrationtypes.CloudProviderTypeAWS)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &awscloudprovider{
|
||||
serviceDefinitions: loader,
|
||||
}, nil
|
||||
}
|
||||
func (provider *awscloudprovider) GetConnectionArtifact(ctx context.Context, creds *cloudintegrationtypes.SignozCredentials, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.ConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
|
||||
// TODO: get this from config
|
||||
agentVersion := "v0.0.8"
|
||||
|
||||
baseURL := fmt.Sprintf("https://%s.console.aws.amazon.com/cloudformation/home", req.Aws.DeploymentRegion)
|
||||
u, _ := url.Parse(baseURL)
|
||||
|
||||
q := u.Query()
|
||||
q.Set("region", req.Aws.DeploymentRegion)
|
||||
u.Fragment = "/stacks/quickcreate"
|
||||
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
q = u.Query()
|
||||
q.Set("stackName", "signoz-integration")
|
||||
q.Set("templateURL", fmt.Sprintf("https://signoz-integrations.s3.us-east-1.amazonaws.com/aws-quickcreate-template-%s.json", agentVersion))
|
||||
q.Set("param_SigNozIntegrationAgentVersion", agentVersion)
|
||||
q.Set("param_SigNozApiUrl", creds.SigNozAPIURL)
|
||||
q.Set("param_SigNozApiKey", creds.SigNozAPIKey)
|
||||
q.Set("param_SigNozAccountId", account.ID.StringValue())
|
||||
q.Set("param_IngestionUrl", creds.IngestionURL)
|
||||
q.Set("param_IngestionKey", creds.IngestionKey)
|
||||
|
||||
return &cloudintegrationtypes.ConnectionArtifact{
|
||||
Aws: &cloudintegrationtypes.AWSConnectionArtifact{
|
||||
ConnectionURL: u.String() + "?&" + q.Encode(), // this format is required by AWS
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (provider *awscloudprovider) ListServiceDefinitions() ([]cloudintegrationtypes.ServiceDefinition, error) {
|
||||
return provider.serviceDefinitions.List()
|
||||
}
|
||||
|
||||
func (provider *awscloudprovider) GetServiceDefinition(serviceID cloudintegrationtypes.ServiceID) (*cloudintegrationtypes.ServiceDefinition, error) {
|
||||
return provider.serviceDefinitions.Get(serviceID)
|
||||
}
|
||||
|
||||
func (provider *awscloudprovider) StorableConfigFromServiceConfig(cfg *cloudintegrationtypes.ServiceConfig, supported cloudintegrationtypes.SupportedSignals) (string, error) {
|
||||
if cfg == nil || cfg.AWS == nil {
|
||||
return "", nil
|
||||
}
|
||||
// Strip signal configs the service does not support before storing.
|
||||
if !supported.Logs {
|
||||
cfg.AWS.Logs = nil
|
||||
}
|
||||
if !supported.Metrics {
|
||||
cfg.AWS.Metrics = nil
|
||||
}
|
||||
b, err := json.Marshal(cfg.AWS)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func (provider *awscloudprovider) ServiceConfigFromStorableServiceConfig(config string) (*cloudintegrationtypes.ServiceConfig, error) {
|
||||
if config == "" {
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "service config is empty")
|
||||
}
|
||||
|
||||
var awsCfg cloudintegrationtypes.AWSServiceConfig
|
||||
if err := json.Unmarshal([]byte(config), &awsCfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &cloudintegrationtypes.ServiceConfig{AWS: &awsCfg}, nil
|
||||
}
|
||||
|
||||
func (provider *awscloudprovider) IsServiceEnabled(config *cloudintegrationtypes.ServiceConfig) bool {
|
||||
if config == nil || config.AWS == nil {
|
||||
return false
|
||||
}
|
||||
logsEnabled := config.AWS.Logs != nil && config.AWS.Logs.Enabled
|
||||
metricsEnabled := config.AWS.Metrics != nil && config.AWS.Metrics.Enabled
|
||||
return logsEnabled || metricsEnabled
|
||||
}
|
||||
|
||||
func (provider *awscloudprovider) IsMetricsEnabled(config *cloudintegrationtypes.ServiceConfig) bool {
|
||||
if config == nil || config.AWS == nil {
|
||||
return false
|
||||
}
|
||||
return awsMetricsEnabled(config.AWS)
|
||||
}
|
||||
|
||||
// awsLogsEnabled returns true if the AWS service config has logs explicitly enabled.
|
||||
func awsLogsEnabled(cfg *cloudintegrationtypes.AWSServiceConfig) bool {
|
||||
return cfg.Logs != nil && cfg.Logs.Enabled
|
||||
}
|
||||
|
||||
// awsMetricsEnabled returns true if the AWS service config has metrics explicitly enabled.
|
||||
func awsMetricsEnabled(cfg *cloudintegrationtypes.AWSServiceConfig) bool {
|
||||
return cfg.Metrics != nil && cfg.Metrics.Enabled
|
||||
}
|
||||
|
||||
func (provider *awscloudprovider) BuildIntegrationConfig(
|
||||
account *cloudintegrationtypes.Account,
|
||||
services []*cloudintegrationtypes.StorableCloudIntegrationService,
|
||||
) (*cloudintegrationtypes.ProviderIntegrationConfig, error) {
|
||||
// Sort services for deterministic output
|
||||
sort.Slice(services, func(i, j int) bool {
|
||||
return services[i].Type.StringValue() < services[j].Type.StringValue()
|
||||
})
|
||||
|
||||
compiledMetrics := &cloudintegrationtypes.AWSMetricsStrategy{}
|
||||
compiledLogs := &cloudintegrationtypes.AWSLogsStrategy{}
|
||||
var compiledS3Buckets map[string][]string
|
||||
|
||||
for _, storedSvc := range services {
|
||||
svcCfg, err := provider.ServiceConfigFromStorableServiceConfig(storedSvc.Config)
|
||||
if err != nil || svcCfg == nil || svcCfg.AWS == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
svcDef, err := provider.GetServiceDefinition(storedSvc.Type)
|
||||
if err != nil || svcDef == nil || svcDef.Strategy == nil || svcDef.Strategy.AWS == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
strategy := svcDef.Strategy.AWS
|
||||
|
||||
// S3Sync: logs come directly from configured S3 buckets, not CloudWatch subscriptions
|
||||
if storedSvc.Type == cloudintegrationtypes.AWSServiceS3Sync {
|
||||
if awsLogsEnabled(svcCfg.AWS) && svcCfg.AWS.Logs.S3Buckets != nil {
|
||||
compiledS3Buckets = svcCfg.AWS.Logs.S3Buckets
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if awsLogsEnabled(svcCfg.AWS) && strategy.Logs != nil {
|
||||
compiledLogs.Subscriptions = append(compiledLogs.Subscriptions, strategy.Logs.Subscriptions...)
|
||||
}
|
||||
|
||||
if awsMetricsEnabled(svcCfg.AWS) && strategy.Metrics != nil {
|
||||
compiledMetrics.StreamFilters = append(compiledMetrics.StreamFilters, strategy.Metrics.StreamFilters...)
|
||||
}
|
||||
}
|
||||
|
||||
awsTelemetry := &cloudintegrationtypes.AWSCollectionStrategy{}
|
||||
if len(compiledMetrics.StreamFilters) > 0 {
|
||||
awsTelemetry.Metrics = compiledMetrics
|
||||
}
|
||||
if len(compiledLogs.Subscriptions) > 0 {
|
||||
awsTelemetry.Logs = compiledLogs
|
||||
}
|
||||
if compiledS3Buckets != nil {
|
||||
awsTelemetry.S3Buckets = compiledS3Buckets
|
||||
}
|
||||
|
||||
enabledRegions := []string{}
|
||||
if account.Config != nil && account.Config.AWS != nil && account.Config.AWS.Regions != nil {
|
||||
enabledRegions = account.Config.AWS.Regions
|
||||
}
|
||||
|
||||
return &cloudintegrationtypes.ProviderIntegrationConfig{
|
||||
AWS: &cloudintegrationtypes.AWSIntegrationConfig{
|
||||
EnabledRegions: enabledRegions,
|
||||
Telemetry: awsTelemetry,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package implcloudprovider
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
)
|
||||
|
||||
type azurecloudprovider struct{}
|
||||
|
||||
func NewAzureCloudProvider() cloudintegration.CloudProviderModule {
|
||||
return &azurecloudprovider{}
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) GetConnectionArtifact(ctx context.Context, creds *cloudintegrationtypes.SignozCredentials, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.ConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) ListServiceDefinitions() ([]cloudintegrationtypes.ServiceDefinition, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) GetServiceDefinition(serviceID cloudintegrationtypes.ServiceID) (*cloudintegrationtypes.ServiceDefinition, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) StorableConfigFromServiceConfig(cfg *cloudintegrationtypes.ServiceConfig, supported cloudintegrationtypes.SupportedSignals) (string, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) ServiceConfigFromStorableServiceConfig(config string) (*cloudintegrationtypes.ServiceConfig, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) IsServiceEnabled(config *cloudintegrationtypes.ServiceConfig) bool {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) IsMetricsEnabled(config *cloudintegrationtypes.ServiceConfig) bool {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (provider *azurecloudprovider) BuildIntegrationConfig(
|
||||
account *cloudintegrationtypes.Account,
|
||||
services []*cloudintegrationtypes.StorableCloudIntegrationService,
|
||||
) (*cloudintegrationtypes.ProviderIntegrationConfig, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "azure cloud provider is not supported")
|
||||
}
|
||||
572
ee/modules/cloudintegration/implcloudintegration/module.go
Normal file
572
ee/modules/cloudintegration/implcloudintegration/module.go
Normal file
@@ -0,0 +1,572 @@
|
||||
package implcloudintegration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/ee/modules/cloudintegration/implcloudintegration/implcloudprovider"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/gateway"
|
||||
"github.com/SigNoz/signoz/pkg/global"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/SigNoz/signoz/pkg/zeus"
|
||||
)
|
||||
|
||||
type module struct {
|
||||
userGetter user.Getter
|
||||
userSetter user.Setter
|
||||
store cloudintegrationtypes.Store
|
||||
gateway gateway.Gateway
|
||||
zeus zeus.Zeus
|
||||
licensing licensing.Licensing
|
||||
globalConfig global.Config
|
||||
cloudProvidersMap map[cloudintegrationtypes.CloudProviderType]cloudintegration.CloudProviderModule
|
||||
}
|
||||
|
||||
func NewModule(
|
||||
store cloudintegrationtypes.Store,
|
||||
globalConfig global.Config,
|
||||
zeus zeus.Zeus,
|
||||
gateway gateway.Gateway,
|
||||
licensing licensing.Licensing,
|
||||
userGetter user.Getter,
|
||||
userSetter user.Setter,
|
||||
) (cloudintegration.Module, error) {
|
||||
awsCloudProviderModule, err := implcloudprovider.NewAWSCloudProvider()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
azureCloudProviderModule := implcloudprovider.NewAzureCloudProvider()
|
||||
cloudProvidersMap := map[cloudintegrationtypes.CloudProviderType]cloudintegration.CloudProviderModule{
|
||||
cloudintegrationtypes.CloudProviderTypeAWS: awsCloudProviderModule,
|
||||
cloudintegrationtypes.CloudProviderTypeAzure: azureCloudProviderModule,
|
||||
}
|
||||
|
||||
return &module{
|
||||
store: store,
|
||||
globalConfig: globalConfig,
|
||||
zeus: zeus,
|
||||
gateway: gateway,
|
||||
licensing: licensing,
|
||||
userGetter: userGetter,
|
||||
userSetter: userSetter,
|
||||
cloudProvidersMap: cloudProvidersMap,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (module *module) CreateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
|
||||
_, err := module.licensing.GetActive(ctx, account.OrgID)
|
||||
if err != nil {
|
||||
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
storableCloudIntegration, err := cloudintegrationtypes.NewStorableCloudIntegration(account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return module.store.CreateAccount(ctx, storableCloudIntegration)
|
||||
}
|
||||
|
||||
func (module *module) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.ConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
|
||||
// TODO: evaluate if this check is really required and remove if the deployment promises to always have this configured.
|
||||
if module.globalConfig.IngestionURL == nil {
|
||||
return nil, errors.New(errors.TypeInternal, errors.CodeInternal, "ingestion URL is not configured")
|
||||
}
|
||||
|
||||
// get license to get the deployment details
|
||||
license, err := module.licensing.GetActive(ctx, account.OrgID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// get deployment details from zeus
|
||||
respBytes, err := module.zeus.GetDeployment(ctx, license.Key)
|
||||
if err != nil {
|
||||
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't get deployment")
|
||||
}
|
||||
|
||||
// parse deployment details
|
||||
deployment, err := zeustypes.NewGettableDeployment(respBytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
apiKey, err := module.getOrCreateAPIKey(ctx, account.OrgID, account.Provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ingestionKey, err := module.getOrCreateIngestionKey(ctx, account.OrgID, account.Provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
creds := &cloudintegrationtypes.SignozCredentials{
|
||||
SigNozAPIURL: deployment.SignozAPIUrl,
|
||||
SigNozAPIKey: apiKey,
|
||||
IngestionURL: module.globalConfig.IngestionURL.String(),
|
||||
IngestionKey: ingestionKey,
|
||||
}
|
||||
|
||||
cloudProviderModule, err := module.GetCloudProvider(account.Provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cloudProviderModule.GetConnectionArtifact(ctx, creds, account, req)
|
||||
}
|
||||
|
||||
func (module *module) GetAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Account, error) {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
storableAccount, err := module.store.GetAccountByID(ctx, orgID, accountID, provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cloudintegrationtypes.NewAccountFromStorable(storableAccount)
|
||||
}
|
||||
|
||||
// ListAccounts return only agent connected accounts
|
||||
func (module *module) ListAccounts(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) ([]*cloudintegrationtypes.Account, error) {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
storableAccounts, err := module.store.ListConnectedAccounts(ctx, orgID, provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cloudintegrationtypes.NewAccountsFromStorables(storableAccounts)
|
||||
}
|
||||
|
||||
func (module *module) AgentCheckIn(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, req *cloudintegrationtypes.AgentCheckInRequest) (*cloudintegrationtypes.AgentCheckInResponse, error) {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
connectedAccount, err := module.store.GetConnectedAccount(ctx, orgID, provider, req.ProviderAccountID)
|
||||
if err != nil && !errors.Ast(err, errors.TypeNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If a different integration is already connected to this provider account ID, reject the check-in.
|
||||
// Allow re-check-in from the same integration (e.g. agent restarting).
|
||||
if connectedAccount != nil && connectedAccount.ID != req.CloudIntegrationID {
|
||||
errMessage := fmt.Sprintf("provider account id %s is already connected to cloud integration id %s", req.ProviderAccountID, connectedAccount.ID)
|
||||
return nil, errors.New(errors.TypeAlreadyExists, cloudintegrationtypes.ErrCodeCloudIntegrationAlreadyConnected, errMessage)
|
||||
}
|
||||
|
||||
account, err := module.store.GetAccountByID(ctx, orgID, req.CloudIntegrationID, provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
account.AccountID = &req.ProviderAccountID
|
||||
account.LastAgentReport = &cloudintegrationtypes.StorableAgentReport{
|
||||
TimestampMillis: time.Now().UnixMilli(),
|
||||
Data: req.Data,
|
||||
}
|
||||
|
||||
err = module.store.UpdateAccount(ctx, account)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If account has been removed (disconnected), return a minimal response with empty integration config.
|
||||
// The agent doesn't act on config for removed accounts.
|
||||
if account.RemovedAt != nil {
|
||||
return &cloudintegrationtypes.AgentCheckInResponse{
|
||||
CloudIntegrationID: account.ID.StringValue(),
|
||||
ProviderAccountID: req.ProviderAccountID,
|
||||
IntegrationConfig: &cloudintegrationtypes.ProviderIntegrationConfig{},
|
||||
RemovedAt: account.RemovedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Get account as domain object for config access (enabled regions, etc.)
|
||||
accountDomain, err := cloudintegrationtypes.NewAccountFromStorable(account)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cloudProvider, err := module.GetCloudProvider(provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
storedServices, err := module.store.ListServices(ctx, req.CloudIntegrationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Delegate integration config building entirely to the provider module
|
||||
integrationConfig, err := cloudProvider.BuildIntegrationConfig(accountDomain, storedServices)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &cloudintegrationtypes.AgentCheckInResponse{
|
||||
CloudIntegrationID: account.ID.StringValue(),
|
||||
ProviderAccountID: req.ProviderAccountID,
|
||||
IntegrationConfig: integrationConfig,
|
||||
RemovedAt: account.RemovedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (module *module) UpdateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
|
||||
_, err := module.licensing.GetActive(ctx, account.OrgID)
|
||||
if err != nil {
|
||||
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
storableAccount, err := cloudintegrationtypes.NewStorableCloudIntegration(account)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return module.store.UpdateAccount(ctx, storableAccount)
|
||||
}
|
||||
|
||||
func (module *module) DisconnectAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) error {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
return module.store.RemoveAccount(ctx, orgID, accountID, provider)
|
||||
}
|
||||
|
||||
func (module *module) ListServicesMetadata(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, integrationID *valuer.UUID) ([]*cloudintegrationtypes.ServiceMetadata, error) {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
cloudProvider, err := module.GetCloudProvider(provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serviceDefinitions, err := cloudProvider.ListServiceDefinitions()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
enabledServiceIDs := map[string]bool{}
|
||||
if integrationID != nil {
|
||||
_, err := module.store.GetAccountByID(ctx, orgID, *integrationID, provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
storedServices, err := module.store.ListServices(ctx, *integrationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, svc := range storedServices {
|
||||
serviceConfig, err := cloudProvider.ServiceConfigFromStorableServiceConfig(svc.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if cloudProvider.IsServiceEnabled(serviceConfig) {
|
||||
enabledServiceIDs[svc.Type.StringValue()] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resp := make([]*cloudintegrationtypes.ServiceMetadata, 0, len(serviceDefinitions))
|
||||
for _, serviceDefinition := range serviceDefinitions {
|
||||
resp = append(resp, cloudintegrationtypes.NewServiceMetadata(serviceDefinition, enabledServiceIDs[serviceDefinition.ID]))
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (module *module) GetService(ctx context.Context, orgID valuer.UUID, integrationID *valuer.UUID, serviceID cloudintegrationtypes.ServiceID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Service, error) {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
cloudProvider, err := module.GetCloudProvider(provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
serviceDefinition, err := cloudProvider.GetServiceDefinition(serviceID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var integrationService *cloudintegrationtypes.CloudIntegrationService
|
||||
|
||||
if integrationID != nil {
|
||||
_, err := module.store.GetAccountByID(ctx, orgID, *integrationID, provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
storedService, err := module.store.GetServiceByServiceID(ctx, *integrationID, serviceID)
|
||||
if err != nil && !errors.Ast(err, errors.TypeNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
if storedService != nil {
|
||||
serviceConfig, err := cloudProvider.ServiceConfigFromStorableServiceConfig(storedService.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
integrationService = cloudintegrationtypes.NewCloudIntegrationServiceFromStorable(storedService, serviceConfig)
|
||||
}
|
||||
}
|
||||
|
||||
return cloudintegrationtypes.NewService(*serviceDefinition, integrationService), nil
|
||||
}
|
||||
|
||||
func (module *module) CreateService(ctx context.Context, orgID valuer.UUID, service *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
cloudProvider, err := module.GetCloudProvider(provider)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
serviceDefinition, err := cloudProvider.GetServiceDefinition(service.Type)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configJSON, err := cloudProvider.StorableConfigFromServiceConfig(service.Config, serviceDefinition.SupportedSignals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return module.store.CreateService(ctx, cloudintegrationtypes.NewStorableCloudIntegrationService(service, configJSON))
|
||||
}
|
||||
|
||||
func (module *module) UpdateService(ctx context.Context, orgID valuer.UUID, integrationService *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
cloudProvider, err := module.GetCloudProvider(provider)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
serviceDefinition, err := cloudProvider.GetServiceDefinition(integrationService.Type)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configJSON, err := cloudProvider.StorableConfigFromServiceConfig(integrationService.Config, serviceDefinition.SupportedSignals)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
storableService := cloudintegrationtypes.NewStorableCloudIntegrationService(integrationService, configJSON)
|
||||
|
||||
return module.store.UpdateService(ctx, storableService)
|
||||
}
|
||||
|
||||
func (module *module) listDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error) {
|
||||
var allDashboards []*dashboardtypes.Dashboard
|
||||
|
||||
for provider := range module.cloudProvidersMap {
|
||||
cloudProvider, err := module.GetCloudProvider(provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
connectedAccounts, err := module.store.ListConnectedAccounts(ctx, orgID, provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, storableAccount := range connectedAccounts {
|
||||
storedServices, err := module.store.ListServices(ctx, storableAccount.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, storedSvc := range storedServices {
|
||||
serviceConfig, err := cloudProvider.ServiceConfigFromStorableServiceConfig(storedSvc.Config)
|
||||
if err != nil || !cloudProvider.IsMetricsEnabled(serviceConfig) {
|
||||
continue
|
||||
}
|
||||
|
||||
svcDef, err := cloudProvider.GetServiceDefinition(storedSvc.Type)
|
||||
if err != nil || svcDef == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
dashboards := cloudintegrationtypes.GetDashboardsFromAssets(
|
||||
storedSvc.Type.StringValue(),
|
||||
orgID,
|
||||
provider,
|
||||
storableAccount.CreatedAt,
|
||||
svcDef.Assets,
|
||||
)
|
||||
allDashboards = append(allDashboards, dashboards...)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(allDashboards, func(i, j int) bool {
|
||||
return allDashboards[i].ID < allDashboards[j].ID
|
||||
})
|
||||
|
||||
return allDashboards, nil
|
||||
}
|
||||
|
||||
func (module *module) GetDashboardByID(ctx context.Context, orgID valuer.UUID, id string) (*dashboardtypes.Dashboard, error) {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
_, _, _, err = cloudintegrationtypes.ParseCloudIntegrationDashboardID(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
allDashboards, err := module.listDashboards(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, d := range allDashboards {
|
||||
if d.ID == id {
|
||||
return d, nil
|
||||
}
|
||||
}
|
||||
|
||||
return nil, errors.New(errors.TypeNotFound, cloudintegrationtypes.ErrCodeCloudIntegrationNotFound, "cloud integration dashboard not found")
|
||||
}
|
||||
|
||||
func (module *module) ListDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error) {
|
||||
_, err := module.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
}
|
||||
|
||||
return module.listDashboards(ctx, orgID)
|
||||
}
|
||||
|
||||
func (module *module) GetCloudProvider(provider cloudintegrationtypes.CloudProviderType) (cloudintegration.CloudProviderModule, error) {
|
||||
if cloudProviderModule, ok := module.cloudProvidersMap[provider]; ok {
|
||||
return cloudProviderModule, nil
|
||||
}
|
||||
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "cloud provider is not supported")
|
||||
}
|
||||
|
||||
func (module *module) getOrCreateIngestionKey(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (string, error) {
|
||||
keyName := cloudintegrationtypes.NewIngestionKeyName(provider)
|
||||
|
||||
result, err := module.gateway.SearchIngestionKeysByName(ctx, orgID, keyName, 1, 10)
|
||||
if err != nil {
|
||||
return "", errors.WrapInternalf(err, errors.CodeInternal, "couldn't search ingestion keys")
|
||||
}
|
||||
|
||||
// ideally there should be only one key per cloud integration provider
|
||||
if len(result.Keys) > 0 {
|
||||
return result.Keys[0].Value, nil
|
||||
}
|
||||
|
||||
createdIngestionKey, err := module.gateway.CreateIngestionKey(ctx, orgID, keyName, []string{"integration"}, time.Time{})
|
||||
if err != nil {
|
||||
return "", errors.WrapInternalf(err, errors.CodeInternal, "couldn't create ingestion key")
|
||||
}
|
||||
|
||||
return createdIngestionKey.Value, nil
|
||||
}
|
||||
|
||||
func (module *module) getOrCreateAPIKey(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (string, error) {
|
||||
integrationUser, err := module.getOrCreateIntegrationUser(ctx, orgID, provider)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
existingKeys, err := module.userSetter.ListAPIKeys(ctx, orgID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
keyName := cloudintegrationtypes.NewAPIKeyName(provider)
|
||||
|
||||
for _, key := range existingKeys {
|
||||
if key.Name == keyName && key.UserID == integrationUser.ID {
|
||||
return key.Token, nil
|
||||
}
|
||||
}
|
||||
|
||||
apiKey, err := types.NewStorableAPIKey(keyName, integrationUser.ID, types.RoleViewer, 0)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
err = module.userSetter.CreateAPIKey(ctx, apiKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return apiKey.Token, nil
|
||||
}
|
||||
|
||||
func (module *module) getOrCreateIntegrationUser(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (*types.User, error) {
|
||||
email, err := cloudintegrationtypes.GetCloudProviderEmail(provider)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// get user by email
|
||||
integrationUser, err := module.userGetter.GetNonDeletedUserByEmailAndOrgID(ctx, email, orgID)
|
||||
if err != nil && !errors.Ast(err, errors.TypeNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if user found, return
|
||||
if integrationUser != nil {
|
||||
return integrationUser, nil
|
||||
}
|
||||
|
||||
// if user not found, create a new one
|
||||
displayName := cloudintegrationtypes.NewIntegrationUserDisplayName(provider)
|
||||
integrationUser, err = types.NewUser(displayName, email, orgID, types.UserStatusActive)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
password := types.MustGenerateFactorPassword(integrationUser.ID.String())
|
||||
|
||||
err = module.userSetter.CreateUser(ctx, integrationUser, user.WithFactorPassword(password))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return integrationUser, nil
|
||||
}
|
||||
@@ -205,6 +205,25 @@ module.exports = {
|
||||
],
|
||||
},
|
||||
overrides: [
|
||||
{
|
||||
files: ['src/**/*.{jsx,tsx,ts}'],
|
||||
excludedFiles: [
|
||||
'**/*.test.{js,jsx,ts,tsx}',
|
||||
'**/*.spec.{js,jsx,ts,tsx}',
|
||||
'**/__tests__/**/*.{js,jsx,ts,tsx}',
|
||||
],
|
||||
rules: {
|
||||
'no-restricted-properties': [
|
||||
'error',
|
||||
{
|
||||
object: 'navigator',
|
||||
property: 'clipboard',
|
||||
message:
|
||||
'Do not use navigator.clipboard directly since it does not work well with specific browsers. Use hook useCopyToClipboard from react-use library. https://streamich.github.io/react-use/?path=/story/side-effects-usecopytoclipboard--docs',
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
files: [
|
||||
'**/*.test.{js,jsx,ts,tsx}',
|
||||
|
||||
@@ -628,6 +628,103 @@ export const useUpdateAccount = <
|
||||
|
||||
return useMutation(mutationOptions);
|
||||
};
|
||||
/**
|
||||
* This endpoint updates a service for the specified cloud provider
|
||||
* @summary Update service
|
||||
*/
|
||||
export const updateService = (
|
||||
{ cloudProvider, id, serviceId }: UpdateServicePathParameters,
|
||||
cloudintegrationtypesUpdatableServiceDTO: BodyType<CloudintegrationtypesUpdatableServiceDTO>,
|
||||
) => {
|
||||
return GeneratedAPIInstance<void>({
|
||||
url: `/api/v1/cloud_integrations/${cloudProvider}/accounts/${id}/services/${serviceId}`,
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
data: cloudintegrationtypesUpdatableServiceDTO,
|
||||
});
|
||||
};
|
||||
|
||||
export const getUpdateServiceMutationOptions = <
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
TContext = unknown
|
||||
>(options?: {
|
||||
mutation?: UseMutationOptions<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
}): UseMutationOptions<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
const mutationKey = ['updateService'];
|
||||
const { mutation: mutationOptions } = options
|
||||
? options.mutation &&
|
||||
'mutationKey' in options.mutation &&
|
||||
options.mutation.mutationKey
|
||||
? options
|
||||
: { ...options, mutation: { ...options.mutation, mutationKey } }
|
||||
: { mutation: { mutationKey } };
|
||||
|
||||
const mutationFn: MutationFunction<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
}
|
||||
> = (props) => {
|
||||
const { pathParams, data } = props ?? {};
|
||||
|
||||
return updateService(pathParams, data);
|
||||
};
|
||||
|
||||
return { mutationFn, ...mutationOptions };
|
||||
};
|
||||
|
||||
export type UpdateServiceMutationResult = NonNullable<
|
||||
Awaited<ReturnType<typeof updateService>>
|
||||
>;
|
||||
export type UpdateServiceMutationBody = BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
export type UpdateServiceMutationError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Update service
|
||||
*/
|
||||
export const useUpdateService = <
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
TContext = unknown
|
||||
>(options?: {
|
||||
mutation?: UseMutationOptions<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
}): UseMutationResult<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
const mutationOptions = getUpdateServiceMutationOptions(options);
|
||||
|
||||
return useMutation(mutationOptions);
|
||||
};
|
||||
/**
|
||||
* This endpoint is called by the deployed agent to check in
|
||||
* @summary Agent check-in
|
||||
@@ -941,101 +1038,3 @@ export const invalidateGetService = async (
|
||||
|
||||
return queryClient;
|
||||
};
|
||||
|
||||
/**
|
||||
* This endpoint updates a service for the specified cloud provider
|
||||
* @summary Update service
|
||||
*/
|
||||
export const updateService = (
|
||||
{ cloudProvider, serviceId }: UpdateServicePathParameters,
|
||||
cloudintegrationtypesUpdatableServiceDTO: BodyType<CloudintegrationtypesUpdatableServiceDTO>,
|
||||
) => {
|
||||
return GeneratedAPIInstance<void>({
|
||||
url: `/api/v1/cloud_integrations/${cloudProvider}/services/${serviceId}`,
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
data: cloudintegrationtypesUpdatableServiceDTO,
|
||||
});
|
||||
};
|
||||
|
||||
export const getUpdateServiceMutationOptions = <
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
TContext = unknown
|
||||
>(options?: {
|
||||
mutation?: UseMutationOptions<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
}): UseMutationOptions<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
const mutationKey = ['updateService'];
|
||||
const { mutation: mutationOptions } = options
|
||||
? options.mutation &&
|
||||
'mutationKey' in options.mutation &&
|
||||
options.mutation.mutationKey
|
||||
? options
|
||||
: { ...options, mutation: { ...options.mutation, mutationKey } }
|
||||
: { mutation: { mutationKey } };
|
||||
|
||||
const mutationFn: MutationFunction<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
}
|
||||
> = (props) => {
|
||||
const { pathParams, data } = props ?? {};
|
||||
|
||||
return updateService(pathParams, data);
|
||||
};
|
||||
|
||||
return { mutationFn, ...mutationOptions };
|
||||
};
|
||||
|
||||
export type UpdateServiceMutationResult = NonNullable<
|
||||
Awaited<ReturnType<typeof updateService>>
|
||||
>;
|
||||
export type UpdateServiceMutationBody = BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
export type UpdateServiceMutationError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Update service
|
||||
*/
|
||||
export const useUpdateService = <
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
TContext = unknown
|
||||
>(options?: {
|
||||
mutation?: UseMutationOptions<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
}): UseMutationResult<
|
||||
Awaited<ReturnType<typeof updateService>>,
|
||||
TError,
|
||||
{
|
||||
pathParams: UpdateServicePathParameters;
|
||||
data: BodyType<CloudintegrationtypesUpdatableServiceDTO>;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
const mutationOptions = getUpdateServiceMutationOptions(options);
|
||||
|
||||
return useMutation(mutationOptions);
|
||||
};
|
||||
|
||||
@@ -550,12 +550,12 @@ export type CloudintegrationtypesAWSCollectionStrategyDTOS3Buckets = {
|
||||
};
|
||||
|
||||
export interface CloudintegrationtypesAWSCollectionStrategyDTO {
|
||||
aws_logs?: CloudintegrationtypesAWSLogsStrategyDTO;
|
||||
aws_metrics?: CloudintegrationtypesAWSMetricsStrategyDTO;
|
||||
logs?: CloudintegrationtypesAWSLogsStrategyDTO;
|
||||
metrics?: CloudintegrationtypesAWSMetricsStrategyDTO;
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
s3_buckets?: CloudintegrationtypesAWSCollectionStrategyDTOS3Buckets;
|
||||
s3Buckets?: CloudintegrationtypesAWSCollectionStrategyDTOS3Buckets;
|
||||
}
|
||||
|
||||
export interface CloudintegrationtypesAWSConnectionArtifactDTO {
|
||||
@@ -588,11 +588,11 @@ export type CloudintegrationtypesAWSLogsStrategyDTOCloudwatchLogsSubscriptionsIt
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
filter_pattern?: string;
|
||||
filterPattern?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
log_group_name_prefix?: string;
|
||||
logGroupNamePrefix?: string;
|
||||
};
|
||||
|
||||
export interface CloudintegrationtypesAWSLogsStrategyDTO {
|
||||
@@ -600,7 +600,7 @@ export interface CloudintegrationtypesAWSLogsStrategyDTO {
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
cloudwatch_logs_subscriptions?:
|
||||
cloudwatchLogsSubscriptions?:
|
||||
| CloudintegrationtypesAWSLogsStrategyDTOCloudwatchLogsSubscriptionsItem[]
|
||||
| null;
|
||||
}
|
||||
@@ -621,7 +621,7 @@ export interface CloudintegrationtypesAWSMetricsStrategyDTO {
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
cloudwatch_metric_stream_filters?:
|
||||
cloudwatchMetricStreamFilters?:
|
||||
| CloudintegrationtypesAWSMetricsStrategyDTOCloudwatchMetricStreamFiltersItem[]
|
||||
| null;
|
||||
}
|
||||
@@ -726,6 +726,32 @@ export interface CloudintegrationtypesAssetsDTO {
|
||||
dashboards?: CloudintegrationtypesDashboardDTO[] | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @nullable
|
||||
*/
|
||||
export type CloudintegrationtypesCloudIntegrationServiceDTO = {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
cloudIntegrationId?: string;
|
||||
config?: CloudintegrationtypesServiceConfigDTO;
|
||||
/**
|
||||
* @type string
|
||||
* @format date-time
|
||||
*/
|
||||
createdAt?: Date;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
id: string;
|
||||
type?: CloudintegrationtypesServiceIDDTO;
|
||||
/**
|
||||
* @type string
|
||||
* @format date-time
|
||||
*/
|
||||
updatedAt?: Date;
|
||||
} | null;
|
||||
|
||||
export interface CloudintegrationtypesCollectedLogAttributeDTO {
|
||||
/**
|
||||
* @type string
|
||||
@@ -864,9 +890,68 @@ export type CloudintegrationtypesIntegrationConfigDTO = {
|
||||
* @type array
|
||||
*/
|
||||
enabled_regions: string[];
|
||||
telemetry: CloudintegrationtypesAWSCollectionStrategyDTO;
|
||||
telemetry: CloudintegrationtypesOldAWSCollectionStrategyDTO;
|
||||
} | null;
|
||||
|
||||
export type CloudintegrationtypesOldAWSCollectionStrategyDTOS3Buckets = {
|
||||
[key: string]: string[];
|
||||
};
|
||||
|
||||
export interface CloudintegrationtypesOldAWSCollectionStrategyDTO {
|
||||
aws_logs?: CloudintegrationtypesOldAWSLogsStrategyDTO;
|
||||
aws_metrics?: CloudintegrationtypesOldAWSMetricsStrategyDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
provider?: string;
|
||||
/**
|
||||
* @type object
|
||||
*/
|
||||
s3_buckets?: CloudintegrationtypesOldAWSCollectionStrategyDTOS3Buckets;
|
||||
}
|
||||
|
||||
export type CloudintegrationtypesOldAWSLogsStrategyDTOCloudwatchLogsSubscriptionsItem = {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
filter_pattern?: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
log_group_name_prefix?: string;
|
||||
};
|
||||
|
||||
export interface CloudintegrationtypesOldAWSLogsStrategyDTO {
|
||||
/**
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
cloudwatch_logs_subscriptions?:
|
||||
| CloudintegrationtypesOldAWSLogsStrategyDTOCloudwatchLogsSubscriptionsItem[]
|
||||
| null;
|
||||
}
|
||||
|
||||
export type CloudintegrationtypesOldAWSMetricsStrategyDTOCloudwatchMetricStreamFiltersItem = {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
MetricNames?: string[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
Namespace?: string;
|
||||
};
|
||||
|
||||
export interface CloudintegrationtypesOldAWSMetricsStrategyDTO {
|
||||
/**
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
cloudwatch_metric_stream_filters?:
|
||||
| CloudintegrationtypesOldAWSMetricsStrategyDTOCloudwatchMetricStreamFiltersItem[]
|
||||
| null;
|
||||
}
|
||||
|
||||
/**
|
||||
* @nullable
|
||||
*/
|
||||
@@ -904,6 +989,7 @@ export interface CloudintegrationtypesProviderIntegrationConfigDTO {
|
||||
|
||||
export interface CloudintegrationtypesServiceDTO {
|
||||
assets: CloudintegrationtypesAssetsDTO;
|
||||
cloudIntegrationService: CloudintegrationtypesCloudIntegrationServiceDTO;
|
||||
dataCollected: CloudintegrationtypesDataCollectedDTO;
|
||||
/**
|
||||
* @type string
|
||||
@@ -917,8 +1003,7 @@ export interface CloudintegrationtypesServiceDTO {
|
||||
* @type string
|
||||
*/
|
||||
overview: string;
|
||||
serviceConfig?: CloudintegrationtypesServiceConfigDTO;
|
||||
supported_signals: CloudintegrationtypesSupportedSignalsDTO;
|
||||
supportedSignals: CloudintegrationtypesSupportedSignalsDTO;
|
||||
telemetryCollectionStrategy: CloudintegrationtypesCollectionStrategyDTO;
|
||||
/**
|
||||
* @type string
|
||||
@@ -930,6 +1015,21 @@ export interface CloudintegrationtypesServiceConfigDTO {
|
||||
aws: CloudintegrationtypesAWSServiceConfigDTO;
|
||||
}
|
||||
|
||||
export enum CloudintegrationtypesServiceIDDTO {
|
||||
alb = 'alb',
|
||||
'api-gateway' = 'api-gateway',
|
||||
dynamodb = 'dynamodb',
|
||||
ec2 = 'ec2',
|
||||
ecs = 'ecs',
|
||||
eks = 'eks',
|
||||
elasticache = 'elasticache',
|
||||
lambda = 'lambda',
|
||||
msk = 'msk',
|
||||
rds = 'rds',
|
||||
s3sync = 's3sync',
|
||||
sns = 'sns',
|
||||
sqs = 'sqs',
|
||||
}
|
||||
export interface CloudintegrationtypesServiceMetadataDTO {
|
||||
/**
|
||||
* @type boolean
|
||||
@@ -3456,6 +3556,11 @@ export type UpdateAccountPathParameters = {
|
||||
cloudProvider: string;
|
||||
id: string;
|
||||
};
|
||||
export type UpdateServicePathParameters = {
|
||||
cloudProvider: string;
|
||||
id: string;
|
||||
serviceId: string;
|
||||
};
|
||||
export type AgentCheckInPathParameters = {
|
||||
cloudProvider: string;
|
||||
};
|
||||
@@ -3490,10 +3595,6 @@ export type GetService200 = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type UpdateServicePathParameters = {
|
||||
cloudProvider: string;
|
||||
serviceId: string;
|
||||
};
|
||||
export type CreateSessionByGoogleCallback303 = {
|
||||
data: AuthtypesGettableTokenDTO;
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { useCopyToClipboard } from 'react-use';
|
||||
import { Badge } from '@signozhq/badge';
|
||||
import { Button } from '@signozhq/button';
|
||||
import { DialogFooter, DialogWrapper } from '@signozhq/dialog';
|
||||
@@ -177,26 +178,30 @@ function EditMemberDrawer({
|
||||
}
|
||||
}, [member, isInvited, setLinkType, onClose]);
|
||||
|
||||
const [copyState, copyToClipboard] = useCopyToClipboard();
|
||||
const handleCopyResetLink = useCallback(async (): Promise<void> => {
|
||||
if (!resetLink) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await navigator.clipboard.writeText(resetLink);
|
||||
setHasCopiedResetLink(true);
|
||||
setTimeout(() => setHasCopiedResetLink(false), 2000);
|
||||
toast.success(
|
||||
linkType === 'invite'
|
||||
? 'Invite link copied to clipboard'
|
||||
: 'Reset link copied to clipboard',
|
||||
{ richColors: true },
|
||||
);
|
||||
} catch {
|
||||
copyToClipboard(resetLink);
|
||||
|
||||
setHasCopiedResetLink(true);
|
||||
setTimeout(() => setHasCopiedResetLink(false), 2000);
|
||||
toast.success(
|
||||
linkType === 'invite'
|
||||
? 'Invite link copied to clipboard'
|
||||
: 'Reset link copied to clipboard',
|
||||
{ richColors: true },
|
||||
);
|
||||
}, [resetLink, copyToClipboard, linkType]);
|
||||
|
||||
useEffect(() => {
|
||||
if (copyState.error) {
|
||||
toast.error('Failed to copy link', {
|
||||
richColors: true,
|
||||
});
|
||||
}
|
||||
}, [resetLink, linkType]);
|
||||
}, [copyState.error]);
|
||||
|
||||
const handleClose = useCallback((): void => {
|
||||
setShowDeleteConfirm(false);
|
||||
|
||||
@@ -7,13 +7,7 @@ import {
|
||||
useUpdateUserDeprecated,
|
||||
} from 'api/generated/services/users';
|
||||
import { MemberStatus } from 'container/MembersSettings/utils';
|
||||
import {
|
||||
fireEvent,
|
||||
render,
|
||||
screen,
|
||||
userEvent,
|
||||
waitFor,
|
||||
} from 'tests/test-utils';
|
||||
import { render, screen, userEvent, waitFor } from 'tests/test-utils';
|
||||
import { ROLES } from 'types/roles';
|
||||
|
||||
import EditMemberDrawer, { EditMemberDrawerProps } from '../EditMemberDrawer';
|
||||
@@ -65,6 +59,16 @@ jest.mock('@signozhq/sonner', () => ({
|
||||
},
|
||||
}));
|
||||
|
||||
const mockCopyToClipboard = jest.fn();
|
||||
const mockCopyState = { value: undefined, error: undefined };
|
||||
|
||||
jest.mock('react-use', () => ({
|
||||
useCopyToClipboard: (): [typeof mockCopyState, typeof mockCopyToClipboard] => [
|
||||
mockCopyState,
|
||||
mockCopyToClipboard,
|
||||
],
|
||||
}));
|
||||
|
||||
const mockUpdateMutate = jest.fn();
|
||||
const mockDeleteMutate = jest.fn();
|
||||
const mockGetResetPasswordToken = jest.mocked(getResetPasswordToken);
|
||||
@@ -361,32 +365,14 @@ describe('EditMemberDrawer', () => {
|
||||
});
|
||||
|
||||
describe('Generate Password Reset Link', () => {
|
||||
const mockWriteText = jest.fn().mockResolvedValue(undefined);
|
||||
let clipboardSpy: jest.SpyInstance | undefined;
|
||||
|
||||
beforeAll(() => {
|
||||
Object.defineProperty(navigator, 'clipboard', {
|
||||
value: { writeText: (): Promise<void> => Promise.resolve() },
|
||||
configurable: true,
|
||||
writable: true,
|
||||
});
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockWriteText.mockClear();
|
||||
clipboardSpy = jest
|
||||
.spyOn(navigator.clipboard, 'writeText')
|
||||
.mockImplementation(mockWriteText);
|
||||
mockCopyToClipboard.mockClear();
|
||||
mockGetResetPasswordToken.mockResolvedValue({
|
||||
status: 'success',
|
||||
data: { token: 'reset-tok-abc', id: 'user-1' },
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
clipboardSpy?.mockRestore();
|
||||
});
|
||||
|
||||
it('calls getResetPasswordToken and opens the reset link dialog with the generated link', async () => {
|
||||
const user = userEvent.setup({ pointerEventsCheck: 0 });
|
||||
|
||||
@@ -421,7 +407,7 @@ describe('EditMemberDrawer', () => {
|
||||
});
|
||||
expect(dialog).toHaveTextContent('reset-tok-abc');
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: /^copy$/i }));
|
||||
await user.click(screen.getByRole('button', { name: /^copy$/i }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockToast.success).toHaveBeenCalledWith(
|
||||
@@ -430,7 +416,7 @@ describe('EditMemberDrawer', () => {
|
||||
);
|
||||
});
|
||||
|
||||
expect(mockWriteText).toHaveBeenCalledWith(
|
||||
expect(mockCopyToClipboard).toHaveBeenCalledWith(
|
||||
expect.stringContaining('reset-tok-abc'),
|
||||
);
|
||||
expect(screen.getByRole('button', { name: /copied!/i })).toBeInTheDocument();
|
||||
|
||||
@@ -202,19 +202,8 @@ function InviteMembersModal({
|
||||
onComplete?.();
|
||||
} catch (err) {
|
||||
const apiErr = err as APIError;
|
||||
if (apiErr?.getHttpStatusCode() === 409) {
|
||||
toast.error(
|
||||
touchedRows.length === 1
|
||||
? `${touchedRows[0].email} is already a member`
|
||||
: 'Invite for one or more users already exists',
|
||||
{ richColors: true },
|
||||
);
|
||||
} else {
|
||||
const errorMessage = apiErr?.getErrorMessage?.() ?? 'An error occurred';
|
||||
toast.error(`Failed to send invites: ${errorMessage}`, {
|
||||
richColors: true,
|
||||
});
|
||||
}
|
||||
const errorMessage = apiErr?.getErrorMessage?.() ?? 'An error occurred';
|
||||
toast.error(errorMessage, { richColors: true });
|
||||
} finally {
|
||||
setIsSubmitting(false);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,18 @@
|
||||
import { toast } from '@signozhq/sonner';
|
||||
import inviteUsers from 'api/v1/invite/bulk/create';
|
||||
import sendInvite from 'api/v1/invite/create';
|
||||
import { StatusCodes } from 'http-status-codes';
|
||||
import { render, screen, userEvent, waitFor } from 'tests/test-utils';
|
||||
import APIError from 'types/api/error';
|
||||
|
||||
import InviteMembersModal from '../InviteMembersModal';
|
||||
|
||||
const makeApiError = (message: string, code = StatusCodes.CONFLICT): APIError =>
|
||||
new APIError({
|
||||
httpStatusCode: code,
|
||||
error: { code: 'already_exists', message, url: '', errors: [] },
|
||||
});
|
||||
|
||||
jest.mock('api/v1/invite/create');
|
||||
jest.mock('api/v1/invite/bulk/create');
|
||||
jest.mock('@signozhq/sonner', () => ({
|
||||
@@ -142,6 +151,90 @@ describe('InviteMembersModal', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('error handling', () => {
|
||||
it('shows BE message on single invite 409', async () => {
|
||||
const user = userEvent.setup({ pointerEventsCheck: 0 });
|
||||
mockSendInvite.mockRejectedValue(
|
||||
makeApiError('An invite already exists for this email: single@signoz.io'),
|
||||
);
|
||||
|
||||
render(<InviteMembersModal {...defaultProps} />);
|
||||
|
||||
await user.type(
|
||||
screen.getAllByPlaceholderText('john@signoz.io')[0],
|
||||
'single@signoz.io',
|
||||
);
|
||||
await user.click(screen.getAllByText('Select roles')[0]);
|
||||
await user.click(await screen.findByText('Viewer'));
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: /invite team members/i }),
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(toast.error).toHaveBeenCalledWith(
|
||||
'An invite already exists for this email: single@signoz.io',
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('shows BE message on bulk invite 409', async () => {
|
||||
const user = userEvent.setup({ pointerEventsCheck: 0 });
|
||||
mockInviteUsers.mockRejectedValue(
|
||||
makeApiError('An invite already exists for this email: alice@signoz.io'),
|
||||
);
|
||||
|
||||
render(<InviteMembersModal {...defaultProps} />);
|
||||
|
||||
const emailInputs = screen.getAllByPlaceholderText('john@signoz.io');
|
||||
await user.type(emailInputs[0], 'alice@signoz.io');
|
||||
await user.click(screen.getAllByText('Select roles')[0]);
|
||||
await user.click(await screen.findByText('Viewer'));
|
||||
|
||||
await user.type(emailInputs[1], 'bob@signoz.io');
|
||||
await user.click(screen.getAllByText('Select roles')[0]);
|
||||
const editorOptions = await screen.findAllByText('Editor');
|
||||
await user.click(editorOptions[editorOptions.length - 1]);
|
||||
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: /invite team members/i }),
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(toast.error).toHaveBeenCalledWith(
|
||||
'An invite already exists for this email: alice@signoz.io',
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('shows BE message on generic error', async () => {
|
||||
const user = userEvent.setup({ pointerEventsCheck: 0 });
|
||||
mockSendInvite.mockRejectedValue(
|
||||
makeApiError('Internal server error', StatusCodes.INTERNAL_SERVER_ERROR),
|
||||
);
|
||||
|
||||
render(<InviteMembersModal {...defaultProps} />);
|
||||
|
||||
await user.type(
|
||||
screen.getAllByPlaceholderText('john@signoz.io')[0],
|
||||
'single@signoz.io',
|
||||
);
|
||||
await user.click(screen.getAllByText('Select roles')[0]);
|
||||
await user.click(await screen.findByText('Viewer'));
|
||||
await user.click(
|
||||
screen.getByRole('button', { name: /invite team members/i }),
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(toast.error).toHaveBeenCalledWith(
|
||||
'Internal server error',
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('uses inviteUsers (bulk) when multiple rows are filled', async () => {
|
||||
const user = userEvent.setup({ pointerEventsCheck: 0 });
|
||||
const onComplete = jest.fn();
|
||||
|
||||
@@ -17,6 +17,7 @@ function CodeCopyBtn({
|
||||
let copiedText = '';
|
||||
if (children && Array.isArray(children)) {
|
||||
setIsSnippetCopied(true);
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
navigator.clipboard.writeText(children[0].props.children[0]).finally(() => {
|
||||
copiedText = (children[0].props.children[0] as string).slice(0, 200); // slicing is done due to the limitation in accepted char length in attributes
|
||||
setTimeout(() => {
|
||||
|
||||
@@ -401,6 +401,7 @@ const CustomMultiSelect: React.FC<CustomMultiSelectProps> = ({
|
||||
|
||||
const textToCopy = selectedTexts.join(', ');
|
||||
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
navigator.clipboard.writeText(textToCopy).catch(console.error);
|
||||
}, [selectedChips, selectedValues]);
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useCallback, useEffect, useState } from 'react';
|
||||
import { useForm } from 'react-hook-form';
|
||||
import { useQueryClient } from 'react-query';
|
||||
import { useCopyToClipboard } from 'react-use';
|
||||
import { DialogWrapper } from '@signozhq/dialog';
|
||||
import { toast } from '@signozhq/sonner';
|
||||
import { convertToApiError } from 'api/ErrorResponseHandlerForGeneratedAPIs';
|
||||
@@ -105,19 +106,23 @@ function AddKeyModal(): JSX.Element {
|
||||
});
|
||||
}
|
||||
|
||||
const [copyState, copyToClipboard] = useCopyToClipboard();
|
||||
const handleCopy = useCallback(async (): Promise<void> => {
|
||||
if (!createdKey?.key) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await navigator.clipboard.writeText(createdKey.key);
|
||||
setHasCopied(true);
|
||||
setTimeout(() => setHasCopied(false), 2000);
|
||||
toast.success('Key copied to clipboard', { richColors: true });
|
||||
} catch {
|
||||
|
||||
copyToClipboard(createdKey.key);
|
||||
setHasCopied(true);
|
||||
setTimeout(() => setHasCopied(false), 2000);
|
||||
toast.success('Key copied to clipboard', { richColors: true });
|
||||
}, [copyToClipboard, createdKey?.key]);
|
||||
|
||||
useEffect(() => {
|
||||
if (copyState.error) {
|
||||
toast.error('Failed to copy key', { richColors: true });
|
||||
}
|
||||
}, [createdKey]);
|
||||
}, [copyState.error]);
|
||||
|
||||
const handleClose = useCallback((): void => {
|
||||
setIsAddKeyOpen(null);
|
||||
|
||||
@@ -9,6 +9,16 @@ jest.mock('@signozhq/sonner', () => ({
|
||||
toast: { success: jest.fn(), error: jest.fn() },
|
||||
}));
|
||||
|
||||
const mockCopyToClipboard = jest.fn();
|
||||
const mockCopyState = { value: undefined, error: undefined };
|
||||
|
||||
jest.mock('react-use', () => ({
|
||||
useCopyToClipboard: (): [typeof mockCopyState, typeof mockCopyToClipboard] => [
|
||||
mockCopyState,
|
||||
mockCopyToClipboard,
|
||||
],
|
||||
}));
|
||||
|
||||
const mockToast = jest.mocked(toast);
|
||||
|
||||
const SA_KEYS_ENDPOINT = '*/api/v1/service_accounts/sa-1/keys';
|
||||
@@ -35,16 +45,9 @@ function renderModal(): ReturnType<typeof render> {
|
||||
}
|
||||
|
||||
describe('AddKeyModal', () => {
|
||||
beforeAll(() => {
|
||||
Object.defineProperty(navigator, 'clipboard', {
|
||||
value: { writeText: jest.fn().mockResolvedValue(undefined) },
|
||||
configurable: true,
|
||||
writable: true,
|
||||
});
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockCopyToClipboard.mockClear();
|
||||
server.use(
|
||||
rest.post(SA_KEYS_ENDPOINT, (_, res, ctx) =>
|
||||
res(ctx.status(201), ctx.json(createdKeyResponse)),
|
||||
@@ -90,9 +93,6 @@ describe('AddKeyModal', () => {
|
||||
|
||||
it('copy button writes key to clipboard and shows toast.success', async () => {
|
||||
const user = userEvent.setup({ pointerEventsCheck: 0 });
|
||||
const writeTextSpy = jest
|
||||
.spyOn(navigator.clipboard, 'writeText')
|
||||
.mockResolvedValue(undefined);
|
||||
|
||||
renderModal();
|
||||
|
||||
@@ -115,14 +115,12 @@ describe('AddKeyModal', () => {
|
||||
await user.click(copyBtn);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(writeTextSpy).toHaveBeenCalledWith('snz_abc123xyz456secret');
|
||||
expect(mockCopyToClipboard).toHaveBeenCalledWith('snz_abc123xyz456secret');
|
||||
expect(mockToast.success).toHaveBeenCalledWith(
|
||||
'Key copied to clipboard',
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
writeTextSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('Cancel button closes the modal', async () => {
|
||||
|
||||
@@ -16,9 +16,9 @@ function AverageResolutionCard({
|
||||
}: TotalTriggeredCardProps): JSX.Element {
|
||||
return (
|
||||
<StatsCard
|
||||
displayValue={formatTime(currentAvgResolutionTime)}
|
||||
totalCurrentCount={currentAvgResolutionTime}
|
||||
totalPastCount={pastAvgResolutionTime}
|
||||
displayValue={formatTime(+currentAvgResolutionTime)}
|
||||
totalCurrentCount={+currentAvgResolutionTime}
|
||||
totalPastCount={+pastAvgResolutionTime}
|
||||
title="Avg. Resolution Time"
|
||||
timeSeries={timeSeries}
|
||||
/>
|
||||
|
||||
@@ -800,14 +800,10 @@
|
||||
|
||||
.ant-table-cell:has(.top-services-item-latency) {
|
||||
text-align: center;
|
||||
opacity: 0.8;
|
||||
background: rgba(171, 189, 255, 0.04);
|
||||
}
|
||||
|
||||
.ant-table-cell:has(.top-services-item-latency-title) {
|
||||
text-align: center;
|
||||
opacity: 0.8;
|
||||
background: rgba(171, 189, 255, 0.04);
|
||||
}
|
||||
|
||||
.ant-table-tbody > tr:hover > td {
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
.api-quick-filters-header {
|
||||
padding: 12px;
|
||||
border-bottom: 1px solid var(--bg-slate-400);
|
||||
border-right: 1px solid var(--bg-slate-400);
|
||||
|
||||
display: flex;
|
||||
align-items: center;
|
||||
@@ -26,6 +25,7 @@
|
||||
width: 100%;
|
||||
|
||||
.toolbar {
|
||||
border-top: 0px;
|
||||
border-bottom: 1px solid var(--bg-slate-400);
|
||||
}
|
||||
|
||||
@@ -220,6 +220,18 @@
|
||||
}
|
||||
|
||||
.lightMode {
|
||||
.api-quick-filter-left-section {
|
||||
.api-quick-filters-header {
|
||||
border-bottom: 1px solid var(--bg-vanilla-300);
|
||||
}
|
||||
}
|
||||
|
||||
.api-module-right-section {
|
||||
.toolbar {
|
||||
border-bottom: 1px solid var(--bg-vanilla-300);
|
||||
}
|
||||
}
|
||||
|
||||
.no-filtered-domains-message-container {
|
||||
.no-filtered-domains-message-content {
|
||||
.no-filtered-domains-message {
|
||||
|
||||
@@ -464,14 +464,10 @@ function GeneralSettings({
|
||||
onModalToggleHandler(type);
|
||||
};
|
||||
|
||||
const {
|
||||
isCloudUser: isCloudUserVal,
|
||||
isEnterpriseSelfHostedUser,
|
||||
} = useGetTenantLicense();
|
||||
const { isCloudUser: isCloudUserVal } = useGetTenantLicense();
|
||||
|
||||
const isAdmin = user.role === USER_ROLES.ADMIN;
|
||||
const showCustomDomainSettings =
|
||||
(isCloudUserVal || isEnterpriseSelfHostedUser) && isAdmin;
|
||||
const showCustomDomainSettings = isCloudUserVal && isAdmin;
|
||||
|
||||
const renderConfig = [
|
||||
{
|
||||
|
||||
@@ -38,7 +38,6 @@ jest.mock('hooks/useComponentPermission', () => ({
|
||||
jest.mock('hooks/useGetTenantLicense', () => ({
|
||||
useGetTenantLicense: jest.fn(() => ({
|
||||
isCloudUser: false,
|
||||
isEnterpriseSelfHostedUser: false,
|
||||
})),
|
||||
}));
|
||||
|
||||
@@ -389,7 +388,6 @@ describe('GeneralSettings - S3 Logs Retention', () => {
|
||||
beforeEach(() => {
|
||||
(useGetTenantLicense as jest.Mock).mockReturnValue({
|
||||
isCloudUser: true,
|
||||
isEnterpriseSelfHostedUser: false,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -411,15 +409,14 @@ describe('GeneralSettings - S3 Logs Retention', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('Enterprise Self-Hosted User Rendering', () => {
|
||||
describe('Non-cloud user rendering', () => {
|
||||
beforeEach(() => {
|
||||
(useGetTenantLicense as jest.Mock).mockReturnValue({
|
||||
isCloudUser: false,
|
||||
isEnterpriseSelfHostedUser: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('should render CustomDomainSettings but not GeneralSettingsCloud', () => {
|
||||
it('should not render CustomDomainSettings or GeneralSettingsCloud', () => {
|
||||
render(
|
||||
<GeneralSettings
|
||||
metricsTtlValuesPayload={mockMetricsRetention}
|
||||
@@ -432,12 +429,14 @@ describe('GeneralSettings - S3 Logs Retention', () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(screen.getByTestId('custom-domain-settings')).toBeInTheDocument();
|
||||
expect(
|
||||
screen.queryByTestId('custom-domain-settings'),
|
||||
).not.toBeInTheDocument();
|
||||
expect(
|
||||
screen.queryByTestId('general-settings-cloud'),
|
||||
).not.toBeInTheDocument();
|
||||
|
||||
// Save buttons should be visible for self-hosted
|
||||
// Save buttons should be visible for non-cloud users (these are from retentions)
|
||||
const saveButtons = screen.getAllByRole('button', { name: /save/i });
|
||||
expect(saveButtons.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import APIError from 'types/api/error';
|
||||
|
||||
import { errorDetails } from '../utils';
|
||||
|
||||
function makeAPIError(
|
||||
message: string,
|
||||
code = 'SOME_CODE',
|
||||
errors: { message: string }[] = [],
|
||||
): APIError {
|
||||
return new APIError({
|
||||
httpStatusCode: 500,
|
||||
error: { code, message, url: '', errors },
|
||||
});
|
||||
}
|
||||
|
||||
describe('errorDetails', () => {
|
||||
describe('when passed an APIError', () => {
|
||||
it('returns the error message', () => {
|
||||
const error = makeAPIError('something went wrong');
|
||||
expect(errorDetails(error)).toBe('something went wrong');
|
||||
});
|
||||
|
||||
it('appends details when errors array is non-empty', () => {
|
||||
const error = makeAPIError('query failed', 'QUERY_ERROR', [
|
||||
{ message: 'field X is invalid' },
|
||||
{ message: 'field Y is missing' },
|
||||
]);
|
||||
const result = errorDetails(error);
|
||||
expect(result).toContain('query failed');
|
||||
expect(result).toContain('field X is invalid');
|
||||
expect(result).toContain('field Y is missing');
|
||||
});
|
||||
|
||||
it('does not append details when errors array is empty', () => {
|
||||
const error = makeAPIError('simple error', 'CODE', []);
|
||||
const result = errorDetails(error);
|
||||
expect(result).toBe('simple error');
|
||||
expect(result).not.toContain('Details');
|
||||
});
|
||||
});
|
||||
|
||||
describe('when passed a plain Error (not an APIError)', () => {
|
||||
it('does not throw', () => {
|
||||
const error = new Error('timeout exceeded');
|
||||
expect(() => errorDetails(error)).not.toThrow();
|
||||
});
|
||||
|
||||
it('returns the plain error message', () => {
|
||||
const error = new Error('timeout exceeded');
|
||||
expect(errorDetails(error)).toBe('timeout exceeded');
|
||||
});
|
||||
|
||||
it('returns fallback when plain Error has no message', () => {
|
||||
const error = new Error('');
|
||||
expect(errorDetails(error)).toBe('Unknown error occurred');
|
||||
});
|
||||
});
|
||||
|
||||
describe('fallback behaviour', () => {
|
||||
it('returns "Unknown error occurred" when message is undefined', () => {
|
||||
const error = makeAPIError('');
|
||||
expect(errorDetails(error)).toBe('Unknown error occurred');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -249,13 +249,14 @@ export const handleGraphClick = async ({
|
||||
}
|
||||
};
|
||||
|
||||
export const errorDetails = (error: APIError): string => {
|
||||
const { message, errors } = error.getErrorDetails()?.error || {};
|
||||
export const errorDetails = (error: APIError | Error): string => {
|
||||
const { message, errors } =
|
||||
(error instanceof APIError ? error.getErrorDetails()?.error : null) || {};
|
||||
|
||||
const details =
|
||||
errors?.length > 0
|
||||
errors && errors.length > 0
|
||||
? `\n\nDetails: ${errors.map((e) => e.message).join('\n')}`
|
||||
: '';
|
||||
const errorDetails = `${message} ${details}`;
|
||||
return errorDetails || 'Unknown error occurred';
|
||||
const errorDetails = `${message ?? error.message} ${details}`;
|
||||
return errorDetails.trim() || 'Unknown error occurred';
|
||||
};
|
||||
|
||||
@@ -217,7 +217,7 @@ function K8sVolumesList({
|
||||
{
|
||||
dataSource: currentQuery.builder.queryData[0].dataSource,
|
||||
aggregateAttribute: GetK8sEntityToAggregateAttribute(
|
||||
K8sCategory.NODES,
|
||||
K8sCategory.VOLUMES,
|
||||
dotMetricsEnabled,
|
||||
),
|
||||
aggregateOperator: 'noop',
|
||||
@@ -228,7 +228,7 @@ function K8sVolumesList({
|
||||
queryKey: [currentQuery.builder.queryData[0].dataSource, 'noop'],
|
||||
},
|
||||
true,
|
||||
K8sCategory.NODES,
|
||||
K8sCategory.VOLUMES,
|
||||
);
|
||||
|
||||
const query = useMemo(() => {
|
||||
@@ -597,7 +597,7 @@ function K8sVolumesList({
|
||||
isLoadingGroupByFilters={isLoadingGroupByFilters}
|
||||
handleGroupByChange={handleGroupByChange}
|
||||
selectedGroupBy={groupBy}
|
||||
entity={K8sCategory.NODES}
|
||||
entity={K8sCategory.VOLUMES}
|
||||
showAutoRefresh={!selectedVolumeData}
|
||||
/>
|
||||
{isError && <Typography>{data?.error || 'Something went wrong'}</Typography>}
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
import setupCommonMocks from '../commonMocks';
|
||||
|
||||
setupCommonMocks();
|
||||
|
||||
import { QueryClient, QueryClientProvider } from 'react-query';
|
||||
// eslint-disable-next-line no-restricted-imports
|
||||
import { Provider } from 'react-redux';
|
||||
import { MemoryRouter } from 'react-router-dom';
|
||||
import { render, waitFor } from '@testing-library/react';
|
||||
import { FeatureKeys } from 'constants/features';
|
||||
import K8sVolumesList from 'container/InfraMonitoringK8s/Volumes/K8sVolumesList';
|
||||
import { rest, server } from 'mocks-server/server';
|
||||
import { IAppContext, IUser } from 'providers/App/types';
|
||||
import store from 'store';
|
||||
import { LicenseResModel } from 'types/api/licensesV3/getActive';
|
||||
|
||||
const queryClient = new QueryClient({
|
||||
defaultOptions: {
|
||||
queries: {
|
||||
retry: false,
|
||||
cacheTime: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const SERVER_URL = 'http://localhost/api';
|
||||
|
||||
describe('K8sVolumesList - useGetAggregateKeys Category Regression', () => {
|
||||
let requestsMade: Array<{
|
||||
url: string;
|
||||
params: URLSearchParams;
|
||||
body?: any;
|
||||
}> = [];
|
||||
|
||||
beforeEach(() => {
|
||||
requestsMade = [];
|
||||
queryClient.clear();
|
||||
|
||||
server.use(
|
||||
rest.get(`${SERVER_URL}/v3/autocomplete/attribute_keys`, (req, res, ctx) => {
|
||||
const url = req.url.toString();
|
||||
const params = req.url.searchParams;
|
||||
|
||||
requestsMade.push({
|
||||
url,
|
||||
params,
|
||||
});
|
||||
|
||||
return res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
status: 'success',
|
||||
data: {
|
||||
attributeKeys: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
}),
|
||||
rest.post(`${SERVER_URL}/v1/pvcs/list`, (req, res, ctx) =>
|
||||
res(
|
||||
ctx.status(200),
|
||||
ctx.json({
|
||||
status: 'success',
|
||||
data: {
|
||||
type: 'list',
|
||||
records: [],
|
||||
groups: null,
|
||||
total: 0,
|
||||
sentAnyHostMetricsData: false,
|
||||
isSendingK8SAgentMetrics: false,
|
||||
},
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
server.resetHandlers();
|
||||
});
|
||||
|
||||
it('should call aggregate keys API with k8s_volume_capacity', async () => {
|
||||
render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<Provider store={store}>
|
||||
<MemoryRouter>
|
||||
<K8sVolumesList
|
||||
isFiltersVisible={false}
|
||||
handleFilterVisibilityChange={jest.fn()}
|
||||
quickFiltersLastUpdated={-1}
|
||||
/>
|
||||
</MemoryRouter>
|
||||
</Provider>
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestsMade.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
// Find the attribute_keys request
|
||||
const attributeKeysRequest = requestsMade.find((req) =>
|
||||
req.url.includes('/autocomplete/attribute_keys'),
|
||||
);
|
||||
|
||||
expect(attributeKeysRequest).toBeDefined();
|
||||
|
||||
const aggregateAttribute = attributeKeysRequest?.params.get(
|
||||
'aggregateAttribute',
|
||||
);
|
||||
|
||||
expect(aggregateAttribute).toBe('k8s_volume_capacity');
|
||||
});
|
||||
|
||||
it('should call aggregate keys API with k8s.volume.capacity when dotMetrics enabled', async () => {
|
||||
jest
|
||||
.spyOn(await import('providers/App/App'), 'useAppContext')
|
||||
.mockReturnValue({
|
||||
featureFlags: [
|
||||
{
|
||||
name: FeatureKeys.DOT_METRICS_ENABLED,
|
||||
active: true,
|
||||
usage: 0,
|
||||
usage_limit: 0,
|
||||
route: '',
|
||||
},
|
||||
],
|
||||
user: { role: 'ADMIN' } as IUser,
|
||||
activeLicense: (null as unknown) as LicenseResModel,
|
||||
} as IAppContext);
|
||||
|
||||
render(
|
||||
<QueryClientProvider client={queryClient}>
|
||||
<Provider store={store}>
|
||||
<MemoryRouter>
|
||||
<K8sVolumesList
|
||||
isFiltersVisible={false}
|
||||
handleFilterVisibilityChange={jest.fn()}
|
||||
quickFiltersLastUpdated={-1}
|
||||
/>
|
||||
</MemoryRouter>
|
||||
</Provider>
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
|
||||
await waitFor(() => {
|
||||
expect(requestsMade.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
const attributeKeysRequest = requestsMade.find((req) =>
|
||||
req.url.includes('/autocomplete/attribute_keys'),
|
||||
);
|
||||
|
||||
expect(attributeKeysRequest).toBeDefined();
|
||||
|
||||
const aggregateAttribute = attributeKeysRequest?.params.get(
|
||||
'aggregateAttribute',
|
||||
);
|
||||
|
||||
expect(aggregateAttribute).toBe('k8s.volume.capacity');
|
||||
});
|
||||
});
|
||||
@@ -137,7 +137,7 @@ describe('useIsPanelWaitingOnVariable', () => {
|
||||
expect(result.current).toBe(false);
|
||||
});
|
||||
|
||||
it('should return true for DYNAMIC variable with allSelected=true that is loading', () => {
|
||||
it('should return false for DYNAMIC variable with allSelected=true that is loading but has a selectedValue', () => {
|
||||
setFetchStates({ dyn: 'loading' });
|
||||
setDashboardVariables({
|
||||
variables: {
|
||||
@@ -152,10 +152,10 @@ describe('useIsPanelWaitingOnVariable', () => {
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useIsPanelWaitingOnVariable(['dyn']));
|
||||
expect(result.current).toBe(true);
|
||||
expect(result.current).toBe(false);
|
||||
});
|
||||
|
||||
it('should return true for DYNAMIC variable with allSelected=true that is waiting', () => {
|
||||
it('should return false for DYNAMIC variable with allSelected=true that is waiting but has a selectedValue', () => {
|
||||
setFetchStates({ dyn: 'waiting' });
|
||||
setDashboardVariables({
|
||||
variables: {
|
||||
@@ -170,7 +170,7 @@ describe('useIsPanelWaitingOnVariable', () => {
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useIsPanelWaitingOnVariable(['dyn']));
|
||||
expect(result.current).toBe(true);
|
||||
expect(result.current).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false for DYNAMIC variable with allSelected=true that is idle', () => {
|
||||
@@ -313,4 +313,39 @@ describe('useIsPanelWaitingOnVariable', () => {
|
||||
const { result } = renderHook(() => useIsPanelWaitingOnVariable(['a']));
|
||||
expect(result.current).toBe(true);
|
||||
});
|
||||
|
||||
it('should find variable by name when store key differs from variable name', () => {
|
||||
setFetchStates({ myVar: 'loading' });
|
||||
setDashboardVariables({
|
||||
variables: {
|
||||
'uuid-abc-123': makeVariable({
|
||||
id: 'uuid-abc-123',
|
||||
name: 'myVar',
|
||||
selectedValue: undefined,
|
||||
}),
|
||||
},
|
||||
variableTypes: { myVar: 'QUERY' },
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useIsPanelWaitingOnVariable(['myVar']));
|
||||
expect(result.current).toBe(true);
|
||||
});
|
||||
|
||||
it('should respect selectedValue when store key differs from variable name', () => {
|
||||
// When the variable has a value, it should not block even if loading
|
||||
setFetchStates({ myVar: 'loading' });
|
||||
setDashboardVariables({
|
||||
variables: {
|
||||
'uuid-abc-123': makeVariable({
|
||||
id: 'uuid-abc-123',
|
||||
name: 'myVar',
|
||||
selectedValue: 'production',
|
||||
}),
|
||||
},
|
||||
variableTypes: { myVar: 'QUERY' },
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useIsPanelWaitingOnVariable(['myVar']));
|
||||
expect(result.current).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -226,6 +226,41 @@ describe('useVariablesFromUrl', () => {
|
||||
expect(urlVariables.undefinedVar).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should return empty object when URL param contains a bare % that makes decodeURIComponent throw', () => {
|
||||
// Simulate a URL where the variables param is a raw JSON string containing a literal %
|
||||
// (e.g. a metric value like "cpu_usage_50%"). URLSearchParams.get() returns the value
|
||||
// as-is when it was set directly; if it contains a bare %, decodeURIComponent throws a URIError.
|
||||
const rawJson = JSON.stringify({ threshold: '50%' });
|
||||
const history = createMemoryHistory({
|
||||
initialEntries: [`/?${QueryParams.variables}=${rawJson}`],
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useVariablesFromUrl(), {
|
||||
wrapper: ({ children }: { children: React.ReactNode }) => (
|
||||
<Router history={history}>{children}</Router>
|
||||
),
|
||||
});
|
||||
|
||||
// Should parse successfully via the raw fallback, not throw or return {}
|
||||
const vars = result.current.getUrlVariables();
|
||||
expect(vars).toEqual({ threshold: '50%' });
|
||||
});
|
||||
|
||||
it('should return empty object when URL param is completely unparseable', () => {
|
||||
// A value that fails both decodeURIComponent and JSON.parse
|
||||
const history = createMemoryHistory({
|
||||
initialEntries: [`/?${QueryParams.variables}=not-json-%ZZ`],
|
||||
});
|
||||
|
||||
const { result } = renderHook(() => useVariablesFromUrl(), {
|
||||
wrapper: ({ children }: { children: React.ReactNode }) => (
|
||||
<Router history={history}>{children}</Router>
|
||||
),
|
||||
});
|
||||
|
||||
expect(result.current.getUrlVariables()).toEqual({});
|
||||
});
|
||||
|
||||
it('should update variables with array values correctly', () => {
|
||||
const history = createMemoryHistory({
|
||||
initialEntries: ['/'],
|
||||
|
||||
@@ -133,21 +133,19 @@ export function useVariableFetchState(
|
||||
export function useIsPanelWaitingOnVariable(variableNames: string[]): boolean {
|
||||
const states = useVariableFetchSelector((s) => s.states);
|
||||
const dashboardVariables = useDashboardVariablesSelector((s) => s.variables);
|
||||
const variableTypesMap = useDashboardVariablesSelector((s) => s.variableTypes);
|
||||
|
||||
return variableNames.some((name) => {
|
||||
const variableFetchState = states[name];
|
||||
const { selectedValue, allSelected } = dashboardVariables?.[name] || {};
|
||||
const variableData = Object.values(dashboardVariables).find(
|
||||
(v) => v.name === name,
|
||||
);
|
||||
const { selectedValue } = variableData || {};
|
||||
|
||||
const isVariableInFetchingOrWaitingState =
|
||||
variableFetchState === 'loading' ||
|
||||
variableFetchState === 'revalidating' ||
|
||||
variableFetchState === 'waiting';
|
||||
|
||||
if (variableTypesMap[name] === 'DYNAMIC' && allSelected) {
|
||||
return isVariableInFetchingOrWaitingState;
|
||||
}
|
||||
|
||||
return isEmpty(selectedValue) ? isVariableInFetchingOrWaitingState : false;
|
||||
});
|
||||
}
|
||||
|
||||
@@ -32,7 +32,14 @@ const useVariablesFromUrl = (): UseVariablesFromUrlReturn => {
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(decodeURIComponent(variablesParam));
|
||||
const decoded = ((): string => {
|
||||
try {
|
||||
return decodeURIComponent(variablesParam);
|
||||
} catch {
|
||||
return variablesParam;
|
||||
}
|
||||
})();
|
||||
return JSON.parse(decoded);
|
||||
} catch (error) {
|
||||
Sentry.captureEvent({
|
||||
message: `Failed to parse dashboard variables from URL: ${error}`,
|
||||
|
||||
@@ -39,6 +39,7 @@ export function useCopyToClipboard(
|
||||
|
||||
const copyToClipboard = useCallback(
|
||||
(text: string, id?: ID): void => {
|
||||
// eslint-disable-next-line no-restricted-properties
|
||||
navigator.clipboard.writeText(text).then(() => {
|
||||
if (timeoutRef.current) {
|
||||
clearTimeout(timeoutRef.current);
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { useMemo } from 'react';
|
||||
import * as Sentry from '@sentry/react';
|
||||
import SeverityCriticalIcon from 'assets/AlertHistory/SeverityCriticalIcon';
|
||||
import SeverityErrorIcon from 'assets/AlertHistory/SeverityErrorIcon';
|
||||
import SeverityInfoIcon from 'assets/AlertHistory/SeverityInfoIcon';
|
||||
@@ -5,34 +7,50 @@ import SeverityWarningIcon from 'assets/AlertHistory/SeverityWarningIcon';
|
||||
|
||||
import './AlertSeverity.styles.scss';
|
||||
|
||||
const severityConfig: Record<string, Record<string, string | JSX.Element>> = {
|
||||
critical: {
|
||||
text: 'Critical',
|
||||
className: 'alert-severity--critical',
|
||||
icon: <SeverityCriticalIcon />,
|
||||
},
|
||||
error: {
|
||||
text: 'Error',
|
||||
className: 'alert-severity--error',
|
||||
icon: <SeverityErrorIcon />,
|
||||
},
|
||||
warning: {
|
||||
text: 'Warning',
|
||||
className: 'alert-severity--warning',
|
||||
icon: <SeverityWarningIcon />,
|
||||
},
|
||||
info: {
|
||||
text: 'Info',
|
||||
className: 'alert-severity--info',
|
||||
icon: <SeverityInfoIcon />,
|
||||
},
|
||||
};
|
||||
|
||||
export default function AlertSeverity({
|
||||
severity,
|
||||
}: {
|
||||
severity: string;
|
||||
}): JSX.Element {
|
||||
const severityConfig: Record<string, Record<string, string | JSX.Element>> = {
|
||||
critical: {
|
||||
text: 'Critical',
|
||||
className: 'alert-severity--critical',
|
||||
icon: <SeverityCriticalIcon />,
|
||||
},
|
||||
error: {
|
||||
text: 'Error',
|
||||
className: 'alert-severity--error',
|
||||
icon: <SeverityErrorIcon />,
|
||||
},
|
||||
warning: {
|
||||
text: 'Warning',
|
||||
className: 'alert-severity--warning',
|
||||
icon: <SeverityWarningIcon />,
|
||||
},
|
||||
info: {
|
||||
text: 'Info',
|
||||
const severityDetails = useMemo(() => {
|
||||
if (severityConfig[severity]) {
|
||||
return severityConfig[severity];
|
||||
}
|
||||
|
||||
Sentry.captureEvent({
|
||||
message: `Received unknown severity on Alert Details: ${severity}`,
|
||||
level: 'error',
|
||||
});
|
||||
|
||||
return {
|
||||
text: severity,
|
||||
className: 'alert-severity--info',
|
||||
icon: <SeverityInfoIcon />,
|
||||
},
|
||||
};
|
||||
const severityDetails = severityConfig[severity];
|
||||
};
|
||||
}, [severity]);
|
||||
return (
|
||||
<div className={`alert-severity ${severityDetails.className}`}>
|
||||
<div className="alert-severity__icon">{severityDetails.icon}</div>
|
||||
|
||||
@@ -20,7 +20,6 @@ import AlertHistory from 'container/AlertHistory';
|
||||
import { TIMELINE_TABLE_PAGE_SIZE } from 'container/AlertHistory/constants';
|
||||
import { AlertDetailsTab, TimelineFilter } from 'container/AlertHistory/types';
|
||||
import { urlKey } from 'container/AllError/utils';
|
||||
import { RelativeTimeMap } from 'container/TopNav/DateTimeSelectionV2/constants';
|
||||
import useAxiosError from 'hooks/useAxiosError';
|
||||
import { useNotifications } from 'hooks/useNotifications';
|
||||
import { useSafeNavigate } from 'hooks/useSafeNavigate';
|
||||
@@ -47,6 +46,8 @@ import { PayloadProps } from 'types/api/alerts/get';
|
||||
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
|
||||
import { nanoToMilli } from 'utils/timeUtils';
|
||||
|
||||
const DEFAULT_TIME_RANGE = '30m';
|
||||
|
||||
export const useAlertHistoryQueryParams = (): {
|
||||
ruleId: string | null;
|
||||
startTime: number;
|
||||
@@ -61,8 +62,8 @@ export const useAlertHistoryQueryParams = (): {
|
||||
const relativeTimeParam = params.get(QueryParams.relativeTime);
|
||||
|
||||
const relativeTime =
|
||||
(relativeTimeParam === 'null' ? null : relativeTimeParam) ??
|
||||
RelativeTimeMap['6hr'];
|
||||
(relativeTimeParam === 'null' ? null : relativeTimeParam) ||
|
||||
DEFAULT_TIME_RANGE;
|
||||
|
||||
const intStartTime = parseInt(startTime || '0', 10);
|
||||
const intEndTime = parseInt(endTime || '0', 10);
|
||||
|
||||
@@ -56,8 +56,8 @@ export interface AlertRuleStats {
|
||||
totalPastTriggers: number;
|
||||
currentTriggersSeries: CurrentTriggersSeries;
|
||||
pastTriggersSeries: CurrentTriggersSeries | null;
|
||||
currentAvgResolutionTime: number;
|
||||
pastAvgResolutionTime: number;
|
||||
currentAvgResolutionTime: string;
|
||||
pastAvgResolutionTime: string;
|
||||
currentAvgResolutionTimeSeries: CurrentTriggersSeries;
|
||||
pastAvgResolutionTimeSeries: any | null;
|
||||
}
|
||||
|
||||
@@ -112,6 +112,12 @@ export function formatEpochTimestamp(epoch: number): string {
|
||||
*/
|
||||
|
||||
export function formatTime(seconds: number): string {
|
||||
seconds = +seconds;
|
||||
|
||||
if (Number.isNaN(seconds)) {
|
||||
return '-';
|
||||
}
|
||||
|
||||
const days = seconds / 86400;
|
||||
|
||||
if (days >= 1) {
|
||||
|
||||
@@ -59,7 +59,7 @@ func (provider *provider) addCloudIntegrationRoutes(router *mux.Router) error {
|
||||
Description: "This endpoint gets an account for the specified cloud provider",
|
||||
Request: nil,
|
||||
RequestContentType: "",
|
||||
Response: new(citypes.GettableAccount),
|
||||
Response: new(citypes.Account),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound},
|
||||
@@ -139,7 +139,7 @@ func (provider *provider) addCloudIntegrationRoutes(router *mux.Router) error {
|
||||
Description: "This endpoint gets a service for the specified cloud provider",
|
||||
Request: nil,
|
||||
RequestContentType: "",
|
||||
Response: new(citypes.GettableService),
|
||||
Response: new(citypes.Service),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{},
|
||||
@@ -150,7 +150,7 @@ func (provider *provider) addCloudIntegrationRoutes(router *mux.Router) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/services/{service_id}", handler.New(
|
||||
if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/accounts/{id}/services/{service_id}", handler.New(
|
||||
provider.authZ.AdminAccess(provider.cloudIntegrationHandler.UpdateService),
|
||||
handler.OpenAPIDef{
|
||||
ID: "UpdateService",
|
||||
|
||||
@@ -13,16 +13,16 @@ type Module interface {
|
||||
CreateAccount(ctx context.Context, account *citypes.Account) error
|
||||
|
||||
// GetAccount returns cloud integration account
|
||||
GetAccount(ctx context.Context, orgID, accountID valuer.UUID) (*citypes.Account, error)
|
||||
GetAccount(ctx context.Context, orgID, accountID valuer.UUID, provider citypes.CloudProviderType) (*citypes.Account, error)
|
||||
|
||||
// ListAccounts lists accounts where agent is connected
|
||||
ListAccounts(ctx context.Context, orgID valuer.UUID) ([]*citypes.Account, error)
|
||||
ListAccounts(ctx context.Context, orgID valuer.UUID, provider citypes.CloudProviderType) ([]*citypes.Account, error)
|
||||
|
||||
// UpdateAccount updates the cloud integration account for a specific organization.
|
||||
UpdateAccount(ctx context.Context, account *citypes.Account) error
|
||||
|
||||
// DisconnectAccount soft deletes/removes a cloud integration account.
|
||||
DisconnectAccount(ctx context.Context, orgID, accountID valuer.UUID) error
|
||||
DisconnectAccount(ctx context.Context, orgID, accountID valuer.UUID, provider citypes.CloudProviderType) error
|
||||
|
||||
// GetConnectionArtifact returns cloud provider specific connection information,
|
||||
// client side handles how this information is shown
|
||||
@@ -30,17 +30,20 @@ type Module interface {
|
||||
|
||||
// ListServicesMetadata returns the list of services metadata for a cloud provider attached with the integrationID.
|
||||
// This just returns a summary of the service and not the whole service definition
|
||||
ListServicesMetadata(ctx context.Context, orgID valuer.UUID, integrationID *valuer.UUID) ([]*citypes.ServiceMetadata, error)
|
||||
ListServicesMetadata(ctx context.Context, orgID valuer.UUID, provider citypes.CloudProviderType, integrationID *valuer.UUID) ([]*citypes.ServiceMetadata, error)
|
||||
|
||||
// GetService returns service definition details for a serviceID. This returns config and
|
||||
// other details required to show in service details page on web client.
|
||||
GetService(ctx context.Context, orgID valuer.UUID, integrationID *valuer.UUID, serviceID string) (*citypes.Service, error)
|
||||
GetService(ctx context.Context, orgID valuer.UUID, integrationID *valuer.UUID, serviceID citypes.ServiceID, provider citypes.CloudProviderType) (*citypes.Service, error)
|
||||
|
||||
// CreateService creates a new service for a cloud integration account.
|
||||
CreateService(ctx context.Context, orgID valuer.UUID, service *citypes.CloudIntegrationService, provider citypes.CloudProviderType) error
|
||||
|
||||
// UpdateService updates cloud integration service
|
||||
UpdateService(ctx context.Context, orgID valuer.UUID, service *citypes.CloudIntegrationService) error
|
||||
UpdateService(ctx context.Context, orgID valuer.UUID, service *citypes.CloudIntegrationService, provider citypes.CloudProviderType) error
|
||||
|
||||
// AgentCheckIn is called by agent to heartbeat and get latest config in response.
|
||||
AgentCheckIn(ctx context.Context, orgID valuer.UUID, req *citypes.AgentCheckInRequest) (*citypes.AgentCheckInResponse, error)
|
||||
AgentCheckIn(ctx context.Context, orgID valuer.UUID, provider citypes.CloudProviderType, req *citypes.AgentCheckInRequest) (*citypes.AgentCheckInResponse, error)
|
||||
|
||||
// GetDashboardByID returns dashboard JSON for a given dashboard id.
|
||||
// this only returns the dashboard when the service (embedded in dashboard id) is enabled
|
||||
@@ -50,6 +53,39 @@ type Module interface {
|
||||
// ListDashboards returns list of dashboards across all connected cloud integration accounts
|
||||
// for enabled services in the org. This list gets added to dashboard list page
|
||||
ListDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error)
|
||||
|
||||
// GetCloudProvider returns cloud provider specific module
|
||||
GetCloudProvider(provider citypes.CloudProviderType) (CloudProviderModule, error)
|
||||
}
|
||||
|
||||
type CloudProviderModule interface {
|
||||
GetConnectionArtifact(ctx context.Context, creds *citypes.SignozCredentials, account *citypes.Account, req *citypes.ConnectionArtifactRequest) (*citypes.ConnectionArtifact, error)
|
||||
|
||||
// ListServiceDefinitions returns all service definitions for this cloud provider.
|
||||
ListServiceDefinitions() ([]citypes.ServiceDefinition, error)
|
||||
|
||||
// GetServiceDefinition returns the service definition for the given service ID.
|
||||
GetServiceDefinition(serviceID citypes.ServiceID) (*citypes.ServiceDefinition, error)
|
||||
|
||||
// IsServiceEnabled returns true if the service has at least one signal (logs or metrics) enabled.
|
||||
IsServiceEnabled(config *citypes.ServiceConfig) bool
|
||||
|
||||
// IsMetricsEnabled returns true if the service config has metrics explicitly enabled.
|
||||
// Used to gate dashboard availability — dashboards are only shown when metrics are enabled.
|
||||
IsMetricsEnabled(config *citypes.ServiceConfig) bool
|
||||
|
||||
// ServiceConfigFromStorableServiceConfig converts a stored service's provider-specific JSON config
|
||||
// into a *ServiceConfig wrapper.
|
||||
ServiceConfigFromStorableServiceConfig(config string) (*citypes.ServiceConfig, error)
|
||||
|
||||
// StorableConfigFromServiceConfig serializes a ServiceConfig into the raw provider-specific
|
||||
// JSON string suitable for storing in StorableCloudIntegrationService.Config.
|
||||
// It strips any signal config (logs/metrics) that the service's SupportedSignals declares unsupported.
|
||||
StorableConfigFromServiceConfig(cfg *citypes.ServiceConfig, supported citypes.SupportedSignals) (string, error)
|
||||
|
||||
// BuildIntegrationConfig compiles the provider-specific integration config from the account
|
||||
// and list of configured services. This is the config returned to the agent on check-in.
|
||||
BuildIntegrationConfig(account *citypes.Account, services []*citypes.StorableCloudIntegrationService) (*citypes.ProviderIntegrationConfig, error)
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
|
||||
@@ -1,58 +1,415 @@
|
||||
package implcloudintegration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/http/binding"
|
||||
"github.com/SigNoz/signoz/pkg/http/render"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
type handler struct{}
|
||||
|
||||
func NewHandler() cloudintegration.Handler {
|
||||
return &handler{}
|
||||
type handler struct {
|
||||
module cloudintegration.Module
|
||||
}
|
||||
|
||||
func (handler *handler) CreateAccount(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func NewHandler(module cloudintegration.Module) cloudintegration.Handler {
|
||||
return &handler{
|
||||
module: module,
|
||||
}
|
||||
}
|
||||
|
||||
func (handler *handler) ListAccounts(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) CreateAccount(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
postableConnectionArtifact := new(cloudintegrationtypes.PostableConnectionArtifact)
|
||||
|
||||
err = binding.JSON.BindBody(r.Body, postableConnectionArtifact)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
accountConfig, err := cloudintegrationtypes.NewAccountConfigFromPostableArtifact(provider, postableConnectionArtifact)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
account := cloudintegrationtypes.NewAccount(valuer.MustNewUUID(claims.OrgID), provider, accountConfig)
|
||||
err = handler.module.CreateAccount(ctx, account)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
connectionArtifactRequest, err := cloudintegrationtypes.NewArtifactRequestFromPostableArtifact(provider, postableConnectionArtifact)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
connectionArtifact, err := handler.module.GetConnectionArtifact(ctx, account, connectionArtifactRequest)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, &cloudintegrationtypes.GettableAccountWithArtifact{
|
||||
ID: account.ID,
|
||||
Artifact: connectionArtifact,
|
||||
})
|
||||
}
|
||||
|
||||
func (handler *handler) GetAccount(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) GetAccount(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
accountIDString := mux.Vars(r)["id"]
|
||||
accountID, err := valuer.NewUUID(accountIDString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
account, err := handler.module.GetAccount(ctx, valuer.MustNewUUID(claims.OrgID), accountID, provider)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, account)
|
||||
}
|
||||
|
||||
func (handler *handler) UpdateAccount(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) ListAccounts(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
accounts, err := handler.module.ListAccounts(ctx, valuer.MustNewUUID(claims.OrgID), provider)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, &cloudintegrationtypes.GettableAccounts{
|
||||
Accounts: accounts,
|
||||
})
|
||||
}
|
||||
|
||||
func (handler *handler) DisconnectAccount(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) UpdateAccount(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
id := mux.Vars(r)["id"]
|
||||
cloudIntegrationID, err := valuer.NewUUID(id)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
req := new(cloudintegrationtypes.UpdatableAccount)
|
||||
if err := binding.JSON.BindBody(r.Body, req); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
account, err := handler.module.GetAccount(ctx, valuer.MustNewUUID(claims.OrgID), cloudIntegrationID, provider)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := account.Update(req.Config); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
err = handler.module.UpdateAccount(ctx, account)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusNoContent, nil)
|
||||
}
|
||||
|
||||
func (handler *handler) ListServicesMetadata(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) DisconnectAccount(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
id := mux.Vars(r)["id"]
|
||||
cloudIntegrationID, err := valuer.NewUUID(id)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
err = handler.module.DisconnectAccount(ctx, valuer.MustNewUUID(claims.OrgID), cloudIntegrationID, provider)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusNoContent, nil)
|
||||
}
|
||||
|
||||
func (handler *handler) GetService(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) ListServicesMetadata(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
var integrationID *valuer.UUID
|
||||
if idStr := r.URL.Query().Get("cloud_integration_id"); idStr != "" {
|
||||
id, err := valuer.NewUUID(idStr)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
integrationID = &id
|
||||
}
|
||||
|
||||
services, err := handler.module.ListServicesMetadata(ctx, valuer.MustNewUUID(claims.OrgID), provider, integrationID)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, &cloudintegrationtypes.GettableServicesMetadata{
|
||||
Services: services,
|
||||
})
|
||||
}
|
||||
|
||||
func (handler *handler) UpdateService(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) GetService(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
serviceIDString := mux.Vars(r)["service_id"]
|
||||
serviceID, err := cloudintegrationtypes.NewServiceID(provider, serviceIDString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
var integrationID *valuer.UUID
|
||||
if idStr := r.URL.Query().Get("cloud_integration_id"); idStr != "" {
|
||||
id, err := valuer.NewUUID(idStr)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
integrationID = &id
|
||||
}
|
||||
|
||||
svc, err := handler.module.GetService(ctx, valuer.MustNewUUID(claims.OrgID), integrationID, serviceID, provider)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, svc)
|
||||
}
|
||||
|
||||
func (handler *handler) AgentCheckIn(writer http.ResponseWriter, request *http.Request) {
|
||||
// TODO implement me
|
||||
panic("implement me")
|
||||
func (handler *handler) UpdateService(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
serviceIDString := mux.Vars(r)["service_id"]
|
||||
serviceID, err := cloudintegrationtypes.NewServiceID(provider, serviceIDString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
req := new(cloudintegrationtypes.UpdatableService)
|
||||
if err := binding.JSON.BindBody(r.Body, req); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
cloudIntegrationID, err := valuer.NewUUID(mux.Vars(r)["id"])
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
orgID := valuer.MustNewUUID(claims.OrgID)
|
||||
|
||||
svc, err := handler.module.GetService(ctx, orgID, &cloudIntegrationID, serviceID, provider)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
if svc.CloudIntegrationService == nil {
|
||||
cloudIntegrationService := cloudintegrationtypes.NewCloudIntegrationService(serviceID, cloudIntegrationID, req.Config)
|
||||
err = handler.module.CreateService(ctx, orgID, cloudIntegrationService, provider)
|
||||
} else {
|
||||
svc.CloudIntegrationService.Update(req.Config)
|
||||
err = handler.module.UpdateService(ctx, orgID, svc.CloudIntegrationService, provider)
|
||||
}
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusNoContent, nil)
|
||||
}
|
||||
|
||||
func (handler *handler) AgentCheckIn(rw http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
providerString := mux.Vars(r)["cloud_provider"]
|
||||
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
req := new(cloudintegrationtypes.PostableAgentCheckInRequest)
|
||||
if err := binding.JSON.BindBody(r.Body, req); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
if err := req.Validate(); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Map old fields → new fields for backward compatibility with old agents
|
||||
// Old agents send account_id (=> cloudIntegrationId) and cloud_account_id (=> providerAccountId)
|
||||
if req.ID != "" {
|
||||
id, err := valuer.NewUUID(req.ID)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
req.CloudIntegrationID = id
|
||||
req.ProviderAccountID = req.AccountID
|
||||
}
|
||||
|
||||
orgID := valuer.MustNewUUID(claims.OrgID)
|
||||
resp, err := handler.module.AgentCheckIn(ctx, orgID, provider, &req.AgentCheckInRequest)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, cloudintegrationtypes.NewGettableAgentCheckInResponse(provider, resp))
|
||||
}
|
||||
|
||||
73
pkg/modules/cloudintegration/implcloudintegration/module.go
Normal file
73
pkg/modules/cloudintegration/implcloudintegration/module.go
Normal file
@@ -0,0 +1,73 @@
|
||||
package implcloudintegration
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
type module struct{}
|
||||
|
||||
func NewModule() cloudintegration.Module {
|
||||
return &module{}
|
||||
}
|
||||
|
||||
func (module *module) CreateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
|
||||
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "create account is not supported")
|
||||
}
|
||||
|
||||
func (module *module) GetAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Account, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get account is not supported")
|
||||
}
|
||||
|
||||
func (module *module) ListAccounts(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) ([]*cloudintegrationtypes.Account, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "list accounts is not supported")
|
||||
}
|
||||
|
||||
func (module *module) UpdateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
|
||||
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "update account is not supported")
|
||||
}
|
||||
|
||||
func (module *module) DisconnectAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) error {
|
||||
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "disconnect account is not supported")
|
||||
}
|
||||
|
||||
func (module *module) CreateService(ctx context.Context, orgID valuer.UUID, service *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
|
||||
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "create service is not supported")
|
||||
}
|
||||
|
||||
func (module *module) GetService(ctx context.Context, orgID valuer.UUID, integrationID *valuer.UUID, serviceID cloudintegrationtypes.ServiceID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Service, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get service is not supported")
|
||||
}
|
||||
|
||||
func (module *module) ListServicesMetadata(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, integrationID *valuer.UUID) ([]*cloudintegrationtypes.ServiceMetadata, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "list services metadata is not supported")
|
||||
}
|
||||
|
||||
func (module *module) UpdateService(ctx context.Context, orgID valuer.UUID, service *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
|
||||
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "update service is not supported")
|
||||
}
|
||||
|
||||
func (module *module) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.ConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get connection artifact is not supported")
|
||||
}
|
||||
|
||||
func (module *module) AgentCheckIn(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, req *cloudintegrationtypes.AgentCheckInRequest) (*cloudintegrationtypes.AgentCheckInResponse, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "agent check-in is not supported")
|
||||
}
|
||||
|
||||
func (module *module) GetDashboardByID(ctx context.Context, orgID valuer.UUID, id string) (*dashboardtypes.Dashboard, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get dashboard by ID is not supported")
|
||||
}
|
||||
|
||||
func (module *module) ListDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "list dashboards is not supported")
|
||||
}
|
||||
|
||||
func (module *module) GetCloudProvider(provider cloudintegrationtypes.CloudProviderType) (cloudintegration.CloudProviderModule, error) {
|
||||
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get cloud provider is not supported")
|
||||
}
|
||||
@@ -172,3 +172,9 @@ func (store *store) UpdateService(ctx context.Context, service *cloudintegration
|
||||
Exec(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (store *store) RunInTx(ctx context.Context, cb func(ctx context.Context) error) error {
|
||||
return store.store.RunInTxCtx(ctx, nil, func(ctx context.Context) error {
|
||||
return cb(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -797,17 +797,17 @@ func (m *module) buildFilterClause(ctx context.Context, filter *qbtypes.Filter,
|
||||
}
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: m.logger,
|
||||
FieldMapper: m.fieldMapper,
|
||||
ConditionBuilder: m.condBuilder,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "metric_name", FieldContext: telemetrytypes.FieldContextMetric},
|
||||
FieldKeys: keys,
|
||||
StartNs: querybuilder.ToNanoSecs(uint64(startMillis)),
|
||||
EndNs: querybuilder.ToNanoSecs(uint64(endMillis)),
|
||||
}
|
||||
|
||||
startNs := querybuilder.ToNanoSecs(uint64(startMillis))
|
||||
endNs := querybuilder.ToNanoSecs(uint64(endMillis))
|
||||
|
||||
whereClause, err := querybuilder.PrepareWhereClause(expression, opts, startNs, endNs)
|
||||
whereClause, err := querybuilder.PrepareWhereClause(expression, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -66,6 +66,7 @@ func newProvider(
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataLocalTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
|
||||
// Create trace statement builder
|
||||
|
||||
@@ -1090,7 +1090,21 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
|
||||
}
|
||||
|
||||
processingPostCache := time.Now()
|
||||
selectedSpans, uncollapsedSpans, rootServiceName, rootServiceEntryPoint := tracedetail.GetSelectedSpans(req.UncollapsedSpans, req.SelectedSpanID, traceRoots, spanIdToSpanNodeMap, req.IsSelectedSpanIDUnCollapsed)
|
||||
// When req.Limit is 0 (not set by the client), selectAllSpans is set to false
|
||||
// preserving the old paged behaviour for backward compatibility
|
||||
limit := min(req.Limit, tracedetail.MaxLimitToSelectAllSpans)
|
||||
selectAllSpans := totalSpans <= uint64(limit)
|
||||
|
||||
var (
|
||||
selectedSpans []*model.Span
|
||||
uncollapsedSpans []string
|
||||
rootServiceName, rootServiceEntryPoint string
|
||||
)
|
||||
if selectAllSpans {
|
||||
selectedSpans, rootServiceName, rootServiceEntryPoint = tracedetail.GetAllSpans(traceRoots)
|
||||
} else {
|
||||
selectedSpans, uncollapsedSpans, rootServiceName, rootServiceEntryPoint = tracedetail.GetSelectedSpans(req.UncollapsedSpans, req.SelectedSpanID, traceRoots, spanIdToSpanNodeMap, req.IsSelectedSpanIDUnCollapsed)
|
||||
}
|
||||
r.logger.Info("getWaterfallSpansForTraceWithMetadata: processing post cache", "duration", time.Since(processingPostCache), "traceID", traceID)
|
||||
|
||||
// convert start timestamp to millis because right now frontend is expecting it in millis
|
||||
@@ -1103,7 +1117,7 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
|
||||
}
|
||||
|
||||
response.Spans = selectedSpans
|
||||
response.UncollapsedSpans = uncollapsedSpans
|
||||
response.UncollapsedSpans = uncollapsedSpans // ignoring if all spans are returning
|
||||
response.StartTimestampMillis = startTime / 1000000
|
||||
response.EndTimestampMillis = endTime / 1000000
|
||||
response.TotalSpansCount = totalSpans
|
||||
@@ -1112,6 +1126,7 @@ func (r *ClickHouseReader) GetWaterfallSpansForTraceWithMetadata(ctx context.Con
|
||||
response.RootServiceEntryPoint = rootServiceEntryPoint
|
||||
response.ServiceNameToTotalDurationMap = serviceNameToTotalDurationMap
|
||||
response.HasMissingSpans = hasMissingSpans
|
||||
response.HasMore = !selectAllSpans
|
||||
return response, nil
|
||||
}
|
||||
|
||||
@@ -3371,8 +3386,8 @@ func (r *ClickHouseReader) GetMetricAggregateAttributes(ctx context.Context, org
|
||||
|
||||
// Query all relevant metric names from time_series_v4, but leave metadata retrieval to cache/db
|
||||
query := fmt.Sprintf(
|
||||
`SELECT DISTINCT metric_name
|
||||
FROM %s.%s
|
||||
`SELECT DISTINCT metric_name
|
||||
FROM %s.%s
|
||||
WHERE metric_name ILIKE $1 AND __normalized = $2`,
|
||||
signozMetricDBName, signozTSTableNameV41Day)
|
||||
|
||||
@@ -3449,8 +3464,8 @@ func (r *ClickHouseReader) GetMeterAggregateAttributes(ctx context.Context, orgI
|
||||
var response v3.AggregateAttributeResponse
|
||||
// Query all relevant metric names from time_series_v4, but leave metadata retrieval to cache/db
|
||||
query := fmt.Sprintf(
|
||||
`SELECT metric_name,type,temporality,is_monotonic
|
||||
FROM %s.%s
|
||||
`SELECT metric_name,type,temporality,is_monotonic
|
||||
FROM %s.%s
|
||||
WHERE metric_name ILIKE $1
|
||||
GROUP BY metric_name,type,temporality,is_monotonic`,
|
||||
signozMeterDBName, signozMeterSamplesName)
|
||||
@@ -5146,7 +5161,7 @@ func (r *ClickHouseReader) GetOverallStateTransitions(ctx context.Context, ruleI
|
||||
state,
|
||||
unix_milli AS firing_time
|
||||
FROM %s.%s
|
||||
WHERE overall_state = '` + model.StateFiring.String() + `'
|
||||
WHERE overall_state = '` + model.StateFiring.String() + `'
|
||||
AND overall_state_changed = true
|
||||
AND rule_id IN ('%s')
|
||||
AND unix_milli >= %d AND unix_milli <= %d
|
||||
@@ -5157,7 +5172,7 @@ resolution_events AS (
|
||||
state,
|
||||
unix_milli AS resolution_time
|
||||
FROM %s.%s
|
||||
WHERE overall_state = '` + model.StateInactive.String() + `'
|
||||
WHERE overall_state = '` + model.StateInactive.String() + `'
|
||||
AND overall_state_changed = true
|
||||
AND rule_id IN ('%s')
|
||||
AND unix_milli >= %d AND unix_milli <= %d
|
||||
@@ -5278,7 +5293,7 @@ WITH firing_events AS (
|
||||
state,
|
||||
unix_milli AS firing_time
|
||||
FROM %s.%s
|
||||
WHERE overall_state = '` + model.StateFiring.String() + `'
|
||||
WHERE overall_state = '` + model.StateFiring.String() + `'
|
||||
AND overall_state_changed = true
|
||||
AND rule_id IN ('%s')
|
||||
AND unix_milli >= %d AND unix_milli <= %d
|
||||
@@ -5289,7 +5304,7 @@ resolution_events AS (
|
||||
state,
|
||||
unix_milli AS resolution_time
|
||||
FROM %s.%s
|
||||
WHERE overall_state = '` + model.StateInactive.String() + `'
|
||||
WHERE overall_state = '` + model.StateInactive.String() + `'
|
||||
AND overall_state_changed = true
|
||||
AND rule_id IN ('%s')
|
||||
AND unix_milli >= %d AND unix_milli <= %d
|
||||
@@ -5335,7 +5350,7 @@ WITH firing_events AS (
|
||||
state,
|
||||
unix_milli AS firing_time
|
||||
FROM %s.%s
|
||||
WHERE overall_state = '` + model.StateFiring.String() + `'
|
||||
WHERE overall_state = '` + model.StateFiring.String() + `'
|
||||
AND overall_state_changed = true
|
||||
AND rule_id IN ('%s')
|
||||
AND unix_milli >= %d AND unix_milli <= %d
|
||||
@@ -5346,7 +5361,7 @@ resolution_events AS (
|
||||
state,
|
||||
unix_milli AS resolution_time
|
||||
FROM %s.%s
|
||||
WHERE overall_state = '` + model.StateInactive.String() + `'
|
||||
WHERE overall_state = '` + model.StateInactive.String() + `'
|
||||
AND overall_state_changed = true
|
||||
AND rule_id IN ('%s')
|
||||
AND unix_milli >= %d AND unix_milli <= %d
|
||||
@@ -5608,7 +5623,7 @@ func (r *ClickHouseReader) GetMetricsDataPoints(ctx context.Context, metricName
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
instrumentationtypes.CodeFunctionName: "GetMetricsDataPoints",
|
||||
})
|
||||
query := fmt.Sprintf(`SELECT
|
||||
query := fmt.Sprintf(`SELECT
|
||||
sum(count) as data_points
|
||||
FROM %s.%s
|
||||
WHERE metric_name = ?
|
||||
@@ -5628,7 +5643,7 @@ func (r *ClickHouseReader) GetMetricsLastReceived(ctx context.Context, metricNam
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
instrumentationtypes.CodeFunctionName: "GetMetricsLastReceived",
|
||||
})
|
||||
query := fmt.Sprintf(`SELECT
|
||||
query := fmt.Sprintf(`SELECT
|
||||
MAX(unix_milli) AS last_received_time
|
||||
FROM %s.%s
|
||||
WHERE metric_name = ?
|
||||
@@ -5639,7 +5654,7 @@ WHERE metric_name = ?
|
||||
if err != nil {
|
||||
return 0, &model.ApiError{Typ: "ClickHouseError", Err: err}
|
||||
}
|
||||
query = fmt.Sprintf(`SELECT
|
||||
query = fmt.Sprintf(`SELECT
|
||||
MAX(unix_milli) AS last_received_time
|
||||
FROM %s.%s
|
||||
WHERE metric_name = ? and unix_milli > ?
|
||||
@@ -5658,7 +5673,7 @@ func (r *ClickHouseReader) GetTotalTimeSeriesForMetricName(ctx context.Context,
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
instrumentationtypes.CodeFunctionName: "GetTotalTimeSeriesForMetricName",
|
||||
})
|
||||
query := fmt.Sprintf(`SELECT
|
||||
query := fmt.Sprintf(`SELECT
|
||||
uniq(fingerprint) AS timeSeriesCount
|
||||
FROM %s.%s
|
||||
WHERE metric_name = ?;`, signozMetricDBName, signozTSTableNameV41Week)
|
||||
@@ -5690,7 +5705,7 @@ func (r *ClickHouseReader) GetAttributesForMetricName(ctx context.Context, metri
|
||||
}
|
||||
|
||||
const baseQueryTemplate = `
|
||||
SELECT
|
||||
SELECT
|
||||
kv.1 AS key,
|
||||
arrayMap(x -> trim(BOTH '"' FROM x), groupUniqArray(1000)(kv.2)) AS values,
|
||||
length(groupUniqArray(10000)(kv.2)) AS valueCount
|
||||
@@ -5799,7 +5814,7 @@ func (r *ClickHouseReader) ListSummaryMetrics(ctx context.Context, orgID valuer.
|
||||
sampleTable, countExp := utils.WhichSampleTableToUse(req.Start, req.End)
|
||||
|
||||
metricsQuery := fmt.Sprintf(
|
||||
`SELECT
|
||||
`SELECT
|
||||
t.metric_name AS metric_name,
|
||||
ANY_VALUE(t.description) AS description,
|
||||
ANY_VALUE(t.type) AS metric_type,
|
||||
@@ -5864,11 +5879,11 @@ func (r *ClickHouseReader) ListSummaryMetrics(ctx context.Context, orgID valuer.
|
||||
|
||||
if whereClause != "" {
|
||||
sb.WriteString(fmt.Sprintf(
|
||||
`SELECT
|
||||
`SELECT
|
||||
s.samples,
|
||||
s.metric_name
|
||||
FROM (
|
||||
SELECT
|
||||
SELECT
|
||||
dm.metric_name,
|
||||
%s AS samples
|
||||
FROM %s.%s AS dm
|
||||
@@ -5898,11 +5913,11 @@ func (r *ClickHouseReader) ListSummaryMetrics(ctx context.Context, orgID valuer.
|
||||
} else {
|
||||
// If no filters, it is a simpler query.
|
||||
sb.WriteString(fmt.Sprintf(
|
||||
`SELECT
|
||||
`SELECT
|
||||
s.samples,
|
||||
s.metric_name
|
||||
FROM (
|
||||
SELECT
|
||||
SELECT
|
||||
metric_name,
|
||||
%s AS samples
|
||||
FROM %s.%s
|
||||
@@ -6007,16 +6022,16 @@ func (r *ClickHouseReader) GetMetricsTimeSeriesPercentage(ctx context.Context, r
|
||||
|
||||
// Construct the query without backticks
|
||||
query := fmt.Sprintf(`
|
||||
SELECT
|
||||
SELECT
|
||||
metric_name,
|
||||
total_value,
|
||||
(total_value * 100.0 / total_time_series) AS percentage
|
||||
FROM (
|
||||
SELECT
|
||||
SELECT
|
||||
metric_name,
|
||||
uniq(fingerprint) AS total_value,
|
||||
(SELECT uniq(fingerprint)
|
||||
FROM %s.%s
|
||||
(SELECT uniq(fingerprint)
|
||||
FROM %s.%s
|
||||
WHERE unix_milli BETWEEN ? AND ? AND __normalized = ?) AS total_time_series
|
||||
FROM %s.%s
|
||||
WHERE unix_milli BETWEEN ? AND ? AND NOT startsWith(metric_name, 'signoz') AND __normalized = ? %s
|
||||
@@ -6092,7 +6107,7 @@ func (r *ClickHouseReader) GetMetricsSamplesPercentage(ctx context.Context, req
|
||||
|
||||
queryLimit := 50 + req.Limit
|
||||
metricsQuery := fmt.Sprintf(
|
||||
`SELECT
|
||||
`SELECT
|
||||
ts.metric_name AS metric_name,
|
||||
uniq(ts.fingerprint) AS timeSeries
|
||||
FROM %s.%s AS ts
|
||||
@@ -6149,13 +6164,13 @@ func (r *ClickHouseReader) GetMetricsSamplesPercentage(ctx context.Context, req
|
||||
FROM %s.%s
|
||||
WHERE unix_milli BETWEEN ? AND ?
|
||||
)
|
||||
SELECT
|
||||
SELECT
|
||||
s.samples,
|
||||
s.metric_name,
|
||||
COALESCE((s.samples * 100.0 / t.total_samples), 0) AS percentage
|
||||
FROM
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
SELECT
|
||||
dm.metric_name,
|
||||
%s AS samples
|
||||
FROM %s.%s AS dm`,
|
||||
@@ -6176,7 +6191,7 @@ func (r *ClickHouseReader) GetMetricsSamplesPercentage(ctx context.Context, req
|
||||
if whereClause != "" {
|
||||
sb.WriteString(fmt.Sprintf(
|
||||
` AND dm.fingerprint IN (
|
||||
SELECT ts.fingerprint
|
||||
SELECT ts.fingerprint
|
||||
FROM %s.%s AS ts
|
||||
WHERE ts.metric_name IN (%s)
|
||||
AND unix_milli BETWEEN ? AND ?
|
||||
@@ -6247,7 +6262,7 @@ func (r *ClickHouseReader) GetNameSimilarity(ctx context.Context, req *metrics_e
|
||||
}
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
SELECT
|
||||
SELECT
|
||||
metric_name,
|
||||
any(type) as type,
|
||||
any(temporality) as temporality,
|
||||
@@ -6306,7 +6321,7 @@ func (r *ClickHouseReader) GetAttributeSimilarity(ctx context.Context, req *metr
|
||||
|
||||
// Get target labels
|
||||
extractedLabelsQuery := fmt.Sprintf(`
|
||||
SELECT
|
||||
SELECT
|
||||
kv.1 AS label_key,
|
||||
topK(10)(JSONExtractString(kv.2)) AS label_values
|
||||
FROM %s.%s
|
||||
@@ -6350,12 +6365,12 @@ func (r *ClickHouseReader) GetAttributeSimilarity(ctx context.Context, req *metr
|
||||
priorityListString := strings.Join(priorityList, ", ")
|
||||
|
||||
candidateLabelsQuery := fmt.Sprintf(`
|
||||
WITH
|
||||
arrayDistinct([%s]) AS filter_keys,
|
||||
WITH
|
||||
arrayDistinct([%s]) AS filter_keys,
|
||||
arrayDistinct([%s]) AS filter_values,
|
||||
[%s] AS priority_pairs_input,
|
||||
%d AS priority_multiplier
|
||||
SELECT
|
||||
SELECT
|
||||
metric_name,
|
||||
any(type) as type,
|
||||
any(temporality) as temporality,
|
||||
@@ -6461,17 +6476,17 @@ func (r *ClickHouseReader) GetMetricsAllResourceAttributes(ctx context.Context,
|
||||
instrumentationtypes.CodeFunctionName: "GetMetricsAllResourceAttributes",
|
||||
})
|
||||
start, end, attTable, _ := utils.WhichAttributesTableToUse(start, end)
|
||||
query := fmt.Sprintf(`SELECT
|
||||
key,
|
||||
query := fmt.Sprintf(`SELECT
|
||||
key,
|
||||
count(distinct value) AS distinct_value_count
|
||||
FROM (
|
||||
SELECT key, value
|
||||
FROM %s.%s
|
||||
ARRAY JOIN
|
||||
ARRAY JOIN
|
||||
arrayConcat(mapKeys(resource_attributes)) AS key,
|
||||
arrayConcat(mapValues(resource_attributes)) AS value
|
||||
WHERE unix_milli between ? and ?
|
||||
)
|
||||
)
|
||||
GROUP BY key
|
||||
ORDER BY distinct_value_count DESC;`, signozMetadataDbName, attTable)
|
||||
valueCtx := context.WithValue(ctx, "clickhouse_max_threads", constants.MetricsExplorerClickhouseThreads)
|
||||
@@ -6618,11 +6633,11 @@ func (r *ClickHouseReader) GetInspectMetricsFingerprints(ctx context.Context, at
|
||||
|
||||
start, end, tsTable, _ := utils.WhichTSTableToUse(req.Start, req.End)
|
||||
query := fmt.Sprintf(`
|
||||
SELECT
|
||||
SELECT
|
||||
arrayDistinct(groupArray(toString(fingerprint))) AS fingerprints
|
||||
FROM
|
||||
(
|
||||
SELECT
|
||||
SELECT
|
||||
metric_name, labels, fingerprint,
|
||||
%s
|
||||
FROM %s.%s
|
||||
@@ -6793,14 +6808,14 @@ func (r *ClickHouseReader) GetUpdatedMetricsMetadata(ctx context.Context, orgID
|
||||
var stillMissing []string
|
||||
if len(missingMetrics) > 0 {
|
||||
metricList := "'" + strings.Join(missingMetrics, "', '") + "'"
|
||||
query := fmt.Sprintf(`SELECT
|
||||
query := fmt.Sprintf(`SELECT
|
||||
metric_name,
|
||||
argMax(type, created_at) AS type,
|
||||
argMax(description, created_at) AS description,
|
||||
argMax(temporality, created_at) AS temporality,
|
||||
argMax(is_monotonic, created_at) AS is_monotonic,
|
||||
argMax(unit, created_at) AS unit
|
||||
FROM %s.%s
|
||||
FROM %s.%s
|
||||
WHERE metric_name IN (%s)
|
||||
GROUP BY metric_name;`,
|
||||
signozMetricDBName,
|
||||
@@ -6848,7 +6863,7 @@ func (r *ClickHouseReader) GetUpdatedMetricsMetadata(ctx context.Context, orgID
|
||||
if len(stillMissing) > 0 {
|
||||
metricList := "'" + strings.Join(stillMissing, "', '") + "'"
|
||||
query := fmt.Sprintf(`SELECT DISTINCT metric_name, type, description, temporality, is_monotonic, unit
|
||||
FROM %s.%s
|
||||
FROM %s.%s
|
||||
WHERE metric_name IN (%s)`, signozMetricDBName, signozTSTableNameV4, metricList)
|
||||
valueCtx := context.WithValue(ctx, "clickhouse_max_threads", constants.MetricsExplorerClickhouseThreads)
|
||||
rows, err := r.db.Query(valueCtx, query)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package tracedetail
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
|
||||
@@ -9,6 +10,9 @@ import (
|
||||
|
||||
var (
|
||||
SPAN_LIMIT_PER_REQUEST_FOR_WATERFALL float64 = 500
|
||||
|
||||
maxDepthForSelectedSpanChildren int = 5
|
||||
MaxLimitToSelectAllSpans uint = 10_000
|
||||
)
|
||||
|
||||
type Interval struct {
|
||||
@@ -63,26 +67,22 @@ func findIndexForSelectedSpanFromPreOrder(spans []*model.Span, selectedSpanId st
|
||||
return selectedSpanIndex
|
||||
}
|
||||
|
||||
func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string, uncollapsedSpans []string, isSelectedSpanIDUnCollapsed bool) (bool, []string) {
|
||||
func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string) (bool, []string) {
|
||||
spansFromRootToNode := []string{}
|
||||
|
||||
spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
|
||||
if node.SpanID == selectedSpanId {
|
||||
if isSelectedSpanIDUnCollapsed && !slices.Contains(uncollapsedSpans, node.SpanID) {
|
||||
spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
|
||||
}
|
||||
return true, spansFromRootToNode
|
||||
}
|
||||
|
||||
isPresentInSubtreeForTheNode := false
|
||||
for _, child := range node.Children {
|
||||
isPresentInThisSubtree, _spansFromRootToNode := getPathFromRootToSelectedSpanId(child, selectedSpanId, uncollapsedSpans, isSelectedSpanIDUnCollapsed)
|
||||
isPresentInThisSubtree, _spansFromRootToNode := getPathFromRootToSelectedSpanId(child, selectedSpanId)
|
||||
// if the interested node is present in the given subtree then add the span node to uncollapsed node list
|
||||
if isPresentInThisSubtree {
|
||||
if !slices.Contains(uncollapsedSpans, node.SpanID) {
|
||||
spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
|
||||
}
|
||||
isPresentInSubtreeForTheNode = true
|
||||
spansFromRootToNode = append(spansFromRootToNode, _spansFromRootToNode...)
|
||||
break
|
||||
}
|
||||
}
|
||||
return isPresentInSubtreeForTheNode, spansFromRootToNode
|
||||
@@ -92,12 +92,23 @@ func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string, un
|
||||
// throughout the recursion. Per-call state (level, isPartOfPreOrder, etc.)
|
||||
// is passed as direct arguments.
|
||||
type traverseOpts struct {
|
||||
uncollapsedSpans []string
|
||||
selectedSpanID string
|
||||
uncollapsedSpans map[string]struct{}
|
||||
selectedSpanID string
|
||||
isSelectedSpanUncollapsed bool
|
||||
selectAll bool
|
||||
}
|
||||
|
||||
func traverseTrace(span *model.Span, opts traverseOpts, level uint64, isPartOfPreOrder bool, hasSibling bool) []*model.Span {
|
||||
func traverseTrace(
|
||||
span *model.Span,
|
||||
opts traverseOpts,
|
||||
level uint64,
|
||||
isPartOfPreOrder bool,
|
||||
hasSibling bool,
|
||||
autoExpandDepth int,
|
||||
) ([]*model.Span, []string) {
|
||||
|
||||
preOrderTraversal := []*model.Span{}
|
||||
autoExpandedSpans := []string{}
|
||||
|
||||
// sort the children to maintain the order across requests
|
||||
sort.Slice(span.Children, func(i, j int) bool {
|
||||
@@ -134,16 +145,36 @@ func traverseTrace(span *model.Span, opts traverseOpts, level uint64, isPartOfPr
|
||||
preOrderTraversal = append(preOrderTraversal, &nodeWithoutChildren)
|
||||
}
|
||||
|
||||
isAlreadyUncollapsed := slices.Contains(opts.uncollapsedSpans, span.SpanID)
|
||||
remainingAutoExpandDepth := 0
|
||||
if span.SpanID == opts.selectedSpanID && opts.isSelectedSpanUncollapsed {
|
||||
remainingAutoExpandDepth = maxDepthForSelectedSpanChildren
|
||||
} else if autoExpandDepth > 0 {
|
||||
remainingAutoExpandDepth = autoExpandDepth - 1
|
||||
}
|
||||
|
||||
_, isAlreadyUncollapsed := opts.uncollapsedSpans[span.SpanID]
|
||||
for index, child := range span.Children {
|
||||
_childTraversal := traverseTrace(child, opts, level+1, isPartOfPreOrder && isAlreadyUncollapsed, index != (len(span.Children)-1))
|
||||
// A child is included in the pre-order output if its parent is uncollapsed
|
||||
// OR if the child falls within MAX_DEPTH_FOR_SELECTED_SPAN_CHILDREN levels
|
||||
// below the selected span.
|
||||
isChildWithinMaxDepth := remainingAutoExpandDepth > 0
|
||||
childIsPartOfPreOrder := opts.selectAll || (isPartOfPreOrder && (isAlreadyUncollapsed || isChildWithinMaxDepth))
|
||||
|
||||
if isPartOfPreOrder && isChildWithinMaxDepth && !isAlreadyUncollapsed {
|
||||
if !slices.Contains(autoExpandedSpans, span.SpanID) {
|
||||
autoExpandedSpans = append(autoExpandedSpans, span.SpanID)
|
||||
}
|
||||
}
|
||||
|
||||
_childTraversal, _autoExpanded := traverseTrace(child, opts, level+1, childIsPartOfPreOrder, index != (len(span.Children)-1), remainingAutoExpandDepth)
|
||||
preOrderTraversal = append(preOrderTraversal, _childTraversal...)
|
||||
autoExpandedSpans = append(autoExpandedSpans, _autoExpanded...)
|
||||
nodeWithoutChildren.SubTreeNodeCount += child.SubTreeNodeCount + 1
|
||||
span.SubTreeNodeCount += child.SubTreeNodeCount + 1
|
||||
}
|
||||
|
||||
nodeWithoutChildren.SubTreeNodeCount += 1
|
||||
return preOrderTraversal
|
||||
return preOrderTraversal, autoExpandedSpans
|
||||
|
||||
}
|
||||
|
||||
@@ -169,19 +200,36 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
|
||||
|
||||
var preOrderTraversal = make([]*model.Span, 0)
|
||||
var rootServiceName, rootServiceEntryPoint string
|
||||
updatedUncollapsedSpans := uncollapsedSpans
|
||||
|
||||
// create a map of uncollapsed spans for quick lookup
|
||||
uncollapsedSpanMap := make(map[string]struct{})
|
||||
for _, spanID := range uncollapsedSpans {
|
||||
uncollapsedSpanMap[spanID] = struct{}{}
|
||||
}
|
||||
|
||||
selectedSpanIndex := -1
|
||||
for _, rootSpanID := range traceRoots {
|
||||
if rootNode, exists := spanIdToSpanNodeMap[rootSpanID.SpanID]; exists {
|
||||
_, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID, updatedUncollapsedSpans, isSelectedSpanIDUnCollapsed)
|
||||
updatedUncollapsedSpans = append(updatedUncollapsedSpans, spansFromRootToNode...)
|
||||
present, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID)
|
||||
if present {
|
||||
for _, spanID := range spansFromRootToNode {
|
||||
if selectedSpanID == spanID && !isSelectedSpanIDUnCollapsed {
|
||||
continue
|
||||
}
|
||||
uncollapsedSpanMap[spanID] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
opts := traverseOpts{
|
||||
uncollapsedSpans: updatedUncollapsedSpans,
|
||||
selectedSpanID: selectedSpanID,
|
||||
uncollapsedSpans: uncollapsedSpanMap,
|
||||
selectedSpanID: selectedSpanID,
|
||||
isSelectedSpanUncollapsed: isSelectedSpanIDUnCollapsed,
|
||||
}
|
||||
_preOrderTraversal, _autoExpanded := traverseTrace(rootNode, opts, 0, true, false, 0)
|
||||
// Merge auto-expanded spans into updatedUncollapsedSpans for returning in response
|
||||
for _, spanID := range _autoExpanded {
|
||||
uncollapsedSpanMap[spanID] = struct{}{}
|
||||
}
|
||||
_preOrderTraversal := traverseTrace(rootNode, opts, 0, true, false)
|
||||
_selectedSpanIndex := findIndexForSelectedSpanFromPreOrder(_preOrderTraversal, selectedSpanID)
|
||||
|
||||
if _selectedSpanIndex != -1 {
|
||||
@@ -223,5 +271,17 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
|
||||
startIndex = 0
|
||||
}
|
||||
|
||||
return preOrderTraversal[startIndex:endIndex], updatedUncollapsedSpans, rootServiceName, rootServiceEntryPoint
|
||||
return preOrderTraversal[startIndex:endIndex], slices.Collect(maps.Keys(uncollapsedSpanMap)), rootServiceName, rootServiceEntryPoint
|
||||
}
|
||||
|
||||
func GetAllSpans(traceRoots []*model.Span) (spans []*model.Span, rootServiceName, rootEntryPoint string) {
|
||||
if len(traceRoots) > 0 {
|
||||
rootServiceName = traceRoots[0].ServiceName
|
||||
rootEntryPoint = traceRoots[0].Name
|
||||
}
|
||||
for _, root := range traceRoots {
|
||||
childSpans, _ := traverseTrace(root, traverseOpts{selectAll: true}, 0, true, false, 0)
|
||||
spans = append(spans, childSpans...)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -81,28 +81,6 @@ func TestGetSelectedSpans_MultipleRoots(t *testing.T) {
|
||||
assert.Equal(t, "root1-op", entryPoint, "metadata comes from first root")
|
||||
}
|
||||
|
||||
// isSelectedSpanIDUnCollapsed=true opens only the selected span's direct children,
|
||||
// not deeper descendants.
|
||||
//
|
||||
// root → selected (expanded)
|
||||
// ├─ child1 ✓
|
||||
// │ └─ grandchild ✗ (only one level opened)
|
||||
// └─ child2 ✓
|
||||
func TestGetSelectedSpans_ExpandedSelectedSpan(t *testing.T) {
|
||||
root := mkSpan("root", "svc",
|
||||
mkSpan("selected", "svc",
|
||||
mkSpan("child1", "svc", mkSpan("grandchild", "svc")),
|
||||
mkSpan("child2", "svc"),
|
||||
),
|
||||
)
|
||||
spanMap := buildSpanMap(root)
|
||||
spans, _, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, true)
|
||||
|
||||
// root and selected are on the auto-uncollapsed path; child1/child2 are direct
|
||||
// children of the expanded selected span; grandchild stays hidden.
|
||||
assert.Equal(t, []string{"root", "selected", "child1", "child2"}, spanIDs(spans))
|
||||
}
|
||||
|
||||
// Multiple spans uncollapsed simultaneously: children of all uncollapsed spans
|
||||
// are visible at once.
|
||||
//
|
||||
@@ -183,7 +161,7 @@ func TestGetSelectedSpans_PathReturnedInUncollapsed(t *testing.T) {
|
||||
spanMap := buildSpanMap(root)
|
||||
spans, uncollapsed, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, false)
|
||||
|
||||
assert.Equal(t, []string{"root", "parent"}, uncollapsed)
|
||||
assert.ElementsMatch(t, []string{"root", "parent"}, uncollapsed)
|
||||
assert.Equal(t, []string{"root", "parent", "selected"}, spanIDs(spans))
|
||||
}
|
||||
|
||||
@@ -206,7 +184,7 @@ func TestGetSelectedSpans_SiblingsNotExpanded(t *testing.T) {
|
||||
// children of root sort alphabetically: parent < unrelated; unrelated-child stays hidden
|
||||
assert.Equal(t, []string{"root", "parent", "selected", "unrelated"}, spanIDs(spans))
|
||||
// only the path nodes are tracked as uncollapsed — unrelated is not
|
||||
assert.Equal(t, []string{"root", "parent"}, uncollapsed)
|
||||
assert.ElementsMatch(t, []string{"root", "parent"}, uncollapsed)
|
||||
}
|
||||
|
||||
// An unknown selectedSpanID must not panic; returns a window from index 0.
|
||||
@@ -320,6 +298,119 @@ func TestGetSelectedSpans_WindowShiftsAtStart(t *testing.T) {
|
||||
assert.Equal(t, "span10", spans[10].SpanID, "selected span still in window")
|
||||
}
|
||||
|
||||
// Auto-expanded span IDs from ALL branches are returned in
|
||||
// updatedUncollapsedSpans. Only internal nodes (spans with children) are
|
||||
// tracked — leaf spans are never added.
|
||||
//
|
||||
// root (selected)
|
||||
// ├─ childA (internal ✓)
|
||||
// │ └─ grandchildA (internal ✓)
|
||||
// │ └─ leafA (leaf ✗)
|
||||
// └─ childB (internal ✓)
|
||||
// └─ grandchildB (internal ✓)
|
||||
// └─ leafB (leaf ✗)
|
||||
func TestGetSelectedSpans_AutoExpandedSpansReturnedInUncollapsed(t *testing.T) {
|
||||
root := mkSpan("root", "svc",
|
||||
mkSpan("childA", "svc",
|
||||
mkSpan("grandchildA", "svc",
|
||||
mkSpan("leafA", "svc"),
|
||||
),
|
||||
),
|
||||
mkSpan("childB", "svc",
|
||||
mkSpan("grandchildB", "svc",
|
||||
mkSpan("leafB", "svc"),
|
||||
),
|
||||
),
|
||||
)
|
||||
spanMap := buildSpanMap(root)
|
||||
_, uncollapsed, _, _ := GetSelectedSpans([]string{}, "root", []*model.Span{root}, spanMap, true)
|
||||
|
||||
// all internal nodes across both branches must be tracked
|
||||
assert.Contains(t, uncollapsed, "root")
|
||||
assert.Contains(t, uncollapsed, "childA", "internal node depth 1, branch A")
|
||||
assert.Contains(t, uncollapsed, "childB", "internal node depth 1, branch B")
|
||||
assert.Contains(t, uncollapsed, "grandchildA", "internal node depth 2, branch A")
|
||||
assert.Contains(t, uncollapsed, "grandchildB", "internal node depth 2, branch B")
|
||||
// leaves have no children to show — never added to uncollapsedSpans
|
||||
assert.NotContains(t, uncollapsed, "leafA", "leaf spans are never added to uncollapsedSpans")
|
||||
assert.NotContains(t, uncollapsed, "leafB", "leaf spans are never added to uncollapsedSpans")
|
||||
}
|
||||
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// maxDepthForSelectedSpanChildren boundary tests
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
// Depth is measured from the selected span, not the trace root.
|
||||
// Ancestors appear via the path-to-root logic, not the depth limit.
|
||||
// Each depth level has two children to confirm the limit is enforced on all
|
||||
// branches, not just the first.
|
||||
//
|
||||
// root
|
||||
// └─ A ancestor ✓ (path-to-root)
|
||||
// └─ selected
|
||||
// ├─ d1a depth 1 ✓
|
||||
// │ ├─ d2a depth 2 ✓
|
||||
// │ │ ├─ d3a depth 3 ✓
|
||||
// │ │ │ ├─ d4a depth 4 ✓
|
||||
// │ │ │ │ ├─ d5a depth 5 ✓
|
||||
// │ │ │ │ │ └─ d6a depth 6 ✗
|
||||
// │ │ │ │ └─ d5b depth 5 ✓
|
||||
// │ │ │ └─ d4b depth 4 ✓
|
||||
// │ │ └─ d3b depth 3 ✓
|
||||
// │ └─ d2b depth 2 ✓
|
||||
// └─ d1b depth 1 ✓
|
||||
func TestGetSelectedSpans_DepthCountedFromSelectedSpan(t *testing.T) {
|
||||
selected := mkSpan("selected", "svc",
|
||||
mkSpan("d1a", "svc",
|
||||
mkSpan("d2a", "svc",
|
||||
mkSpan("d3a", "svc",
|
||||
mkSpan("d4a", "svc",
|
||||
mkSpan("d5a", "svc",
|
||||
mkSpan("d6a", "svc"), // depth 6 — excluded
|
||||
),
|
||||
mkSpan("d5b", "svc"), // depth 5 — included
|
||||
),
|
||||
mkSpan("d4b", "svc"), // depth 4 — included
|
||||
),
|
||||
mkSpan("d3b", "svc"), // depth 3 — included
|
||||
),
|
||||
mkSpan("d2b", "svc"), // depth 2 — included
|
||||
),
|
||||
mkSpan("d1b", "svc"), // depth 1 — included
|
||||
)
|
||||
root := mkSpan("root", "svc", mkSpan("A", "svc", selected))
|
||||
|
||||
spanMap := buildSpanMap(root)
|
||||
spans, _, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, true)
|
||||
ids := spanIDs(spans)
|
||||
|
||||
assert.Contains(t, ids, "root", "ancestor shown via path-to-root")
|
||||
assert.Contains(t, ids, "A", "ancestor shown via path-to-root")
|
||||
for _, id := range []string{"d1a", "d1b", "d2a", "d2b", "d3a", "d3b", "d4a", "d4b", "d5a", "d5b"} {
|
||||
assert.Contains(t, ids, id, "depth ≤ 5 — must be included")
|
||||
}
|
||||
assert.NotContains(t, ids, "d6a", "depth 6 > limit — excluded")
|
||||
}
|
||||
|
||||
func TestGetAllSpans(t *testing.T) {
|
||||
root := mkSpan("root", "svc",
|
||||
mkSpan("childA", "svc",
|
||||
mkSpan("grandchildA", "svc",
|
||||
mkSpan("leafA", "svc2"),
|
||||
),
|
||||
),
|
||||
mkSpan("childB", "svc3",
|
||||
mkSpan("grandchildB", "svc",
|
||||
mkSpan("leafB", "svc2"),
|
||||
),
|
||||
),
|
||||
)
|
||||
spans, rootServiceName, rootEntryPoint := GetAllSpans([]*model.Span{root})
|
||||
assert.ElementsMatch(t, spanIDs(spans), []string{"root", "childA", "grandchildA", "leafA", "childB", "grandchildB", "leafB"})
|
||||
assert.Equal(t, rootServiceName, "svc")
|
||||
assert.Equal(t, rootEntryPoint, "root-op")
|
||||
}
|
||||
|
||||
func mkSpan(id, service string, children ...*model.Span) *model.Span {
|
||||
return &model.Span{
|
||||
SpanID: id,
|
||||
|
||||
@@ -333,6 +333,7 @@ type GetWaterfallSpansForTraceWithMetadataParams struct {
|
||||
SelectedSpanID string `json:"selectedSpanId"`
|
||||
IsSelectedSpanIDUnCollapsed bool `json:"isSelectedSpanIDUnCollapsed"`
|
||||
UncollapsedSpans []string `json:"uncollapsedSpans"`
|
||||
Limit uint `json:"limit"`
|
||||
}
|
||||
|
||||
type GetFlamegraphSpansForTraceParams struct {
|
||||
|
||||
@@ -329,6 +329,7 @@ type GetWaterfallSpansForTraceWithMetadataResponse struct {
|
||||
HasMissingSpans bool `json:"hasMissingSpans"`
|
||||
// this is needed for frontend and query service sync
|
||||
UncollapsedSpans []string `json:"uncollapsedSpans"`
|
||||
HasMore bool `json:"hasMore"`
|
||||
}
|
||||
|
||||
type GetFlamegraphSpansForTraceResponse struct {
|
||||
|
||||
@@ -48,6 +48,8 @@ func NewAggExprRewriter(
|
||||
// and the args if the parametric aggregation function is used.
|
||||
func (r *aggExprRewriter) Rewrite(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
expr string,
|
||||
rateInterval uint64,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
@@ -74,7 +76,12 @@ func (r *aggExprRewriter) Rewrite(
|
||||
return "", nil, errors.NewInternalf(errors.CodeInternal, "no SELECT items for %q", expr)
|
||||
}
|
||||
|
||||
visitor := newExprVisitor(r.logger, keys,
|
||||
visitor := newExprVisitor(
|
||||
ctx,
|
||||
startNs,
|
||||
endNs,
|
||||
r.logger,
|
||||
keys,
|
||||
r.fullTextColumn,
|
||||
r.fieldMapper,
|
||||
r.conditionBuilder,
|
||||
@@ -94,6 +101,8 @@ func (r *aggExprRewriter) Rewrite(
|
||||
// RewriteMulti rewrites a slice of expressions.
|
||||
func (r *aggExprRewriter) RewriteMulti(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
exprs []string,
|
||||
rateInterval uint64,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
@@ -102,7 +111,7 @@ func (r *aggExprRewriter) RewriteMulti(
|
||||
var errs []error
|
||||
var chArgsList [][]any
|
||||
for i, e := range exprs {
|
||||
w, chArgs, err := r.Rewrite(ctx, e, rateInterval, keys)
|
||||
w, chArgs, err := r.Rewrite(ctx, startNs, endNs, e, rateInterval, keys)
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
out[i] = e
|
||||
@@ -119,6 +128,9 @@ func (r *aggExprRewriter) RewriteMulti(
|
||||
|
||||
// exprVisitor walks FunctionExpr nodes and applies the mappers.
|
||||
type exprVisitor struct {
|
||||
ctx context.Context
|
||||
startNs uint64
|
||||
endNs uint64
|
||||
chparser.DefaultASTVisitor
|
||||
logger *slog.Logger
|
||||
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey
|
||||
@@ -132,6 +144,9 @@ type exprVisitor struct {
|
||||
}
|
||||
|
||||
func newExprVisitor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
logger *slog.Logger,
|
||||
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey,
|
||||
@@ -140,6 +155,9 @@ func newExprVisitor(
|
||||
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
|
||||
) *exprVisitor {
|
||||
return &exprVisitor{
|
||||
ctx: ctx,
|
||||
startNs: startNs,
|
||||
endNs: endNs,
|
||||
logger: logger,
|
||||
fieldKeys: fieldKeys,
|
||||
fullTextColumn: fullTextColumn,
|
||||
@@ -186,13 +204,16 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
whereClause, err := PrepareWhereClause(
|
||||
origPred,
|
||||
FilterExprVisitorOpts{
|
||||
Context: v.ctx,
|
||||
Logger: v.logger,
|
||||
FieldKeys: v.fieldKeys,
|
||||
FieldMapper: v.fieldMapper,
|
||||
ConditionBuilder: v.conditionBuilder,
|
||||
FullTextColumn: v.fullTextColumn,
|
||||
JsonKeyToKey: v.jsonKeyToKey,
|
||||
}, 0, 0,
|
||||
StartNs: v.startNs,
|
||||
EndNs: v.endNs,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -212,7 +233,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
for i := 0; i < len(args)-1; i++ {
|
||||
origVal := args[i].String()
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(origVal)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(v.ctx, v.startNs, v.endNs, &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to get table field name for %q", origVal)
|
||||
}
|
||||
@@ -230,7 +251,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
|
||||
for i, arg := range args {
|
||||
orig := arg.String()
|
||||
fieldKey := telemetrytypes.GetFieldKeyFromKeyText(orig)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
expr, exprArgs, err := CollisionHandledFinalExpr(v.ctx, v.startNs, v.endNs, &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -147,12 +147,7 @@ func AdjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemet
|
||||
// So we can safely override the context and data type
|
||||
|
||||
actions = append(actions, fmt.Sprintf("Overriding key: %s to %s", key, intrinsicOrCalculatedField))
|
||||
key.FieldContext = intrinsicOrCalculatedField.FieldContext
|
||||
key.FieldDataType = intrinsicOrCalculatedField.FieldDataType
|
||||
key.JSONDataType = intrinsicOrCalculatedField.JSONDataType
|
||||
key.Indexes = intrinsicOrCalculatedField.Indexes
|
||||
key.Materialized = intrinsicOrCalculatedField.Materialized
|
||||
key.JSONPlan = intrinsicOrCalculatedField.JSONPlan
|
||||
key.OverrideMetadataFrom(intrinsicOrCalculatedField)
|
||||
return actions
|
||||
|
||||
}
|
||||
@@ -198,13 +193,9 @@ func AdjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemet
|
||||
if !key.Equal(matchingKey) {
|
||||
actions = append(actions, fmt.Sprintf("Adjusting key %s to %s", key, matchingKey))
|
||||
}
|
||||
|
||||
key.Name = matchingKey.Name
|
||||
key.FieldContext = matchingKey.FieldContext
|
||||
key.FieldDataType = matchingKey.FieldDataType
|
||||
key.JSONDataType = matchingKey.JSONDataType
|
||||
key.Indexes = matchingKey.Indexes
|
||||
key.Materialized = matchingKey.Materialized
|
||||
key.JSONPlan = matchingKey.JSONPlan
|
||||
key.OverrideMetadataFrom(matchingKey)
|
||||
|
||||
return actions
|
||||
} else {
|
||||
|
||||
@@ -19,6 +19,8 @@ import (
|
||||
|
||||
func CollisionHandledFinalExpr(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
fm qbtypes.FieldMapper,
|
||||
cb qbtypes.ConditionBuilder,
|
||||
@@ -44,7 +46,7 @@ func CollisionHandledFinalExpr(
|
||||
|
||||
addCondition := func(key *telemetrytypes.TelemetryFieldKey) error {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
condition, err := cb.ConditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb, 0, 0)
|
||||
condition, err := cb.ConditionFor(ctx, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -57,7 +59,7 @@ func CollisionHandledFinalExpr(
|
||||
return nil
|
||||
}
|
||||
|
||||
colName, fieldForErr := fm.FieldFor(ctx, field)
|
||||
fieldExpression, fieldForErr := fm.FieldFor(ctx, startNs, endNs, field)
|
||||
if errors.Is(fieldForErr, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -92,9 +94,9 @@ func CollisionHandledFinalExpr(
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
colName, _ = fm.FieldFor(ctx, key)
|
||||
colName, _ = DataTypeCollisionHandledFieldName(key, dummyValue, colName, qbtypes.FilterOperatorUnknown)
|
||||
stmts = append(stmts, colName)
|
||||
fieldExpression, _ = fm.FieldFor(ctx, startNs, endNs, key)
|
||||
fieldExpression, _ = DataTypeCollisionHandledFieldName(key, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown)
|
||||
stmts = append(stmts, fieldExpression)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -109,10 +111,10 @@ func CollisionHandledFinalExpr(
|
||||
} else if strings.Contains(field.Name, telemetrytypes.ArraySep) || strings.Contains(field.Name, telemetrytypes.ArrayAnyIndex) {
|
||||
return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the Array Paths: %s", field.Name)
|
||||
} else {
|
||||
colName, _ = DataTypeCollisionHandledFieldName(field, dummyValue, colName, qbtypes.FilterOperatorUnknown)
|
||||
fieldExpression, _ = DataTypeCollisionHandledFieldName(field, dummyValue, fieldExpression, qbtypes.FilterOperatorUnknown)
|
||||
}
|
||||
|
||||
stmts = append(stmts, colName)
|
||||
stmts = append(stmts, fieldExpression)
|
||||
}
|
||||
|
||||
for idx := range stmts {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -44,12 +45,12 @@ func keyIndexFilter(key *telemetrytypes.TelemetryFieldKey) any {
|
||||
|
||||
func (b *defaultConditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
op qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource {
|
||||
@@ -60,15 +61,23 @@ func (b *defaultConditionBuilder) ConditionFor(
|
||||
// as we store resource values as string
|
||||
formattedValue := querybuilder.FormatValueForContains(value)
|
||||
|
||||
column, err := b.fm.ColumnFor(ctx, key)
|
||||
columns, err := b.fm.ColumnFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(columns) != 1 {
|
||||
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
|
||||
}
|
||||
|
||||
// resource evolution on main table doesn't affect this
|
||||
// as we have not changed the resource column in the resource fingerprint table.
|
||||
column := columns[0]
|
||||
|
||||
keyIdxFilter := sb.Like(column.Name, keyIndexFilter(key))
|
||||
valueForIndexFilter := valueForIndexFilter(op, key, value)
|
||||
|
||||
fieldName, err := b.fm.FieldFor(ctx, key)
|
||||
fieldName, err := b.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -206,7 +206,7 @@ func TestConditionBuilder(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(context.Background(), tc.key, tc.op, tc.value, sb, 0, 0)
|
||||
cond, err := conditionBuilder.ConditionFor(context.Background(), 0, 0, tc.key, tc.op, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedErr != nil {
|
||||
|
||||
@@ -27,46 +27,50 @@ func NewFieldMapper() *defaultFieldMapper {
|
||||
|
||||
func (m *defaultFieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (*schema.Column, error) {
|
||||
) ([]*schema.Column, error) {
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
return resourceColumns["labels"], nil
|
||||
return []*schema.Column{resourceColumns["labels"]}, nil
|
||||
}
|
||||
if col, ok := resourceColumns[key.Name]; ok {
|
||||
return col, nil
|
||||
return []*schema.Column{col}, nil
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) ColumnFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) FieldFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", column.Name, key.Name), nil
|
||||
return fmt.Sprintf("simpleJSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
}
|
||||
return column.Name, nil
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
_ map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
colName, err := m.FieldFor(ctx, key)
|
||||
fieldExpression, err := m.FieldFor(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return fmt.Sprintf("%s AS `%s`", colName, key.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", fieldExpression, key.Name), nil
|
||||
}
|
||||
|
||||
@@ -148,7 +148,7 @@ func (b *resourceFilterStatementBuilder[T]) Build(
|
||||
|
||||
// addConditions adds both filter and time conditions to the query
|
||||
func (b *resourceFilterStatementBuilder[T]) addConditions(
|
||||
_ context.Context,
|
||||
ctx context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[T],
|
||||
@@ -160,6 +160,7 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
|
||||
|
||||
// warnings would be encountered as part of the main condition already
|
||||
filterWhereClause, err := querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fieldMapper,
|
||||
ConditionBuilder: b.conditionBuilder,
|
||||
@@ -171,7 +172,9 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
|
||||
// there is no need for "key" not found error for resource filtering
|
||||
IgnoreNotFoundKeys: true,
|
||||
Variables: variables,
|
||||
}, start, end)
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -23,6 +23,7 @@ const stringMatchingOperatorDocURL = "https://signoz.io/docs/userguide/operators
|
||||
// filterExpressionVisitor implements the FilterQueryVisitor interface
|
||||
// to convert the parsed filter expressions into ClickHouse WHERE clause
|
||||
type filterExpressionVisitor struct {
|
||||
context context.Context
|
||||
logger *slog.Logger
|
||||
fieldMapper qbtypes.FieldMapper
|
||||
conditionBuilder qbtypes.ConditionBuilder
|
||||
@@ -46,6 +47,7 @@ type filterExpressionVisitor struct {
|
||||
}
|
||||
|
||||
type FilterExprVisitorOpts struct {
|
||||
Context context.Context
|
||||
Logger *slog.Logger
|
||||
FieldMapper qbtypes.FieldMapper
|
||||
ConditionBuilder qbtypes.ConditionBuilder
|
||||
@@ -65,6 +67,7 @@ type FilterExprVisitorOpts struct {
|
||||
// newFilterExpressionVisitor creates a new filterExpressionVisitor
|
||||
func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVisitor {
|
||||
return &filterExpressionVisitor{
|
||||
context: opts.Context,
|
||||
logger: opts.Logger,
|
||||
fieldMapper: opts.FieldMapper,
|
||||
conditionBuilder: opts.ConditionBuilder,
|
||||
@@ -90,7 +93,7 @@ type PreparedWhereClause struct {
|
||||
}
|
||||
|
||||
// PrepareWhereClause generates a ClickHouse compatible WHERE clause from the filter query
|
||||
func PrepareWhereClause(query string, opts FilterExprVisitorOpts, startNs uint64, endNs uint64) (*PreparedWhereClause, error) {
|
||||
func PrepareWhereClause(query string, opts FilterExprVisitorOpts) (*PreparedWhereClause, error) {
|
||||
|
||||
// Setup the ANTLR parsing pipeline
|
||||
input := antlr.NewInputStream(query)
|
||||
@@ -124,8 +127,6 @@ func PrepareWhereClause(query string, opts FilterExprVisitorOpts, startNs uint64
|
||||
}
|
||||
tokens.Reset()
|
||||
|
||||
opts.StartNs = startNs
|
||||
opts.EndNs = endNs
|
||||
visitor := newFilterExpressionVisitor(opts)
|
||||
|
||||
// Handle syntax errors
|
||||
@@ -331,7 +332,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
return ""
|
||||
}
|
||||
}
|
||||
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText), v.builder, v.startNs, v.endNs)
|
||||
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.startNs, v.endNs, v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(searchText), v.builder)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -374,7 +375,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
}
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, nil, v.builder, v.startNs, v.endNs)
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, nil, v.builder)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -447,7 +448,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
}
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, values, v.builder, v.startNs, v.endNs)
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, values, v.builder)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -495,7 +496,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, []any{value1, value2}, v.builder, v.startNs, v.endNs)
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, []any{value1, value2}, v.builder)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
@@ -580,7 +581,7 @@ func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext
|
||||
|
||||
var conds []string
|
||||
for _, key := range keys {
|
||||
condition, err := v.conditionBuilder.ConditionFor(context.Background(), key, op, value, v.builder, v.startNs, v.endNs)
|
||||
condition, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, key, op, value, v.builder)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -658,7 +659,7 @@ func (v *filterExpressionVisitor) VisitFullText(ctx *grammar.FullTextContext) an
|
||||
v.errors = append(v.errors, "full text search is not supported")
|
||||
return ""
|
||||
}
|
||||
cond, err := v.conditionBuilder.ConditionFor(context.Background(), v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder, v.startNs, v.endNs)
|
||||
cond, err := v.conditionBuilder.ConditionFor(v.context, v.startNs, v.endNs, v.fullTextColumn, qbtypes.FilterOperatorRegexp, FormatFullTextSearch(text), v.builder)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to build full text search condition: %s", err.Error()))
|
||||
return ""
|
||||
@@ -744,13 +745,13 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
|
||||
if key.FieldContext == telemetrytypes.FieldContextBody {
|
||||
var err error
|
||||
if BodyJSONQueryEnabled {
|
||||
fieldName, err = v.fieldMapper.FieldFor(context.Background(), key)
|
||||
fieldName, err = v.fieldMapper.FieldFor(v.context, v.startNs, v.endNs, key)
|
||||
if err != nil {
|
||||
v.errors = append(v.errors, fmt.Sprintf("failed to get field name for key %s: %s", key.Name, err.Error()))
|
||||
return ""
|
||||
}
|
||||
} else {
|
||||
fieldName, _ = v.jsonKeyToKey(context.Background(), key, qbtypes.FilterOperatorUnknown, value)
|
||||
fieldName, _ = v.jsonKeyToKey(v.context, key, qbtypes.FilterOperatorUnknown, value)
|
||||
}
|
||||
} else {
|
||||
// TODO(add docs for json body search)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package querybuilder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -54,11 +55,12 @@ func TestPrepareWhereClause_EmptyVariableList(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
opts := FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
FieldKeys: keys,
|
||||
Variables: tt.variables,
|
||||
}
|
||||
|
||||
_, err := PrepareWhereClause(tt.expr, opts, 0, 0)
|
||||
_, err := PrepareWhereClause(tt.expr, opts)
|
||||
|
||||
if tt.expectError {
|
||||
if err == nil {
|
||||
@@ -467,7 +469,7 @@ func TestVisitKey(t *testing.T) {
|
||||
expectedWarnings: nil,
|
||||
expectedMainWrnURL: "",
|
||||
},
|
||||
{
|
||||
{
|
||||
name: "only attribute.custom_field is selected",
|
||||
keyText: "attribute.attribute.custom_field",
|
||||
fieldKeys: map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
|
||||
@@ -94,6 +94,6 @@ func NewHandlers(
|
||||
QuerierHandler: querierHandler,
|
||||
ServiceAccountHandler: implserviceaccount.NewHandler(modules.ServiceAccount),
|
||||
RegistryHandler: registryHandler,
|
||||
CloudIntegrationHandler: implcloudintegration.NewHandler(),
|
||||
CloudIntegrationHandler: implcloudintegration.NewHandler(modules.CloudIntegration),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/quickfilter/implquickfilter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
@@ -52,8 +54,11 @@ func TestNewHandlers(t *testing.T) {
|
||||
userRoleStore := impluser.NewUserRoleStore(sqlstore, providerSettings)
|
||||
|
||||
userGetter := impluser.NewGetter(impluser.NewStore(sqlstore, providerSettings), userRoleStore, flagger)
|
||||
quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore))
|
||||
orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter)
|
||||
userSetter := impluser.NewSetter(impluser.NewStore(sqlstore, providerSettings), tokenizer, emailing, providerSettings, orgSetter, nil, nil, user.Config{}, userRoleStore, userGetter)
|
||||
|
||||
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore)
|
||||
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, userSetter, orgSetter, quickfilter, nil)
|
||||
|
||||
querierHandler := querier.NewHandler(providerSettings, nil, nil)
|
||||
registryHandler := factory.NewHandler(nil)
|
||||
|
||||
@@ -12,17 +12,16 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/apdex/implapdex"
|
||||
"github.com/SigNoz/signoz/pkg/modules/authdomain"
|
||||
"github.com/SigNoz/signoz/pkg/modules/authdomain/implauthdomain"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
|
||||
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer/implmetricsexplorer"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/preference"
|
||||
"github.com/SigNoz/signoz/pkg/modules/preference/implpreference"
|
||||
"github.com/SigNoz/signoz/pkg/modules/promote"
|
||||
"github.com/SigNoz/signoz/pkg/modules/promote/implpromote"
|
||||
"github.com/SigNoz/signoz/pkg/modules/quickfilter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/quickfilter/implquickfilter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rawdataexport/implrawdataexport"
|
||||
"github.com/SigNoz/signoz/pkg/modules/savedview"
|
||||
@@ -38,7 +37,6 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/tracefunnel"
|
||||
"github.com/SigNoz/signoz/pkg/modules/tracefunnel/impltracefunnel"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
"github.com/SigNoz/signoz/pkg/ruler/rulestore/sqlrulestore"
|
||||
@@ -51,24 +49,25 @@ import (
|
||||
)
|
||||
|
||||
type Modules struct {
|
||||
OrgGetter organization.Getter
|
||||
OrgSetter organization.Setter
|
||||
Preference preference.Module
|
||||
UserSetter user.Setter
|
||||
UserGetter user.Getter
|
||||
SavedView savedview.Module
|
||||
Apdex apdex.Module
|
||||
Dashboard dashboard.Module
|
||||
QuickFilter quickfilter.Module
|
||||
TraceFunnel tracefunnel.Module
|
||||
RawDataExport rawdataexport.Module
|
||||
AuthDomain authdomain.Module
|
||||
Session session.Module
|
||||
Services services.Module
|
||||
SpanPercentile spanpercentile.Module
|
||||
MetricsExplorer metricsexplorer.Module
|
||||
Promote promote.Module
|
||||
ServiceAccount serviceaccount.Module
|
||||
OrgGetter organization.Getter
|
||||
OrgSetter organization.Setter
|
||||
Preference preference.Module
|
||||
UserSetter user.Setter
|
||||
UserGetter user.Getter
|
||||
SavedView savedview.Module
|
||||
Apdex apdex.Module
|
||||
Dashboard dashboard.Module
|
||||
QuickFilter quickfilter.Module
|
||||
TraceFunnel tracefunnel.Module
|
||||
RawDataExport rawdataexport.Module
|
||||
AuthDomain authdomain.Module
|
||||
Session session.Module
|
||||
Services services.Module
|
||||
SpanPercentile spanpercentile.Module
|
||||
MetricsExplorer metricsexplorer.Module
|
||||
Promote promote.Module
|
||||
ServiceAccount serviceaccount.Module
|
||||
CloudIntegration cloudintegration.Module
|
||||
}
|
||||
|
||||
func NewModules(
|
||||
@@ -90,30 +89,32 @@ func NewModules(
|
||||
dashboard dashboard.Module,
|
||||
userGetter user.Getter,
|
||||
userRoleStore authtypes.UserRoleStore,
|
||||
userSetter user.Setter,
|
||||
orgSetter organization.Setter,
|
||||
quickfilter quickfilter.Module,
|
||||
cloudIntegrationModule cloudintegration.Module,
|
||||
) Modules {
|
||||
quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore))
|
||||
orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter)
|
||||
userSetter := impluser.NewSetter(impluser.NewStore(sqlstore, providerSettings), tokenizer, emailing, providerSettings, orgSetter, authz, analytics, config.User, userRoleStore, userGetter)
|
||||
ruleStore := sqlrulestore.NewRuleStore(sqlstore, queryParser, providerSettings)
|
||||
|
||||
return Modules{
|
||||
OrgGetter: orgGetter,
|
||||
OrgSetter: orgSetter,
|
||||
Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewAvailablePreference()),
|
||||
SavedView: implsavedview.NewModule(sqlstore),
|
||||
Apdex: implapdex.NewModule(sqlstore),
|
||||
Dashboard: dashboard,
|
||||
UserSetter: userSetter,
|
||||
UserGetter: userGetter,
|
||||
QuickFilter: quickfilter,
|
||||
TraceFunnel: impltracefunnel.NewModule(impltracefunnel.NewStore(sqlstore)),
|
||||
RawDataExport: implrawdataexport.NewModule(querier),
|
||||
AuthDomain: implauthdomain.NewModule(implauthdomain.NewStore(sqlstore), authNs),
|
||||
Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, implauthdomain.NewModule(implauthdomain.NewStore(sqlstore), authNs), tokenizer, orgGetter),
|
||||
SpanPercentile: implspanpercentile.NewModule(querier, providerSettings),
|
||||
Services: implservices.NewModule(querier, telemetryStore),
|
||||
MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, providerSettings, config.MetricsExplorer),
|
||||
Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore),
|
||||
ServiceAccount: implserviceaccount.NewModule(implserviceaccount.NewStore(sqlstore), authz, emailing, providerSettings),
|
||||
OrgGetter: orgGetter,
|
||||
OrgSetter: orgSetter,
|
||||
Preference: implpreference.NewModule(implpreference.NewStore(sqlstore), preferencetypes.NewAvailablePreference()),
|
||||
SavedView: implsavedview.NewModule(sqlstore),
|
||||
Apdex: implapdex.NewModule(sqlstore),
|
||||
Dashboard: dashboard,
|
||||
UserSetter: userSetter,
|
||||
UserGetter: userGetter,
|
||||
QuickFilter: quickfilter,
|
||||
TraceFunnel: impltracefunnel.NewModule(impltracefunnel.NewStore(sqlstore)),
|
||||
RawDataExport: implrawdataexport.NewModule(querier),
|
||||
AuthDomain: implauthdomain.NewModule(implauthdomain.NewStore(sqlstore), authNs),
|
||||
Session: implsession.NewModule(providerSettings, authNs, userSetter, userGetter, implauthdomain.NewModule(implauthdomain.NewStore(sqlstore), authNs), tokenizer, orgGetter),
|
||||
SpanPercentile: implspanpercentile.NewModule(querier, providerSettings),
|
||||
Services: implservices.NewModule(querier, telemetryStore),
|
||||
MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, providerSettings, config.MetricsExplorer),
|
||||
Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore),
|
||||
ServiceAccount: implserviceaccount.NewModule(implserviceaccount.NewStore(sqlstore), authz, emailing, providerSettings),
|
||||
CloudIntegration: cloudIntegrationModule,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,8 +13,11 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory/factorytest"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/quickfilter/implquickfilter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
"github.com/SigNoz/signoz/pkg/sharder"
|
||||
@@ -51,7 +54,11 @@ func TestNewModules(t *testing.T) {
|
||||
|
||||
userGetter := impluser.NewGetter(impluser.NewStore(sqlstore, providerSettings), userRoleStore, flagger)
|
||||
|
||||
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore)
|
||||
quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore))
|
||||
orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter)
|
||||
userSetter := impluser.NewSetter(impluser.NewStore(sqlstore, providerSettings), tokenizer, emailing, providerSettings, orgSetter, nil, nil, user.Config{}, userRoleStore, userGetter)
|
||||
|
||||
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, userSetter, orgSetter, quickfilter, implcloudintegration.NewModule())
|
||||
|
||||
reflectVal := reflect.ValueOf(modules)
|
||||
for i := 0; i < reflectVal.NumField(); i++ {
|
||||
|
||||
@@ -20,9 +20,13 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/identn"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
pkgimplcloudintegration "github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/quickfilter/implquickfilter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
@@ -41,6 +45,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
pkgtokenizer "github.com/SigNoz/signoz/pkg/tokenizer"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
"github.com/SigNoz/signoz/pkg/zeus"
|
||||
@@ -94,6 +99,7 @@ func New(
|
||||
dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing) dashboard.Module,
|
||||
gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config],
|
||||
querierHandlerCallback func(factory.ProviderSettings, querier.Querier, analytics.Analytics) querier.Handler,
|
||||
cloudIntegrationCallback func(cloudintegrationtypes.Store, zeus.Zeus, gateway.Gateway, licensing.Licensing, user.Getter, user.Setter) (cloudintegration.Module, error),
|
||||
) (*SigNoz, error) {
|
||||
// Initialize instrumentation
|
||||
instrumentation, err := instrumentation.New(ctx, config.Instrumentation, version.Info, "signoz")
|
||||
@@ -394,6 +400,7 @@ func New(
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
telemetrymetadata.DBName,
|
||||
telemetrymetadata.AttributesMetadataLocalTableName,
|
||||
telemetrymetadata.ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
|
||||
global, err := factory.NewProviderFromNamedMap(
|
||||
@@ -407,8 +414,18 @@ func New(
|
||||
return nil, err
|
||||
}
|
||||
|
||||
quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore))
|
||||
orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter)
|
||||
userSetter := impluser.NewSetter(impluser.NewStore(sqlstore, providerSettings), tokenizer, emailing, providerSettings, orgSetter, authz, analytics, config.User, userRoleStore, userGetter)
|
||||
|
||||
cloudIntegrationStore := pkgimplcloudintegration.NewStore(sqlstore)
|
||||
cloudIntegrationModule, err := cloudIntegrationCallback(cloudIntegrationStore, zeus, gateway, licensing, userGetter, userSetter)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize all modules
|
||||
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter, userRoleStore)
|
||||
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter, userRoleStore, userSetter, orgSetter, quickfilter, cloudIntegrationModule)
|
||||
|
||||
// Initialize identN resolver
|
||||
identNFactories := NewIdentNProviderFactories(sqlstore, tokenizer, orgGetter, userGetter, config.User)
|
||||
|
||||
@@ -23,56 +23,61 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) (string, error) {
|
||||
column, err := c.fm.ColumnFor(ctx, key)
|
||||
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled && key.Name != messageSubField {
|
||||
valueType, value := InferDataType(value, operator, key)
|
||||
cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
|
||||
for _, column := range columns {
|
||||
if column.Type.GetType() == schema.ColumnTypeEnumJSON && querybuilder.BodyJSONQueryEnabled && key.Name != messageSubField {
|
||||
valueType, value := InferDataType(value, operator, key)
|
||||
cond, err := NewJSONConditionBuilder(key, valueType).buildJSONCondition(operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return cond, nil
|
||||
}
|
||||
return cond, nil
|
||||
|
||||
}
|
||||
|
||||
if operator.IsStringSearchOperator() {
|
||||
value = querybuilder.FormatValueForContains(value)
|
||||
}
|
||||
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Check if this is a body JSON search - either by FieldContext
|
||||
if key.FieldContext == telemetrytypes.FieldContextBody && !querybuilder.BodyJSONQueryEnabled {
|
||||
tblFieldName, value = GetBodyJSONKey(ctx, key, operator, value)
|
||||
fieldExpression, value = GetBodyJSONKey(ctx, key, operator, value)
|
||||
}
|
||||
|
||||
tblFieldName, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, tblFieldName, operator)
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
|
||||
// make use of case insensitive index for body
|
||||
if tblFieldName == "body" || tblFieldName == messageSubColumn {
|
||||
if fieldExpression == "body" || fieldExpression == messageSubColumn {
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(LOWER(%s), LOWER(%s))`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -80,40 +85,41 @@ func (c *conditionBuilder) conditionFor(
|
||||
switch operator {
|
||||
// regular operators
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.G(tblFieldName, value), nil
|
||||
return sb.G(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThanOrEq:
|
||||
return sb.GE(tblFieldName, value), nil
|
||||
return sb.GE(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorLessThan:
|
||||
return sb.LT(tblFieldName, value), nil
|
||||
return sb.LT(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorLessThanOrEq:
|
||||
return sb.LE(tblFieldName, value), nil
|
||||
return sb.LE(fieldExpression, value), nil
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.Like(tblFieldName, value), nil
|
||||
return sb.Like(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotLike(tblFieldName, value), nil
|
||||
return sb.NotLike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorILike:
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
// between and not between
|
||||
case qbtypes.FilterOperatorBetween:
|
||||
values, ok := value.([]any)
|
||||
@@ -123,7 +129,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.Between(tblFieldName, values[0], values[1]), nil
|
||||
return sb.Between(fieldExpression, values[0], values[1]), nil
|
||||
case qbtypes.FilterOperatorNotBetween:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -132,7 +138,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.NotBetween(tblFieldName, values[0], values[1]), nil
|
||||
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -143,7 +149,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(tblFieldName, value))
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -154,7 +160,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(tblFieldName, value))
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
@@ -171,36 +177,61 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
|
||||
var value any
|
||||
column := columns[0]
|
||||
if len(key.Evolutions) > 0 {
|
||||
// we will use the corresponding column and its evolution entry for the query
|
||||
newColumns, _, err := selectEvolutionsForColumns(columns, key.Evolutions, startNs, endNs)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if len(newColumns) == 0 {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "no valid evolution found for field %s in the given time range", key.Name)
|
||||
}
|
||||
|
||||
// This mean tblFieldName is with multiIf, we just need to do a null check.
|
||||
if len(newColumns) > 1 {
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.IsNotNull(fieldExpression), nil
|
||||
} else {
|
||||
return sb.IsNull(fieldExpression), nil
|
||||
}
|
||||
}
|
||||
|
||||
// otherwise we have to find the correct exist operator based on the column type
|
||||
column = newColumns[0]
|
||||
}
|
||||
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.IsNotNull(tblFieldName), nil
|
||||
return sb.IsNotNull(fieldExpression), nil
|
||||
}
|
||||
return sb.IsNull(tblFieldName), nil
|
||||
return sb.IsNull(fieldExpression), nil
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
}
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
case schema.ColumnTypeEnumString:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
} else {
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
|
||||
value = 0
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
} else {
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
@@ -224,6 +255,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
|
||||
|
||||
}
|
||||
}
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
|
||||
@@ -231,14 +263,15 @@ func (c *conditionBuilder) conditionFor(
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
condition, err := c.conditionFor(ctx, key, operator, value, sb)
|
||||
|
||||
condition, err := c.conditionFor(ctx, startNs, endNs, key, operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -261,7 +294,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
|
||||
if buildExistCondition {
|
||||
existsCondition, err := c.conditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
existsCondition, err := c.conditionFor(ctx, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package telemetrylogs
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -11,14 +12,148 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExistsConditionForWithEvolutions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
startTs uint64
|
||||
endTs uint64
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
operator qbtypes.FilterOperator
|
||||
value any
|
||||
expectedSQL string
|
||||
expectedArgs []any
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "New column",
|
||||
startTs: uint64(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
endTs: uint64(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE resource.`service.name`::String IS NOT NULL",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Old column",
|
||||
startTs: uint64(time.Date(2023, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
endTs: uint64(time.Date(2023, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE mapContains(resources_string, 'service.name') = ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Both Old column and new - empty filter",
|
||||
startTs: uint64(time.Date(2023, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
endTs: uint64(time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC).UnixNano()),
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
},
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL",
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
ctx := context.Background()
|
||||
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, tc.startTs, tc.endTs, &tc.key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
sql, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, sql, tc.expectedSQL)
|
||||
assert.Equal(t, tc.expectedArgs, args)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionFor(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
mockEvolution := mockEvolutionData(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC))
|
||||
testCases := []struct {
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
operator qbtypes.FilterOperator
|
||||
value any
|
||||
evolutions []*telemetrytypes.EvolutionEntry
|
||||
expectedSQL string
|
||||
expectedArgs []any
|
||||
expectedError error
|
||||
@@ -241,9 +376,11 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL",
|
||||
expectedSQL: "mapContains(resources_string, 'service.name') = ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -253,9 +390,11 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorNotExists,
|
||||
value: nil,
|
||||
expectedSQL: "WHERE multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NULL",
|
||||
expectedSQL: "mapContains(resources_string, 'service.name') <> ?",
|
||||
expectedArgs: []any{true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -316,10 +455,11 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorRegexp,
|
||||
value: "frontend-.*",
|
||||
expectedSQL: "(match(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL), ?) AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL) IS NOT NULL)",
|
||||
expectedArgs: []any{"frontend-.*"},
|
||||
expectedSQL: "WHERE (match(`resource_string_service$$name`, ?) AND `resource_string_service$$name_exists` = ?)",
|
||||
expectedArgs: []any{"frontend-.*", true},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -330,9 +470,10 @@ func TestConditionFor(t *testing.T) {
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
},
|
||||
evolutions: mockEvolution,
|
||||
operator: qbtypes.FilterOperatorNotRegexp,
|
||||
value: "test-.*",
|
||||
expectedSQL: "WHERE NOT match(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL), ?)",
|
||||
expectedSQL: "WHERE NOT match(`resource_string_service$$name`, ?)",
|
||||
expectedArgs: []any{"test-.*"},
|
||||
expectedError: nil,
|
||||
},
|
||||
@@ -372,14 +513,13 @@ func TestConditionFor(t *testing.T) {
|
||||
expectedError: qbtypes.ErrColumnNotFound,
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
tc.key.Evolutions = tc.evolutions
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
@@ -434,7 +574,7 @@ func TestConditionForMultipleKeys(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var err error
|
||||
for _, key := range tc.keys {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &key, tc.operator, tc.value, sb, 0, 0)
|
||||
cond, err := conditionBuilder.conditionFor(ctx, 0, 0, &key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting condition for key %s: %v", key.Name, err)
|
||||
@@ -691,7 +831,7 @@ func TestConditionForJSONBodySearch(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
cond, err := conditionBuilder.conditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
|
||||
@@ -3,7 +3,11 @@ package telemetrylogs
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz-otel-collector/utils"
|
||||
@@ -67,40 +71,42 @@ type fieldMapper struct{}
|
||||
func NewFieldMapper() qbtypes.FieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
return logsV2Columns["resource"], nil
|
||||
columns := []*schema.Column{logsV2Columns["resources_string"], logsV2Columns["resource"]}
|
||||
return columns, nil
|
||||
case telemetrytypes.FieldContextScope:
|
||||
switch key.Name {
|
||||
case "name", "scope.name", "scope_name":
|
||||
return logsV2Columns["scope_name"], nil
|
||||
return []*schema.Column{logsV2Columns["scope_name"]}, nil
|
||||
case "version", "scope.version", "scope_version":
|
||||
return logsV2Columns["scope_version"], nil
|
||||
return []*schema.Column{logsV2Columns["scope_version"]}, nil
|
||||
}
|
||||
return logsV2Columns["scope_string"], nil
|
||||
return []*schema.Column{logsV2Columns["scope_string"]}, nil
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
switch key.FieldDataType {
|
||||
case telemetrytypes.FieldDataTypeString:
|
||||
return logsV2Columns["attributes_string"], nil
|
||||
return []*schema.Column{logsV2Columns["attributes_string"]}, nil
|
||||
case telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64, telemetrytypes.FieldDataTypeNumber:
|
||||
return logsV2Columns["attributes_number"], nil
|
||||
return []*schema.Column{logsV2Columns["attributes_number"]}, nil
|
||||
case telemetrytypes.FieldDataTypeBool:
|
||||
return logsV2Columns["attributes_bool"], nil
|
||||
return []*schema.Column{logsV2Columns["attributes_bool"]}, nil
|
||||
}
|
||||
case telemetrytypes.FieldContextBody:
|
||||
// Body context is for JSON body fields. Use body_v2 if feature flag is enabled.
|
||||
if querybuilder.BodyJSONQueryEnabled {
|
||||
if key.Name == messageSubField {
|
||||
return logsV2Columns[messageSubColumn], nil
|
||||
return []*schema.Column{logsV2Columns[messageSubColumn]}, nil
|
||||
}
|
||||
return logsV2Columns[LogsV2BodyV2Column], nil
|
||||
return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}, nil
|
||||
}
|
||||
// Fall back to legacy body column
|
||||
return logsV2Columns["body"], nil
|
||||
return []*schema.Column{logsV2Columns["body"]}, nil
|
||||
case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextUnspecified:
|
||||
if key.Name == LogsV2BodyColumn && querybuilder.BodyJSONQueryEnabled {
|
||||
return logsV2Columns[messageSubColumn], nil
|
||||
return []*schema.Column{logsV2Columns[messageSubColumn]}, nil
|
||||
}
|
||||
col, ok := logsV2Columns[key.Name]
|
||||
if !ok {
|
||||
@@ -108,100 +114,248 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry
|
||||
if strings.HasPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix) {
|
||||
// Use body_v2 if feature flag is enabled and we have a body condition builder
|
||||
if querybuilder.BodyJSONQueryEnabled {
|
||||
return logsV2Columns[LogsV2BodyV2Column], nil
|
||||
// TODO(Piyush): Update this to support multiple JSON columns based on evolutions
|
||||
// i.e return both the body json and body json promoted and let the evolutions decide which one to use
|
||||
// based on the query range time.
|
||||
return []*schema.Column{logsV2Columns[LogsV2BodyV2Column]}, nil
|
||||
}
|
||||
// Fall back to legacy body column
|
||||
return logsV2Columns["body"], nil
|
||||
return []*schema.Column{logsV2Columns["body"]}, nil
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
return col, nil
|
||||
return []*schema.Column{col}, nil
|
||||
}
|
||||
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
// selectEvolutionsForColumns selects the appropriate evolution entries for each column based on the time range.
|
||||
// Logic:
|
||||
// - Finds the latest base evolution (<= tsStartTime) across ALL columns
|
||||
// - Rejects all evolutions before this latest base evolution
|
||||
// - For duplicate evolutions it considers the oldest one (first in ReleaseTime)
|
||||
// - For each column, includes its evolution if it's >= latest base evolution and <= tsEndTime
|
||||
// - Results are sorted by ReleaseTime descending (newest first)
|
||||
func selectEvolutionsForColumns(columns []*schema.Column, evolutions []*telemetrytypes.EvolutionEntry, tsStart, tsEnd uint64) ([]*schema.Column, []*telemetrytypes.EvolutionEntry, error) {
|
||||
|
||||
sortedEvolutions := make([]*telemetrytypes.EvolutionEntry, len(evolutions))
|
||||
copy(sortedEvolutions, evolutions)
|
||||
|
||||
// sort the evolutions by ReleaseTime ascending
|
||||
sort.Slice(sortedEvolutions, func(i, j int) bool {
|
||||
return sortedEvolutions[i].ReleaseTime.Before(sortedEvolutions[j].ReleaseTime)
|
||||
})
|
||||
|
||||
tsStartTime := time.Unix(0, int64(tsStart))
|
||||
tsEndTime := time.Unix(0, int64(tsEnd))
|
||||
|
||||
// Build evolution map: column name -> evolution
|
||||
evolutionMap := make(map[string]*telemetrytypes.EvolutionEntry)
|
||||
for _, evolution := range sortedEvolutions {
|
||||
if _, exists := evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))]; exists {
|
||||
// since if there is duplicate we would just use the oldest one.
|
||||
continue
|
||||
}
|
||||
evolutionMap[evolution.ColumnName+":"+evolution.FieldName+":"+strconv.Itoa(int(evolution.Version))] = evolution
|
||||
}
|
||||
|
||||
// Find the latest base evolution (<= tsStartTime) across ALL columns
|
||||
// Evolutions are sorted, so we can break early
|
||||
var latestBaseEvolutionAcrossAll *telemetrytypes.EvolutionEntry
|
||||
for _, evolution := range sortedEvolutions {
|
||||
if evolution.ReleaseTime.After(tsStartTime) {
|
||||
break
|
||||
}
|
||||
latestBaseEvolutionAcrossAll = evolution
|
||||
}
|
||||
|
||||
// We shouldn't reach this, it basically means there is something wrong with the evolutions data
|
||||
if latestBaseEvolutionAcrossAll == nil {
|
||||
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "no base evolution found for columns %v", columns)
|
||||
}
|
||||
|
||||
columnLookUpMap := make(map[string]*schema.Column)
|
||||
for _, column := range columns {
|
||||
columnLookUpMap[column.Name] = column
|
||||
}
|
||||
|
||||
// Collect column-evolution pairs
|
||||
type colEvoPair struct {
|
||||
column *schema.Column
|
||||
evolution *telemetrytypes.EvolutionEntry
|
||||
}
|
||||
pairs := []colEvoPair{}
|
||||
|
||||
for _, evolution := range evolutionMap {
|
||||
// Reject evolutions before the latest base evolution
|
||||
if evolution.ReleaseTime.Before(latestBaseEvolutionAcrossAll.ReleaseTime) {
|
||||
continue
|
||||
}
|
||||
// skip evolutions after tsEndTime
|
||||
if evolution.ReleaseTime.After(tsEndTime) || evolution.ReleaseTime.Equal(tsEndTime) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, exists := columnLookUpMap[evolution.ColumnName]; !exists {
|
||||
return nil, nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "evolution column %s not found in columns %v", evolution.ColumnName, columns)
|
||||
}
|
||||
|
||||
pairs = append(pairs, colEvoPair{columnLookUpMap[evolution.ColumnName], evolution})
|
||||
}
|
||||
|
||||
// If no pairs found, fall back to latestBaseEvolutionAcrossAll for matching columns
|
||||
if len(pairs) == 0 {
|
||||
for _, column := range columns {
|
||||
// Use latestBaseEvolutionAcrossAll if this column name matches its column name
|
||||
if column.Name == latestBaseEvolutionAcrossAll.ColumnName {
|
||||
pairs = append(pairs, colEvoPair{column, latestBaseEvolutionAcrossAll})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by ReleaseTime descending (newest first)
|
||||
slices.SortFunc(pairs, func(a, b colEvoPair) int {
|
||||
// Sort by ReleaseTime descending (newest first)
|
||||
if a.evolution.ReleaseTime.After(b.evolution.ReleaseTime) {
|
||||
return -1
|
||||
}
|
||||
if a.evolution.ReleaseTime.Before(b.evolution.ReleaseTime) {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
|
||||
// Extract results
|
||||
newColumns := make([]*schema.Column, len(pairs))
|
||||
evolutionsEntries := make([]*telemetrytypes.EvolutionEntry, len(pairs))
|
||||
for i, pair := range pairs {
|
||||
newColumns[i] = pair.column
|
||||
evolutionsEntries[i] = pair.evolution
|
||||
}
|
||||
|
||||
return newColumns, evolutionsEntries, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
columns, err := m.getColumn(ctx, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
// json is only supported for resource context as of now
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
oldColumn := logsV2Columns["resources_string"]
|
||||
oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
|
||||
|
||||
// have to add ::string as clickHouse throws an error :- data types Variant/Dynamic are not allowed in GROUP BY
|
||||
// once clickHouse dependency is updated, we need to check if we can remove it.
|
||||
if key.Materialized {
|
||||
oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
|
||||
oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
|
||||
return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
|
||||
}
|
||||
return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
|
||||
case telemetrytypes.FieldContextBody:
|
||||
if key.Name == messageSubField {
|
||||
return messageSubColumn, nil
|
||||
}
|
||||
|
||||
if key.JSONDataType == nil {
|
||||
return "", qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
if key.KeyNameContainsArray() && !key.JSONDataType.IsArray {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "FieldFor not supported for nested fields; only supported for flat paths (e.g. body.status.detail) and paths of Array type: %s(%s)", key.Name, key.FieldDataType)
|
||||
}
|
||||
|
||||
return m.buildFieldForJSON(key)
|
||||
default:
|
||||
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/body context fields are supported for json columns, got %s", key.FieldContext.String)
|
||||
var newColumns []*schema.Column
|
||||
var evolutionsEntries []*telemetrytypes.EvolutionEntry
|
||||
if len(key.Evolutions) > 0 {
|
||||
// we will use the corresponding column and its evolution entry for the query
|
||||
newColumns, evolutionsEntries, err = selectEvolutionsForColumns(columns, key.Evolutions, tsStart, tsEnd)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
return column.Name, nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
|
||||
return column.Name, nil
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
|
||||
} else {
|
||||
newColumns = columns
|
||||
}
|
||||
|
||||
exprs := []string{}
|
||||
existExpr := []string{}
|
||||
for i, column := range newColumns {
|
||||
// Use evolution column name if available, otherwise use the column name
|
||||
columnName := column.Name
|
||||
if evolutionsEntries != nil && evolutionsEntries[i] != nil {
|
||||
columnName = evolutionsEntries[i].ColumnName
|
||||
}
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
|
||||
case telemetrytypes.FieldContextBody:
|
||||
if key.Name == messageSubField {
|
||||
exprs = append(exprs, messageSubColumn)
|
||||
continue
|
||||
}
|
||||
|
||||
if key.JSONDataType == nil {
|
||||
return "", qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
if key.KeyNameContainsArray() && !key.JSONDataType.IsArray {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "FieldFor not supported for nested fields; only supported for flat paths (e.g. body.status.detail) and paths of Array type: %s(%s)", key.Name, key.FieldDataType)
|
||||
}
|
||||
|
||||
expr, err := m.buildFieldForJSON(key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
exprs = append(exprs, expr)
|
||||
default:
|
||||
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource/body context fields are supported for json columns, got %s", key.FieldContext.String)
|
||||
}
|
||||
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
exprs = append(exprs, column.Name)
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
|
||||
exprs = append(exprs, column.Name)
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
|
||||
}
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s==true", telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)))
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
}
|
||||
return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
}
|
||||
}
|
||||
|
||||
if len(exprs) == 1 {
|
||||
return exprs[0], nil
|
||||
} else if len(exprs) > 1 {
|
||||
// Ensure existExpr has the same length as exprs
|
||||
if len(existExpr) != len(exprs) {
|
||||
return "", errors.New(errors.TypeInternal, errors.CodeInternal, "length of exist exprs doesn't match to that of exprs")
|
||||
}
|
||||
finalExprs := []string{}
|
||||
for i, expr := range exprs {
|
||||
finalExprs = append(finalExprs, fmt.Sprintf("%s, %s", existExpr[i], expr))
|
||||
}
|
||||
return "multiIf(" + strings.Join(finalExprs, ", ") + ", NULL)", nil
|
||||
}
|
||||
|
||||
// should not reach here
|
||||
return column.Name, nil
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
fieldExpression, err := m.FieldFor(ctx, tsStart, tsEnd, field)
|
||||
if errors.Is(err, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -211,7 +365,7 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
if _, ok := logsV2Columns[field.Name]; ok {
|
||||
// if it is, attach the column name directly
|
||||
field.FieldContext = telemetrytypes.FieldContextLog
|
||||
colName, _ = m.FieldFor(ctx, field)
|
||||
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, field)
|
||||
} else {
|
||||
// - the context is not provided
|
||||
// - there are not keys for the field
|
||||
@@ -229,19 +383,19 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
} else if len(keysForField) == 1 {
|
||||
// we have a single key for the field, use it
|
||||
colName, _ = m.FieldFor(ctx, keysForField[0])
|
||||
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, keysForField[0])
|
||||
} else {
|
||||
// select any non-empty value from the keys
|
||||
args := []string{}
|
||||
for _, key := range keysForField {
|
||||
colName, _ = m.FieldFor(ctx, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", colName, colName))
|
||||
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", fieldExpression, fieldExpression))
|
||||
}
|
||||
colName = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
fieldExpression = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
}
|
||||
|
||||
// buildFieldForJSON builds the field expression for body JSON fields using arrayConcat pattern
|
||||
|
||||
@@ -3,6 +3,7 @@ package telemetrylogs
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
@@ -17,7 +18,7 @@ func TestGetColumn(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedCol *schema.Column
|
||||
expectedCol []*schema.Column
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
@@ -26,7 +27,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
},
|
||||
expectedCol: logsV2Columns["resource"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["resources_string"], logsV2Columns["resource"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -35,7 +36,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "name",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: logsV2Columns["scope_name"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_name"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -44,7 +45,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "scope.name",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: logsV2Columns["scope_name"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_name"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -53,7 +54,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "scope_name",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: logsV2Columns["scope_name"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_name"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -62,7 +63,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "version",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: logsV2Columns["scope_version"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_version"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -71,7 +72,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "custom.scope.field",
|
||||
FieldContext: telemetrytypes.FieldContextScope,
|
||||
},
|
||||
expectedCol: logsV2Columns["scope_string"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["scope_string"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -81,7 +82,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
expectedCol: logsV2Columns["attributes_string"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_string"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -91,7 +92,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeNumber,
|
||||
},
|
||||
expectedCol: logsV2Columns["attributes_number"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_number"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -101,7 +102,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeInt64,
|
||||
},
|
||||
expectedCol: logsV2Columns["attributes_number"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_number"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -111,7 +112,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeFloat64,
|
||||
},
|
||||
expectedCol: logsV2Columns["attributes_number"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_number"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -121,7 +122,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
expectedCol: logsV2Columns["attributes_bool"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_bool"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -130,7 +131,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "timestamp",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
expectedCol: logsV2Columns["timestamp"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["timestamp"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -139,7 +140,7 @@ func TestGetColumn(t *testing.T) {
|
||||
Name: "body",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
expectedCol: logsV2Columns["body"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["body"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
@@ -159,7 +160,7 @@ func TestGetColumn(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
expectedCol: logsV2Columns["attributes_bool"],
|
||||
expectedCol: []*schema.Column{logsV2Columns["attributes_bool"]},
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
@@ -168,7 +169,7 @@ func TestGetColumn(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
col, err := fm.ColumnFor(ctx, &tc.key)
|
||||
col, err := fm.ColumnFor(ctx, 0, 0, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
@@ -183,11 +184,14 @@ func TestGetColumn(t *testing.T) {
|
||||
func TestGetFieldKeyName(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
resourceEvolution := mockEvolutionData(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC))
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedResult string
|
||||
expectedError error
|
||||
name string
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedResult string
|
||||
expectedError error
|
||||
addExistsFilter bool
|
||||
}{
|
||||
{
|
||||
name: "Simple column type - timestamp",
|
||||
@@ -195,8 +199,9 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
Name: "timestamp",
|
||||
FieldContext: telemetrytypes.FieldContextLog,
|
||||
},
|
||||
expectedResult: "timestamp",
|
||||
expectedError: nil,
|
||||
expectedResult: "timestamp",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
},
|
||||
{
|
||||
name: "Map column type - string attribute",
|
||||
@@ -205,8 +210,9 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
expectedResult: "attributes_string['user.id']",
|
||||
expectedError: nil,
|
||||
expectedResult: "attributes_string['user.id']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
},
|
||||
{
|
||||
name: "Map column type - number attribute",
|
||||
@@ -215,8 +221,9 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeNumber,
|
||||
},
|
||||
expectedResult: "attributes_number['request.size']",
|
||||
expectedError: nil,
|
||||
expectedResult: "attributes_number['request.size']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
},
|
||||
{
|
||||
name: "Map column type - bool attribute",
|
||||
@@ -225,28 +232,33 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeBool,
|
||||
},
|
||||
expectedResult: "attributes_bool['request.success']",
|
||||
expectedError: nil,
|
||||
expectedResult: "attributes_bool['request.success']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
},
|
||||
{
|
||||
name: "Map column type - resource attribute",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
Evolutions: resourceEvolution,
|
||||
},
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
|
||||
expectedError: nil,
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
},
|
||||
{
|
||||
name: "Map column type - resource attribute - Materialized",
|
||||
name: "Map column type - resource attribute - Materialized - json",
|
||||
key: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
Evolutions: resourceEvolution,
|
||||
},
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL)",
|
||||
expectedError: nil,
|
||||
expectedResult: "`resource_string_service$$name`",
|
||||
expectedError: nil,
|
||||
addExistsFilter: false,
|
||||
},
|
||||
{
|
||||
name: "Non-existent column",
|
||||
@@ -262,7 +274,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
@@ -273,3 +285,693 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldForWithEvolutions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
evolutions []*telemetrytypes.EvolutionEntry
|
||||
key *telemetrytypes.TelemetryFieldKey
|
||||
tsStartTime time.Time
|
||||
tsEndTime time.Time
|
||||
expectedResult string
|
||||
expectedError error
|
||||
}{
|
||||
{
|
||||
name: "Single evolution before tsStartTime",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Single evolution exactly at tsStartTime",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Single evolution after tsStartTime",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
// TODO(piyush): to be added once integration with JSON is done.
|
||||
// {
|
||||
// name: "Single evolution after tsStartTime - JSON body",
|
||||
// evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
// {
|
||||
// Signal: telemetrytypes.SignalLogs,
|
||||
// ColumnName: LogsV2BodyV2Column,
|
||||
// ColumnType: "JSON(max_dynamic_paths=0)",
|
||||
// FieldContext: telemetrytypes.FieldContextBody,
|
||||
// FieldName: "__all__",
|
||||
// ReleaseTime: time.Unix(0, 0),
|
||||
// },
|
||||
// {
|
||||
// Signal: telemetrytypes.SignalLogs,
|
||||
// ColumnName: LogsV2BodyPromotedColumn,
|
||||
// ColumnType: "JSON()",
|
||||
// FieldContext: telemetrytypes.FieldContextBody,
|
||||
// FieldName: "user.name",
|
||||
// ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
// },
|
||||
// },
|
||||
// key: &telemetrytypes.TelemetryFieldKey{
|
||||
// Name: "user.name",
|
||||
// FieldContext: telemetrytypes.FieldContextBody,
|
||||
// JSONDataType: &telemetrytypes.String,
|
||||
// Materialized: true,
|
||||
// },
|
||||
// tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
// tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
// expectedResult: "coalesce(dynamicElement(body_json.`user.name`, 'String'), dynamicElement(body_promoted.`user.name`, 'String'))",
|
||||
// expectedError: nil,
|
||||
// },
|
||||
{
|
||||
name: "Multiple evolutions before tsStartTime - only latest should be included",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resource.`service.name`::String",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Multiple evolutions after tsStartTime - all should be included",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Unix(0, 0),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL)",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Duplicate evolutions after tsStartTime - all should be included",
|
||||
// Note: on production when this happens, we should go ahead and clean it up if required
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resource.`service.name`::String",
|
||||
expectedError: nil,
|
||||
},
|
||||
{
|
||||
name: "Evolution exactly at tsEndTime - should not be included",
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
key: key,
|
||||
tsStartTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
tsEndTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "resources_string['service.name']",
|
||||
expectedError: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
|
||||
tsStart := uint64(tc.tsStartTime.UnixNano())
|
||||
tsEnd := uint64(tc.tsEndTime.UnixNano())
|
||||
tc.key.Evolutions = tc.evolutions
|
||||
|
||||
result, err := fm.FieldFor(ctx, tsStart, tsEnd, tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedResult, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectEvolutionsForColumns(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
columns []*schema.Column
|
||||
evolutions []*telemetrytypes.EvolutionEntry
|
||||
tsStart uint64
|
||||
tsEnd uint64
|
||||
expectedColumns []string // column names
|
||||
expectedEvols []string // evolution column names
|
||||
expectedError bool
|
||||
errorStr string
|
||||
}{
|
||||
{
|
||||
name: "New evolutions at tsStartTime - should include latest evolution",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resource"},
|
||||
expectedEvols: []string{"resource"},
|
||||
},
|
||||
{
|
||||
name: "New evolutions after tsStartTime but less than tsEndTime - should include both",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resource", "resources_string"}, // sorted by ReleaseTime desc
|
||||
expectedEvols: []string{"resource", "resources_string"},
|
||||
},
|
||||
{
|
||||
name: "Columns without matching evolutions - should exclude them",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"], // no evolution for this
|
||||
logsV2Columns["attributes_string"], // no evolution for this
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"},
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "New evolutions at tsEndTime - should not include new evolution",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 30, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"},
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "New evolutions after tsEndTime - should exclude new",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 2, 25, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"},
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "Empty columns array",
|
||||
columns: []*schema.Column{},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{},
|
||||
expectedEvols: []string{},
|
||||
expectedError: true,
|
||||
errorStr: "column resources_string not found",
|
||||
},
|
||||
{
|
||||
name: "Duplicate evolutions - should use first encountered (oldest if sorted)",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resource"},
|
||||
expectedEvols: []string{"resource"}, // should use first one (older)
|
||||
},
|
||||
{
|
||||
name: "Genuine Duplicate evolutions with new version- should consider both",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 0,
|
||||
ReleaseTime: time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 1,
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
Version: 2,
|
||||
ReleaseTime: time.Date(2024, 1, 20, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 1, 16, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string", "resource"},
|
||||
expectedEvols: []string{"resources_string", "resource"}, // should use first one (older)
|
||||
},
|
||||
{
|
||||
name: "Evolution exactly at tsEndTime",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns["resources_string"],
|
||||
logsV2Columns["resource"],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 1, 15, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC), // exactly at tsEnd
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{"resources_string"}, // resource excluded because After(tsEnd) is true
|
||||
expectedEvols: []string{"resources_string"},
|
||||
},
|
||||
{
|
||||
name: "Single evolution after tsStartTime - JSON body",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns[LogsV2BodyV2Column],
|
||||
logsV2Columns[LogsV2BodyPromotedColumn],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyV2Column,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyPromotedColumn,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "user.name",
|
||||
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column}, // sorted by ReleaseTime desc (newest first)
|
||||
expectedEvols: []string{LogsV2BodyPromotedColumn, LogsV2BodyV2Column},
|
||||
},
|
||||
{
|
||||
name: "No evolution after tsStartTime - JSON body",
|
||||
columns: []*schema.Column{
|
||||
logsV2Columns[LogsV2BodyV2Column],
|
||||
logsV2Columns[LogsV2BodyPromotedColumn],
|
||||
},
|
||||
evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyV2Column,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: LogsV2BodyPromotedColumn,
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldName: "user.name",
|
||||
ReleaseTime: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
tsStart: uint64(time.Date(2024, 2, 3, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
tsEnd: uint64(time.Date(2024, 2, 15, 0, 0, 0, 0, time.UTC).UnixNano()),
|
||||
expectedColumns: []string{LogsV2BodyPromotedColumn},
|
||||
expectedEvols: []string{LogsV2BodyPromotedColumn},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
resultColumns, resultEvols, err := selectEvolutionsForColumns(tc.columns, tc.evolutions, tc.tsStart, tc.tsEnd)
|
||||
|
||||
if tc.expectedError {
|
||||
assert.Contains(t, err.Error(), tc.errorStr)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, len(tc.expectedColumns), len(resultColumns), "column count mismatch")
|
||||
assert.Equal(t, len(tc.expectedEvols), len(resultEvols), "evolution count mismatch")
|
||||
|
||||
resultColumnNames := make([]string, len(resultColumns))
|
||||
for i, col := range resultColumns {
|
||||
resultColumnNames[i] = col.Name
|
||||
}
|
||||
resultEvolNames := make([]string, len(resultEvols))
|
||||
for i, evol := range resultEvols {
|
||||
resultEvolNames[i] = evol.ColumnName
|
||||
}
|
||||
|
||||
for i := range tc.expectedColumns {
|
||||
assert.Equal(t, resultColumnNames[i], tc.expectedColumns[i], "expected column missing: "+tc.expectedColumns[i])
|
||||
}
|
||||
for i := range tc.expectedEvols {
|
||||
assert.Equal(t, resultEvolNames[i], tc.expectedEvols[i], "expected evolution missing: "+tc.expectedEvols[i])
|
||||
}
|
||||
// Verify sorting: should be descending by ReleaseTime
|
||||
for i := 0; i < len(resultEvols)-1; i++ {
|
||||
assert.True(t, !resultEvols[i].ReleaseTime.Before(resultEvols[i+1].ReleaseTime),
|
||||
"evolutions should be sorted descending by ReleaseTime")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFieldForWithMaterialized(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
materializedKey := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
Evolutions: []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Date(2024, 3, 2, 0, 0, 0, 0, time.UTC),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
start, end time.Time
|
||||
expectedResult string
|
||||
}{
|
||||
{
|
||||
name: "Map column in use (pre-evolution to JSON)",
|
||||
start: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2024, 2, 2, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "`resource_string_service$$name`",
|
||||
},
|
||||
{
|
||||
name: "Multi evolution - both columns (JSON + materialized)",
|
||||
start: time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2024, 4, 2, 0, 0, 0, 0, time.UTC),
|
||||
expectedResult: "multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, `resource_string_service$$name_exists`==true, `resource_string_service$$name`, NULL)",
|
||||
},
|
||||
}
|
||||
|
||||
fm := NewFieldMapper()
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
start := uint64(tc.start.UnixNano())
|
||||
end := uint64(tc.end.UnixNano())
|
||||
result, err := fm.FieldFor(ctx, start, end, materializedKey)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedResult, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -10,12 +12,15 @@ import (
|
||||
|
||||
// TestLikeAndILikeWithoutWildcards_Warns Tests that LIKE/ILIKE without wildcards add warnings and include docs URL
|
||||
func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -33,7 +38,7 @@ func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
|
||||
|
||||
for _, expr := range tests {
|
||||
t.Run(expr, func(t *testing.T) {
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts, 0, 0)
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, clause)
|
||||
|
||||
@@ -49,9 +54,11 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -69,7 +76,7 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
|
||||
|
||||
for _, expr := range tests {
|
||||
t.Run(expr, func(t *testing.T) {
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts, 0, 0)
|
||||
clause, err := querybuilder.PrepareWhereClause(expr, opts)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, clause)
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -16,9 +18,11 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -161,7 +165,7 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("%s: %s", tc.category, limitString(tc.query, 50)), func(t *testing.T) {
|
||||
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts, 0, 0)
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts)
|
||||
|
||||
if tc.shouldPass {
|
||||
if err != nil {
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
package telemetrylogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
@@ -15,19 +17,24 @@ import (
|
||||
|
||||
// TestFilterExprLogs tests a comprehensive set of query patterns for logs search
|
||||
func TestFilterExprLogs(t *testing.T) {
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: DefaultFullTextColumn,
|
||||
JsonKeyToKey: GetBodyJSONKey,
|
||||
StartNs: uint64(releaseTime.Add(-5 * time.Minute).UnixNano()),
|
||||
EndNs: uint64(releaseTime.Add(5 * time.Minute).UnixNano()),
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
@@ -466,7 +473,7 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
expectedErrorContains: "",
|
||||
},
|
||||
|
||||
// fulltext with parenthesized expression
|
||||
//fulltext with parenthesized expression
|
||||
{
|
||||
category: "FREETEXT with parentheses",
|
||||
query: "error (status.code=500 OR status.code=503)",
|
||||
@@ -2386,7 +2393,7 @@ func TestFilterExprLogs(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("%s: %s", tc.category, limitString(tc.query, 50)), func(t *testing.T) {
|
||||
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts, 0, 0)
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts)
|
||||
|
||||
if tc.shouldPass {
|
||||
if err != nil {
|
||||
@@ -2426,7 +2433,8 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
// Define a comprehensive set of field keys to support all test cases
|
||||
keys := buildCompleteFieldKeyMap()
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
keys := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
keys["body"] = []*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
@@ -2442,6 +2450,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
}
|
||||
|
||||
opts := querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
@@ -2504,7 +2513,7 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(fmt.Sprintf("%s: %s", tc.category, limitString(tc.query, 50)), func(t *testing.T) {
|
||||
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts, 0, 0)
|
||||
clause, err := querybuilder.PrepareWhereClause(tc.query, opts)
|
||||
|
||||
if tc.shouldPass {
|
||||
if err != nil {
|
||||
|
||||
@@ -283,7 +283,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
|
||||
// get column expression for the field - use array index directly to avoid pointer to loop variable
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &query.SelectFields[index], keys)
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &query.SelectFields[index], keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -292,7 +292,6 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
}
|
||||
|
||||
sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
|
||||
|
||||
// Add filter conditions
|
||||
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
|
||||
|
||||
@@ -302,7 +301,8 @@ func (b *logQueryStatementBuilder) buildListQuery(
|
||||
|
||||
// Add order by
|
||||
for _, orderBy := range query.Order {
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -368,7 +368,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
// Keep original column expressions so we can build the tuple
|
||||
fieldNames := make([]string, 0, len(query.GroupBy))
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -383,7 +383,7 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
allAggChArgs := make([]any, 0)
|
||||
for i, agg := range query.Aggregations {
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, agg.Expression,
|
||||
ctx, start, end, agg.Expression,
|
||||
uint64(query.StepInterval.Seconds()),
|
||||
keys,
|
||||
)
|
||||
@@ -515,7 +515,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
var allGroupByArgs []any
|
||||
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -533,7 +533,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
for idx := range query.Aggregations {
|
||||
aggExpr := query.Aggregations[idx]
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, aggExpr.Expression,
|
||||
ctx, start, end, aggExpr.Expression,
|
||||
rateInterval,
|
||||
keys,
|
||||
)
|
||||
@@ -605,7 +605,7 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
|
||||
|
||||
// buildFilterCondition builds SQL condition from filter expression
|
||||
func (b *logQueryStatementBuilder) addFilterCondition(
|
||||
_ context.Context,
|
||||
ctx context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
|
||||
@@ -619,6 +619,7 @@ func (b *logQueryStatementBuilder) addFilterCondition(
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
// add filter expression
|
||||
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
@@ -627,7 +628,9 @@ func (b *logQueryStatementBuilder) addFilterCondition(
|
||||
FullTextColumn: b.fullTextColumn,
|
||||
JsonKeyToKey: b.jsonKeyToKey,
|
||||
Variables: variables,
|
||||
}, start, end)
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -19,7 +19,7 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation
|
||||
fm := resourcefilter.NewFieldMapper()
|
||||
cb := resourcefilter.NewConditionBuilder(fm)
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
keysMap := buildCompleteFieldKeyMap()
|
||||
keysMap := buildCompleteFieldKeyMap(time.Now())
|
||||
for _, keys := range keysMap {
|
||||
for _, key := range keys {
|
||||
key.Signal = telemetrytypes.SignalLogs
|
||||
@@ -38,7 +38,14 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation
|
||||
}
|
||||
|
||||
func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
releaseTimeNano := uint64(releaseTime.UnixNano())
|
||||
|
||||
cases := []struct {
|
||||
startTs uint64
|
||||
endTs uint64
|
||||
name string
|
||||
requestType qbtypes.RequestType
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
|
||||
@@ -46,14 +53,16 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
expectedErr error
|
||||
}{
|
||||
{
|
||||
name: "Time series with limit",
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with limit and count distinct on service.name",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
||||
Aggregations: []qbtypes.LogAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
Expression: "count_distinct(service.name)",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
@@ -69,20 +78,22 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, countDistinct(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, countDistinct(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
name: "Time series with OR b/w resource attr and attribute filter",
|
||||
startTs: releaseTimeNano - uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with OR b/w resource attr and attribute filter and count distinct on service.name",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
StepInterval: qbtypes.Step{Duration: 30 * time.Second},
|
||||
Aggregations: []qbtypes.LogAggregation{
|
||||
{
|
||||
Expression: "count()",
|
||||
Expression: "count_distinct(service.name)",
|
||||
},
|
||||
},
|
||||
Filter: &qbtypes.Filter{
|
||||
@@ -98,12 +109,14 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) OR true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1747945619), uint64(1747983448), "redis-manual", "GET", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "redis-manual", "GET", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) OR true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, countDistinct(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) = ? AND multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL) OR (attributes_string['http.method'] = ? AND mapContains(attributes_string, 'http.method') = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name`",
|
||||
Args: []any{"redis-manual", "%service.name%", "%service.name\":\"redis-manual%", uint64(1705224600), uint64(1705485600), "redis-manual", "GET", true, "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600), 10, "redis-manual", "GET", true, "1705226400000000000", uint64(1705224600), "1705485600000000000", uint64(1705485600)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with limit + custom order by",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -137,12 +150,14 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY `service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL) IS NOT NULL, multiIf(resource.`service.name` IS NOT NULL, resource.`service.name`::String, mapContains(resources_string, 'service.name'), resources_string['service.name'], NULL), NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name` ORDER BY `service.name` desc, ts desc",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `service.name` ORDER BY `service.name` desc LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(resource.`service.name`::String IS NOT NULL, resource.`service.name`::String, NULL)) AS `service.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`service.name`) GLOBAL IN (SELECT `service.name` FROM __limit_cte) GROUP BY ts, `service.name` ORDER BY `service.name` desc, ts desc",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
{
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with group by on materialized column",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -169,10 +184,12 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(`attribute_string_materialized$$key$$name_exists` = ?, `attribute_string_materialized$$key$$name`, NULL)) AS `materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `materialized.key.name` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, toString(multiIf(`attribute_string_materialized$$key$$name_exists` = ?, `attribute_string_materialized$$key$$name`, NULL)) AS `materialized.key.name`, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`materialized.key.name`) GLOBAL IN (SELECT `materialized.key.name` FROM __limit_cte) GROUP BY ts, `materialized.key.name`",
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10, true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
Args: []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1705397400), uint64(1705485600), true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600), 10, true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
},
|
||||
},
|
||||
{
|
||||
startTs: releaseTimeNano + uint64(24*time.Hour.Nanoseconds()),
|
||||
endTs: releaseTimeNano + uint64(48*time.Hour.Nanoseconds()),
|
||||
name: "Time series with materialised column using or with regex operator",
|
||||
requestType: qbtypes.RequestTypeTimeSeries,
|
||||
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
@@ -190,14 +207,19 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
},
|
||||
expected: qbtypes.Statement{
|
||||
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (true OR true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 30 SECOND) AS ts, count() AS __result_0 FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((match(`attribute_string_materialized$$key$$name`, ?) AND `attribute_string_materialized$$key$$name_exists` = ?) OR (`attribute_string_materialized$$key$$name` = ? AND `attribute_string_materialized$$key$$name_exists` = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY ts",
|
||||
Args: []any{uint64(1747945619), uint64(1747983448), "redis.*", true, "memcached", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
|
||||
Args: []any{uint64(1705397400), uint64(1705485600), "redis.*", true, "memcached", true, "1705399200000000000", uint64(1705397400), "1705485600000000000", uint64(1705485600)},
|
||||
},
|
||||
expectedErr: nil,
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
keysMap := buildCompleteFieldKeyMap(releaseTime)
|
||||
|
||||
mockMetadataStore.KeysMap = keysMap
|
||||
|
||||
fm := NewFieldMapper()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
@@ -219,7 +241,7 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(ctx, c.startTs, c.endTs, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -316,9 +338,13 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
fm := NewFieldMapper()
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -339,7 +365,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -456,9 +482,12 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
fm := NewFieldMapper()
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -476,12 +505,10 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
|
||||
GetBodyJSONKey,
|
||||
)
|
||||
|
||||
//
|
||||
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -532,9 +559,12 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
|
||||
fm := NewFieldMapper()
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMap(releaseTime)
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -555,7 +585,7 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErrContains != "" {
|
||||
require.Error(t, err)
|
||||
@@ -627,9 +657,10 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
|
||||
fm := NewFieldMapper()
|
||||
mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
|
||||
cb := NewConditionBuilder(fm)
|
||||
|
||||
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
|
||||
@@ -650,7 +681,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
|
||||
q, err := statementBuilder.Build(context.Background(), 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, c.requestType, c.query, nil)
|
||||
|
||||
if c.expectedErr != nil {
|
||||
require.Error(t, err)
|
||||
@@ -668,6 +699,9 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestAdjustKey(t *testing.T) {
|
||||
|
||||
// Create a test release time
|
||||
releaseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
|
||||
cases := []struct {
|
||||
name string
|
||||
inputKey telemetrytypes.TelemetryFieldKey
|
||||
@@ -681,7 +715,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: IntrinsicFields["severity_text"],
|
||||
},
|
||||
{
|
||||
@@ -718,7 +752,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "severity_number",
|
||||
FieldContext: telemetrytypes.FieldContextBody,
|
||||
@@ -732,8 +766,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "single matching key with incorrect context specified - no override",
|
||||
@@ -742,7 +776,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "service.name",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -756,8 +790,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["service.name"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["service.name"][0],
|
||||
},
|
||||
{
|
||||
name: "multiple matching keys - all materialized",
|
||||
@@ -766,7 +800,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "multi.mat.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -780,7 +814,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "mixed.materialization.key",
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -794,8 +828,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["mixed.materialization.key"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mixed.materialization.key"][0],
|
||||
},
|
||||
{
|
||||
name: "no matching keys - unknown field",
|
||||
@@ -804,7 +838,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
@@ -819,7 +853,7 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: telemetrytypes.TelemetryFieldKey{
|
||||
Name: "unknown.field",
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
@@ -834,8 +868,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["mat.key"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["mat.key"][0],
|
||||
},
|
||||
{
|
||||
name: "non-materialized field",
|
||||
@@ -844,8 +878,8 @@ func TestAdjustKey(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextUnspecified,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
|
||||
},
|
||||
keysMap: buildCompleteFieldKeyMap(),
|
||||
expectedKey: *buildCompleteFieldKeyMap()["user.id"][0],
|
||||
keysMap: buildCompleteFieldKeyMap(releaseTime),
|
||||
expectedKey: *buildCompleteFieldKeyMap(releaseTime)["user.id"][0],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package telemetrylogs
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
@@ -18,7 +19,7 @@ func limitString(s string, maxLen int) string {
|
||||
}
|
||||
|
||||
// Function to build a complete field key map for testing all scenarios
|
||||
func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
func buildCompleteFieldKeyMap(releaseTime time.Time) map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
keysMap := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
"service.name": {
|
||||
{
|
||||
@@ -943,6 +944,9 @@ func buildCompleteFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
|
||||
for _, keys := range keysMap {
|
||||
for _, key := range keys {
|
||||
key.Signal = telemetrytypes.SignalLogs
|
||||
if key.FieldContext == telemetrytypes.FieldContextResource {
|
||||
key.Evolutions = mockEvolutionData(releaseTime)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1008,3 +1012,24 @@ func buildCompleteFieldKeyMapCollision() map[string][]*telemetrytypes.TelemetryF
|
||||
}
|
||||
return keysMap
|
||||
}
|
||||
|
||||
func mockEvolutionData(releaseTime time.Time) []*telemetrytypes.EvolutionEntry {
|
||||
return []*telemetrytypes.EvolutionEntry{
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resources_string",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
ColumnType: "Map(LowCardinality(String), String)",
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: time.Unix(0, 0),
|
||||
},
|
||||
{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
ColumnName: "resource",
|
||||
ColumnType: "JSON()",
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldName: "__all__",
|
||||
ReleaseTime: releaseTime,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,12 +21,11 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
tsStart, tsEnd uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
|
||||
switch operator {
|
||||
@@ -39,13 +38,13 @@ func (c *conditionBuilder) ConditionFor(
|
||||
value = querybuilder.FormatValueForContains(value)
|
||||
}
|
||||
|
||||
column, err := c.fm.ColumnFor(ctx, key)
|
||||
columns, err := c.fm.ColumnFor(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
// if we don't have a column, we can't build a condition for related values
|
||||
return "", nil
|
||||
}
|
||||
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
// if we don't have a table field name, we can't build a condition for related values
|
||||
return "", nil
|
||||
@@ -57,7 +56,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return "", nil
|
||||
}
|
||||
|
||||
tblFieldName, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, tblFieldName, operator)
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
|
||||
// key must exists to apply main filter
|
||||
expr := `if(mapContains(%s, %s), %s, true)`
|
||||
@@ -68,29 +67,29 @@ func (c *conditionBuilder) ConditionFor(
|
||||
switch operator {
|
||||
// regular operators
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
cond = sb.E(tblFieldName, value)
|
||||
cond = sb.E(fieldExpression, value)
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
cond = sb.NE(tblFieldName, value)
|
||||
cond = sb.NE(fieldExpression, value)
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
cond = sb.Like(tblFieldName, value)
|
||||
cond = sb.Like(fieldExpression, value)
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
cond = sb.NotLike(tblFieldName, value)
|
||||
cond = sb.NotLike(fieldExpression, value)
|
||||
case qbtypes.FilterOperatorILike:
|
||||
cond = sb.ILike(tblFieldName, value)
|
||||
cond = sb.ILike(fieldExpression, value)
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
cond = sb.NotILike(tblFieldName, value)
|
||||
cond = sb.NotILike(fieldExpression, value)
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
cond = sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value))
|
||||
cond = sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value))
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
cond = sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value))
|
||||
cond = sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value))
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
cond = fmt.Sprintf(`match(%s, %s)`, tblFieldName, sb.Var(value))
|
||||
cond = fmt.Sprintf(`match(%s, %s)`, fieldExpression, sb.Var(value))
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
cond = fmt.Sprintf(`NOT match(%s, %s)`, tblFieldName, sb.Var(value))
|
||||
cond = fmt.Sprintf(`NOT match(%s, %s)`, fieldExpression, sb.Var(value))
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -101,7 +100,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(tblFieldName, value))
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
}
|
||||
cond = sb.Or(conditions...)
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -112,7 +111,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(tblFieldName, value))
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
}
|
||||
cond = sb.And(conditions...)
|
||||
|
||||
@@ -120,12 +119,12 @@ func (c *conditionBuilder) ConditionFor(
|
||||
// in the query builder, `exists` and `not exists` are used for
|
||||
// key membership checks, so depending on the column type, the condition changes
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
switch column.Type {
|
||||
switch columns[0].Type {
|
||||
case schema.MapColumnType{
|
||||
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
|
||||
ValueType: schema.ColumnTypeString,
|
||||
}:
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
cond = sb.E(leftOperand, true)
|
||||
} else {
|
||||
@@ -134,5 +133,5 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf(expr, column.Name, sb.Var(key.Name), cond), nil
|
||||
return fmt.Sprintf(expr, columns[0].Name, sb.Var(key.Name), cond), nil
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ func TestConditionFor(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
|
||||
@@ -33,47 +33,48 @@ func NewFieldMapper() qbtypes.FieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
func (m *fieldMapper) getColumn(_ context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
return attributeMetadataColumns["resource_attributes"], nil
|
||||
return []*schema.Column{attributeMetadataColumns["resource_attributes"]}, nil
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
return attributeMetadataColumns["attributes"], nil
|
||||
return []*schema.Column{attributeMetadataColumns["attributes"]}, nil
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
columns, err := m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return column, nil
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
columns, err := m.getColumn(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch column.Type {
|
||||
switch columns[0].Type {
|
||||
case schema.MapColumnType{
|
||||
KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
|
||||
ValueType: schema.ColumnTypeString,
|
||||
}:
|
||||
return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
|
||||
return fmt.Sprintf("%s['%s']", columns[0].Name, key.Name), nil
|
||||
}
|
||||
return column.Name, nil
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
fieldExpression, err := m.FieldFor(ctx, startNs, endNs, field)
|
||||
if errors.Is(err, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -83,7 +84,7 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
if _, ok := attributeMetadataColumns[field.Name]; ok {
|
||||
// if it is, attach the column name directly
|
||||
field.FieldContext = telemetrytypes.FieldContextSpan
|
||||
colName, _ = m.FieldFor(ctx, field)
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, field)
|
||||
} else {
|
||||
// - the context is not provided
|
||||
// - there are not keys for the field
|
||||
@@ -101,17 +102,17 @@ func (m *fieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
} else if len(keysForField) == 1 {
|
||||
// we have a single key for the field, use it
|
||||
colName, _ = m.FieldFor(ctx, keysForField[0])
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, keysForField[0])
|
||||
} else {
|
||||
// select any non-empty value from the keys
|
||||
args := []string{}
|
||||
for _, key := range keysForField {
|
||||
colName, _ = m.FieldFor(ctx, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", colName, colName))
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", fieldExpression, fieldExpression))
|
||||
}
|
||||
colName = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
fieldExpression = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
}
|
||||
|
||||
@@ -128,13 +128,13 @@ func TestGetColumn(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
col, err := fm.ColumnFor(context.Background(), &tc.key)
|
||||
col, err := fm.ColumnFor(context.Background(), 0, 0, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedCol, col)
|
||||
assert.Equal(t, tc.expectedCol, col[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -145,6 +145,8 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
tsStart uint64
|
||||
tsEnd uint64
|
||||
key telemetrytypes.TelemetryFieldKey
|
||||
expectedResult string
|
||||
expectedError error
|
||||
@@ -203,7 +205,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, tc.tsStart, tc.tsEnd, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"golang.org/x/exp/maps"
|
||||
@@ -34,23 +35,24 @@ var (
|
||||
)
|
||||
|
||||
type telemetryMetaStore struct {
|
||||
logger *slog.Logger
|
||||
telemetrystore telemetrystore.TelemetryStore
|
||||
tracesDBName string
|
||||
tracesFieldsTblName string
|
||||
spanAttributesKeysTblName string
|
||||
indexV3TblName string
|
||||
metricsDBName string
|
||||
metricsFieldsTblName string
|
||||
meterDBName string
|
||||
meterFieldsTblName string
|
||||
logsDBName string
|
||||
logsFieldsTblName string
|
||||
logAttributeKeysTblName string
|
||||
logResourceKeysTblName string
|
||||
logsV2TblName string
|
||||
relatedMetadataDBName string
|
||||
relatedMetadataTblName string
|
||||
logger *slog.Logger
|
||||
telemetrystore telemetrystore.TelemetryStore
|
||||
tracesDBName string
|
||||
tracesFieldsTblName string
|
||||
spanAttributesKeysTblName string
|
||||
indexV3TblName string
|
||||
metricsDBName string
|
||||
metricsFieldsTblName string
|
||||
meterDBName string
|
||||
meterFieldsTblName string
|
||||
logsDBName string
|
||||
logsFieldsTblName string
|
||||
logAttributeKeysTblName string
|
||||
logResourceKeysTblName string
|
||||
logsV2TblName string
|
||||
relatedMetadataDBName string
|
||||
relatedMetadataTblName string
|
||||
columnEvolutionMetadataTblName string
|
||||
|
||||
fm qbtypes.FieldMapper
|
||||
conditionBuilder qbtypes.ConditionBuilder
|
||||
@@ -79,27 +81,29 @@ func NewTelemetryMetaStore(
|
||||
logResourceKeysTblName string,
|
||||
relatedMetadataDBName string,
|
||||
relatedMetadataTblName string,
|
||||
columnEvolutionMetadataTblName string,
|
||||
) telemetrytypes.MetadataStore {
|
||||
metadataSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrymetadata")
|
||||
|
||||
t := &telemetryMetaStore{
|
||||
logger: metadataSettings.Logger(),
|
||||
telemetrystore: telemetrystore,
|
||||
tracesDBName: tracesDBName,
|
||||
tracesFieldsTblName: tracesFieldsTblName,
|
||||
spanAttributesKeysTblName: spanAttributesKeysTblName,
|
||||
indexV3TblName: indexV3TblName,
|
||||
metricsDBName: metricsDBName,
|
||||
metricsFieldsTblName: metricsFieldsTblName,
|
||||
meterDBName: meterDBName,
|
||||
meterFieldsTblName: meterFieldsTblName,
|
||||
logsDBName: logsDBName,
|
||||
logsV2TblName: logsV2TblName,
|
||||
logsFieldsTblName: logsFieldsTblName,
|
||||
logAttributeKeysTblName: logAttributeKeysTblName,
|
||||
logResourceKeysTblName: logResourceKeysTblName,
|
||||
relatedMetadataDBName: relatedMetadataDBName,
|
||||
relatedMetadataTblName: relatedMetadataTblName,
|
||||
logger: metadataSettings.Logger(),
|
||||
telemetrystore: telemetrystore,
|
||||
tracesDBName: tracesDBName,
|
||||
tracesFieldsTblName: tracesFieldsTblName,
|
||||
spanAttributesKeysTblName: spanAttributesKeysTblName,
|
||||
indexV3TblName: indexV3TblName,
|
||||
metricsDBName: metricsDBName,
|
||||
metricsFieldsTblName: metricsFieldsTblName,
|
||||
meterDBName: meterDBName,
|
||||
meterFieldsTblName: meterFieldsTblName,
|
||||
logsDBName: logsDBName,
|
||||
logsV2TblName: logsV2TblName,
|
||||
logsFieldsTblName: logsFieldsTblName,
|
||||
logAttributeKeysTblName: logAttributeKeysTblName,
|
||||
logResourceKeysTblName: logResourceKeysTblName,
|
||||
relatedMetadataDBName: relatedMetadataDBName,
|
||||
relatedMetadataTblName: relatedMetadataTblName,
|
||||
columnEvolutionMetadataTblName: columnEvolutionMetadataTblName,
|
||||
jsonColumnMetadata: map[telemetrytypes.Signal]map[telemetrytypes.FieldContext]telemetrytypes.JSONColumnMetadata{
|
||||
telemetrytypes.SignalLogs: {
|
||||
telemetrytypes.FieldContextBody: telemetrytypes.JSONColumnMetadata{
|
||||
@@ -588,6 +592,11 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
|
||||
keys = append(keys, bodyJSONPaths...)
|
||||
complete = complete && finished
|
||||
}
|
||||
|
||||
if _, err := t.updateColumnEvolutionMetadataForKeys(ctx, keys); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
}
|
||||
|
||||
@@ -1027,18 +1036,18 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
|
||||
FieldDataType: fieldValueSelector.FieldDataType,
|
||||
}
|
||||
|
||||
selectColumn, err := t.fm.FieldFor(ctx, key)
|
||||
selectColumn, err := t.fm.FieldFor(ctx, 0, 0, key)
|
||||
|
||||
if err != nil {
|
||||
// we don't have a explicit column to select from the related metadata table
|
||||
// so we will select either from resource_attributes or attributes table
|
||||
// in that order
|
||||
resourceColumn, _ := t.fm.FieldFor(ctx, &telemetrytypes.TelemetryFieldKey{
|
||||
resourceColumn, _ := t.fm.FieldFor(ctx, 0, 0, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key.Name,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
})
|
||||
attributeColumn, _ := t.fm.FieldFor(ctx, &telemetrytypes.TelemetryFieldKey{
|
||||
attributeColumn, _ := t.fm.FieldFor(ctx, 0, 0, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: key.Name,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
@@ -1059,11 +1068,12 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
|
||||
}
|
||||
|
||||
whereClause, err := querybuilder.PrepareWhereClause(fieldValueSelector.ExistingQuery, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: t.logger,
|
||||
FieldMapper: t.fm,
|
||||
ConditionBuilder: t.conditionBuilder,
|
||||
FieldKeys: keys,
|
||||
}, 0, 0)
|
||||
})
|
||||
if err == nil {
|
||||
sb.AddWhereClause(whereClause.WhereClause)
|
||||
} else {
|
||||
@@ -1087,20 +1097,20 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
|
||||
|
||||
// search on attributes
|
||||
key.FieldContext = telemetrytypes.FieldContextAttribute
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, 0, 0, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb)
|
||||
if err == nil {
|
||||
conds = append(conds, cond)
|
||||
}
|
||||
|
||||
// search on resource
|
||||
key.FieldContext = telemetrytypes.FieldContextResource
|
||||
cond, err = t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
|
||||
cond, err = t.conditionBuilder.ConditionFor(ctx, 0, 0, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb)
|
||||
if err == nil {
|
||||
conds = append(conds, cond)
|
||||
}
|
||||
key.FieldContext = origContext
|
||||
} else {
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
|
||||
cond, err := t.conditionBuilder.ConditionFor(ctx, 0, 0, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb)
|
||||
if err == nil {
|
||||
conds = append(conds, cond)
|
||||
}
|
||||
@@ -1844,6 +1854,113 @@ func (t *telemetryMetaStore) fetchMeterSourceMetricsTemporalityAndType(ctx conte
|
||||
return temporalities, types, nil
|
||||
}
|
||||
|
||||
func (k *telemetryMetaStore) fetchEvolutionEntryFromClickHouse(ctx context.Context, selectors []*telemetrytypes.EvolutionSelector) ([]*telemetrytypes.EvolutionEntry, error) {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("signal", "column_name", "column_type", "field_context", "field_name", "version", "release_time")
|
||||
sb.From(fmt.Sprintf("%s.%s", k.relatedMetadataDBName, k.columnEvolutionMetadataTblName))
|
||||
sb.OrderBy("release_time ASC")
|
||||
|
||||
var clauses []string
|
||||
for _, selector := range selectors {
|
||||
var clause string
|
||||
|
||||
if selector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
clause = sb.E("field_context", selector.FieldContext)
|
||||
}
|
||||
|
||||
clause = sb.And(clause,
|
||||
sb.Or(sb.E("field_name", selector.FieldName), sb.E("field_name", "__all__")),
|
||||
)
|
||||
|
||||
clauses = append(clauses, sb.And(sb.E("signal", selector.Signal), clause))
|
||||
}
|
||||
sb.Where(sb.Or(clauses...))
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var entries []*telemetrytypes.EvolutionEntry
|
||||
rows, err := k.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var entry telemetrytypes.EvolutionEntry
|
||||
var releaseTimeNs float64
|
||||
if err := rows.Scan(
|
||||
&entry.Signal,
|
||||
&entry.ColumnName,
|
||||
&entry.ColumnType,
|
||||
&entry.FieldContext,
|
||||
&entry.FieldName,
|
||||
&entry.Version,
|
||||
&releaseTimeNs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Convert nanoseconds to time.Time
|
||||
releaseTime := time.Unix(0, int64(releaseTimeNs))
|
||||
entry.ReleaseTime = releaseTime
|
||||
entries = append(entries, &entry)
|
||||
}
|
||||
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
// Get retrieves all evolutions for the given selectors from DB.
|
||||
func (k *telemetryMetaStore) updateColumnEvolutionMetadataForKeys(ctx context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) (map[string][]*telemetrytypes.EvolutionEntry, error) {
|
||||
|
||||
var metadataKeySelectors []*telemetrytypes.EvolutionSelector
|
||||
for _, keySelector := range keysToUpdate {
|
||||
selector := &telemetrytypes.EvolutionSelector{
|
||||
Signal: keySelector.Signal,
|
||||
FieldContext: keySelector.FieldContext,
|
||||
FieldName: keySelector.Name,
|
||||
}
|
||||
metadataKeySelectors = append(metadataKeySelectors, selector)
|
||||
}
|
||||
|
||||
evolutions, err := k.fetchEvolutionEntryFromClickHouse(ctx, metadataKeySelectors)
|
||||
if err != nil {
|
||||
return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to fetch evolution from clickhouse %s", err.Error())
|
||||
}
|
||||
|
||||
evolutionsByUniqueKey := make(map[string][]*telemetrytypes.EvolutionEntry)
|
||||
for _, evolution := range evolutions {
|
||||
key := &telemetrytypes.EvolutionSelector{
|
||||
Signal: evolution.Signal,
|
||||
FieldContext: evolution.FieldContext,
|
||||
FieldName: evolution.FieldName,
|
||||
}
|
||||
evolutionsByUniqueKey[key.QualifiedName()] = append(evolutionsByUniqueKey[key.QualifiedName()], evolution)
|
||||
}
|
||||
|
||||
if len(keysToUpdate) > 0 {
|
||||
for i, key := range keysToUpdate {
|
||||
selector := &telemetrytypes.EvolutionSelector{
|
||||
Signal: key.Signal,
|
||||
FieldContext: key.FieldContext,
|
||||
FieldName: "__all__",
|
||||
}
|
||||
// first check if there is evolutions that with field name as __all__
|
||||
if keyEvolutions, ok := evolutionsByUniqueKey[selector.QualifiedName()]; ok {
|
||||
keysToUpdate[i].Evolutions = keyEvolutions
|
||||
}
|
||||
// then check for specific field name
|
||||
selector.FieldName = key.Name
|
||||
if keyEvolutions, ok := evolutionsByUniqueKey[selector.QualifiedName()]; ok {
|
||||
keysToUpdate[i].Evolutions = keyEvolutions
|
||||
}
|
||||
}
|
||||
}
|
||||
return evolutionsByUniqueKey, nil
|
||||
}
|
||||
|
||||
// chunkSizeFirstSeenMetricMetadata limits the number of tuples per SQL query to avoid hitting the max_query_size limit.
|
||||
//
|
||||
// Calculation Logic:
|
||||
|
||||
@@ -39,6 +39,7 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
DBName,
|
||||
AttributesMetadataLocalTableName,
|
||||
ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
|
||||
lookupKeys := []telemetrytypes.MetricMetadataLookupKey{
|
||||
|
||||
@@ -38,6 +38,7 @@ func newTestTelemetryMetaStoreTestHelper(store telemetrystore.TelemetryStore) te
|
||||
telemetrylogs.LogResourceKeysTblName,
|
||||
DBName,
|
||||
AttributesMetadataLocalTableName,
|
||||
ColumnEvolutionMetadataTableName,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ const (
|
||||
DBName = "signoz_metadata"
|
||||
AttributesMetadataTableName = "distributed_attributes_metadata"
|
||||
AttributesMetadataLocalTableName = "attributes_metadata"
|
||||
ColumnEvolutionMetadataTableName = "distributed_column_evolution_metadata"
|
||||
PathTypesTableName = otelcollectorconst.DistributedPathTypesTable
|
||||
// Column Evolution table stores promoted paths as (signal, column_name, field_context, field_name); see signoz-otel-collector metadata_migrations.
|
||||
PromotedPathsTableName = "distributed_column_evolution_metadata"
|
||||
|
||||
@@ -122,7 +122,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
stepSec,
|
||||
))
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -147,13 +147,16 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
)
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
}, start, end)
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -205,7 +208,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
))
|
||||
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -233,13 +236,16 @@ func (b *meterQueryStatementBuilder) buildTemporalAggDelta(
|
||||
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
}, start, end)
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -278,7 +284,7 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
stepSec,
|
||||
))
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -300,13 +306,16 @@ func (b *meterQueryStatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
)
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
filterWhere, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
}, start, end)
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
@@ -33,7 +35,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
value = querybuilder.FormatValueForContains(value)
|
||||
}
|
||||
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -41,52 +43,52 @@ func (c *conditionBuilder) conditionFor(
|
||||
// TODO(srikanthccv): use the same data type collision handling when metrics schemas are updated
|
||||
switch v := value.(type) {
|
||||
case float64:
|
||||
tblFieldName = fmt.Sprintf("toFloat64OrNull(%s)", tblFieldName)
|
||||
fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)
|
||||
case []any:
|
||||
if len(v) > 0 && (operator == qbtypes.FilterOperatorBetween || operator == qbtypes.FilterOperatorNotBetween) {
|
||||
if _, ok := v[0].(float64); ok {
|
||||
tblFieldName = fmt.Sprintf("toFloat64OrNull(%s)", tblFieldName)
|
||||
fieldExpression = fmt.Sprintf("toFloat64OrNull(%s)", fieldExpression)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
switch operator {
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.G(tblFieldName, value), nil
|
||||
return sb.G(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThanOrEq:
|
||||
return sb.GE(tblFieldName, value), nil
|
||||
return sb.GE(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorLessThan:
|
||||
return sb.LT(tblFieldName, value), nil
|
||||
return sb.LT(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorLessThanOrEq:
|
||||
return sb.LE(tblFieldName, value), nil
|
||||
return sb.LE(fieldExpression, value), nil
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.Like(tblFieldName, value), nil
|
||||
return sb.Like(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotLike(tblFieldName, value), nil
|
||||
return sb.NotLike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorILike:
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
// between and not between
|
||||
case qbtypes.FilterOperatorBetween:
|
||||
values, ok := value.([]any)
|
||||
@@ -96,7 +98,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.Between(tblFieldName, values[0], values[1]), nil
|
||||
return sb.Between(fieldExpression, values[0], values[1]), nil
|
||||
case qbtypes.FilterOperatorNotBetween:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -105,7 +107,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.NotBetween(tblFieldName, values[0], values[1]), nil
|
||||
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -113,13 +115,13 @@ func (c *conditionBuilder) conditionFor(
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
return sb.In(tblFieldName, values), nil
|
||||
return sb.In(fieldExpression, values), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
return "", qbtypes.ErrInValues
|
||||
}
|
||||
return sb.NotIn(tblFieldName, values), nil
|
||||
return sb.NotIn(fieldExpression, values), nil
|
||||
|
||||
// exists and not exists
|
||||
// in the UI based query builder, `exists` and `not exists` are used for
|
||||
@@ -141,14 +143,14 @@ func (c *conditionBuilder) conditionFor(
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
_ uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
condition, err := c.conditionFor(ctx, key, operator, value, sb)
|
||||
condition, err := c.conditionFor(ctx, startNs, endNs, key, operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -234,7 +234,7 @@ func TestConditionFor(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 0, 0)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &tc.key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
@@ -289,7 +289,7 @@ func TestConditionForMultipleKeys(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var err error
|
||||
for _, key := range tc.keys {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &key, tc.operator, tc.value, sb, 0, 0)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 0, 0, &key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
if err != nil {
|
||||
t.Fatalf("Error getting condition for key %s: %v", key.Name, err)
|
||||
|
||||
@@ -41,65 +41,66 @@ func NewFieldMapper() qbtypes.FieldMapper {
|
||||
return &fieldMapper{}
|
||||
}
|
||||
|
||||
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
func (m *fieldMapper) getColumn(_ context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
|
||||
return timeSeriesV4Columns["labels"], nil
|
||||
return []*schema.Column{timeSeriesV4Columns["labels"]}, nil
|
||||
case telemetrytypes.FieldContextMetric:
|
||||
col, ok := timeSeriesV4Columns[key.Name]
|
||||
if !ok {
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
return []*schema.Column{}, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
return col, nil
|
||||
return []*schema.Column{col}, nil
|
||||
case telemetrytypes.FieldContextUnspecified:
|
||||
col, ok := timeSeriesV4Columns[key.Name]
|
||||
if !ok {
|
||||
// if nothing is found, return labels column
|
||||
// as we keep all the labels in the labels column
|
||||
return timeSeriesV4Columns["labels"], nil
|
||||
return []*schema.Column{timeSeriesV4Columns["labels"]}, nil
|
||||
}
|
||||
return col, nil
|
||||
return []*schema.Column{col}, nil
|
||||
}
|
||||
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
}
|
||||
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
column, err := m.getColumn(ctx, key)
|
||||
func (m *fieldMapper) FieldFor(ctx context.Context, startNs, endNs uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
|
||||
columns, err := m.getColumn(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", column.Name, key.Name), nil
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
case telemetrytypes.FieldContextMetric:
|
||||
return column.Name, nil
|
||||
return columns[0].Name, nil
|
||||
case telemetrytypes.FieldContextUnspecified:
|
||||
if slices.Contains(IntrinsicFields, key.Name) {
|
||||
return column.Name, nil
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", column.Name, key.Name), nil
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
}
|
||||
|
||||
return column.Name, nil
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
fieldExpression, err := m.FieldFor(ctx, startNs, endNs, field)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
}
|
||||
|
||||
@@ -123,13 +123,13 @@ func TestGetColumn(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
col, err := fm.ColumnFor(ctx, &tc.key)
|
||||
col, err := fm.ColumnFor(ctx, 0, 0, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.expectedCol, col)
|
||||
assert.Equal(t, tc.expectedCol, col[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -207,7 +207,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
|
||||
@@ -269,13 +269,16 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
FullTextColumn: &telemetrytypes.TelemetryFieldKey{Name: "labels"},
|
||||
Variables: variables,
|
||||
}, start, end)
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
@@ -286,7 +289,7 @@ func (b *MetricQueryStatementBuilder) buildTimeSeriesCTE(
|
||||
|
||||
sb.Select("fingerprint")
|
||||
for _, g := range query.GroupBy {
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, &g.TelemetryFieldKey, keys)
|
||||
col, err := b.fm.ColumnExpressionFor(ctx, start, end, &g.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
|
||||
@@ -29,6 +29,8 @@ func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
|
||||
|
||||
func (c *conditionBuilder) conditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
@@ -40,13 +42,13 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
|
||||
// first, locate the raw column type (so we can choose the right EXISTS logic)
|
||||
column, err := c.fm.ColumnFor(ctx, key)
|
||||
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// then ask the mapper for the actual SQL reference
|
||||
tblFieldName, err := c.fm.FieldFor(ctx, key)
|
||||
fieldExpression, err := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -67,48 +69,48 @@ func (c *conditionBuilder) conditionFor(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tblFieldName, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, tblFieldName, operator)
|
||||
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
|
||||
}
|
||||
|
||||
// regular operators
|
||||
switch operator {
|
||||
// regular operators
|
||||
case qbtypes.FilterOperatorEqual:
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotEqual:
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThan:
|
||||
return sb.G(tblFieldName, value), nil
|
||||
return sb.G(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorGreaterThanOrEq:
|
||||
return sb.GE(tblFieldName, value), nil
|
||||
return sb.GE(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorLessThan:
|
||||
return sb.LT(tblFieldName, value), nil
|
||||
return sb.LT(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorLessThanOrEq:
|
||||
return sb.LE(tblFieldName, value), nil
|
||||
return sb.LE(fieldExpression, value), nil
|
||||
|
||||
// like and not like
|
||||
case qbtypes.FilterOperatorLike:
|
||||
return sb.Like(tblFieldName, value), nil
|
||||
return sb.Like(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotLike:
|
||||
return sb.NotLike(tblFieldName, value), nil
|
||||
return sb.NotLike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorILike:
|
||||
return sb.ILike(tblFieldName, value), nil
|
||||
return sb.ILike(fieldExpression, value), nil
|
||||
case qbtypes.FilterOperatorNotILike:
|
||||
return sb.NotILike(tblFieldName, value), nil
|
||||
return sb.NotILike(fieldExpression, value), nil
|
||||
|
||||
case qbtypes.FilterOperatorContains:
|
||||
return sb.ILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
case qbtypes.FilterOperatorNotContains:
|
||||
return sb.NotILike(tblFieldName, fmt.Sprintf("%%%s%%", value)), nil
|
||||
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
|
||||
|
||||
case qbtypes.FilterOperatorRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
case qbtypes.FilterOperatorNotRegexp:
|
||||
// Note: Escape $$ to $$$$ to avoid sqlbuilder interpreting materialized $ signs
|
||||
// Only needed because we are using sprintf instead of sb.Match (not implemented in sqlbuilder)
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(tblFieldName), sb.Var(value)), nil
|
||||
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
|
||||
// between and not between
|
||||
case qbtypes.FilterOperatorBetween:
|
||||
values, ok := value.([]any)
|
||||
@@ -118,7 +120,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.Between(tblFieldName, values[0], values[1]), nil
|
||||
return sb.Between(fieldExpression, values[0], values[1]), nil
|
||||
case qbtypes.FilterOperatorNotBetween:
|
||||
values, ok := value.([]any)
|
||||
if !ok {
|
||||
@@ -127,7 +129,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
if len(values) != 2 {
|
||||
return "", qbtypes.ErrBetweenValues
|
||||
}
|
||||
return sb.NotBetween(tblFieldName, values[0], values[1]), nil
|
||||
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
|
||||
|
||||
// in and not in
|
||||
case qbtypes.FilterOperatorIn:
|
||||
@@ -138,7 +140,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using IN, we use `=` + `OR` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.E(tblFieldName, value))
|
||||
conditions = append(conditions, sb.E(fieldExpression, value))
|
||||
}
|
||||
return sb.Or(conditions...), nil
|
||||
case qbtypes.FilterOperatorNotIn:
|
||||
@@ -149,7 +151,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
// instead of using NOT IN, we use `!=` + `AND` to make use of index
|
||||
conditions := []string{}
|
||||
for _, value := range values {
|
||||
conditions = append(conditions, sb.NE(tblFieldName, value))
|
||||
conditions = append(conditions, sb.NE(fieldExpression, value))
|
||||
}
|
||||
return sb.And(conditions...), nil
|
||||
|
||||
@@ -159,30 +161,30 @@ func (c *conditionBuilder) conditionFor(
|
||||
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
|
||||
|
||||
var value any
|
||||
switch column.Type.GetType() {
|
||||
switch columns[0].Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.IsNotNull(tblFieldName), nil
|
||||
return sb.IsNotNull(fieldExpression), nil
|
||||
} else {
|
||||
return sb.IsNull(tblFieldName), nil
|
||||
return sb.IsNull(fieldExpression), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumFixedString,
|
||||
schema.ColumnTypeEnumDateTime64:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
} else {
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumLowCardinality:
|
||||
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
switch elementType := columns[0].Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
|
||||
case schema.ColumnTypeEnumString:
|
||||
value = ""
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
}
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
|
||||
}
|
||||
@@ -195,19 +197,19 @@ func (c *conditionBuilder) conditionFor(
|
||||
schema.ColumnTypeEnumBool:
|
||||
value = 0
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return sb.NE(tblFieldName, value), nil
|
||||
return sb.NE(fieldExpression, value), nil
|
||||
} else {
|
||||
return sb.E(tblFieldName, value), nil
|
||||
return sb.E(fieldExpression, value), nil
|
||||
}
|
||||
case schema.ColumnTypeEnumMap:
|
||||
keyType := column.Type.(schema.MapColumnType).KeyType
|
||||
keyType := columns[0].Type.(schema.MapColumnType).KeyType
|
||||
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, columns[0].Type)
|
||||
}
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
switch valueType := columns[0].Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", columns[0].Name, key.Name)
|
||||
if key.Materialized {
|
||||
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
|
||||
}
|
||||
@@ -220,7 +222,7 @@ func (c *conditionBuilder) conditionFor(
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", columns[0].Type)
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
@@ -228,25 +230,25 @@ func (c *conditionBuilder) conditionFor(
|
||||
|
||||
func (c *conditionBuilder) ConditionFor(
|
||||
ctx context.Context,
|
||||
startNs uint64,
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
startNs uint64,
|
||||
_ uint64,
|
||||
) (string, error) {
|
||||
if c.isSpanScopeField(key.Name) {
|
||||
return c.buildSpanScopeCondition(key, operator, value, startNs)
|
||||
}
|
||||
|
||||
condition, err := c.conditionFor(ctx, key, operator, value, sb)
|
||||
condition, err := c.conditionFor(ctx, startNs, endNs, key, operator, value, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if operator.AddDefaultExistsFilter() {
|
||||
// skip adding exists filter for intrinsic fields
|
||||
field, _ := c.fm.FieldFor(ctx, key)
|
||||
field, _ := c.fm.FieldFor(ctx, startNs, endNs, key)
|
||||
if slices.Contains(maps.Keys(IntrinsicFields), field) ||
|
||||
slices.Contains(maps.Keys(IntrinsicFieldsDeprecated), field) ||
|
||||
slices.Contains(maps.Keys(CalculatedFields), field) ||
|
||||
@@ -254,7 +256,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return condition, nil
|
||||
}
|
||||
|
||||
existsCondition, err := c.conditionFor(ctx, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
existsCondition, err := c.conditionFor(ctx, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
@@ -289,7 +289,7 @@ func TestConditionFor(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, &tc.key, tc.operator, tc.value, sb, 1761437108000000000, 1761458708000000000)
|
||||
cond, err := conditionBuilder.ConditionFor(ctx, 1761437108000000000, 1761458708000000000, &tc.key, tc.operator, tc.value, sb)
|
||||
sb.Where(cond)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
|
||||
@@ -169,23 +169,24 @@ func NewFieldMapper() *defaultFieldMapper {
|
||||
|
||||
func (m *defaultFieldMapper) getColumn(
|
||||
_ context.Context,
|
||||
_, _ uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (*schema.Column, error) {
|
||||
) ([]*schema.Column, error) {
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
return indexV3Columns["resource"], nil
|
||||
return []*schema.Column{indexV3Columns["resource"]}, nil
|
||||
case telemetrytypes.FieldContextScope:
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
return []*schema.Column{}, qbtypes.ErrColumnNotFound
|
||||
case telemetrytypes.FieldContextAttribute:
|
||||
switch key.FieldDataType {
|
||||
case telemetrytypes.FieldDataTypeString:
|
||||
return indexV3Columns["attributes_string"], nil
|
||||
return []*schema.Column{indexV3Columns["attributes_string"]}, nil
|
||||
case telemetrytypes.FieldDataTypeInt64,
|
||||
telemetrytypes.FieldDataTypeFloat64,
|
||||
telemetrytypes.FieldDataTypeNumber:
|
||||
return indexV3Columns["attributes_number"], nil
|
||||
return []*schema.Column{indexV3Columns["attributes_number"]}, nil
|
||||
case telemetrytypes.FieldDataTypeBool:
|
||||
return indexV3Columns["attributes_bool"], nil
|
||||
return []*schema.Column{indexV3Columns["attributes_bool"]}, nil
|
||||
}
|
||||
case telemetrytypes.FieldContextSpan, telemetrytypes.FieldContextUnspecified:
|
||||
/*
|
||||
@@ -196,7 +197,7 @@ func (m *defaultFieldMapper) getColumn(
|
||||
// Check if this is a span scope field
|
||||
if strings.ToLower(key.Name) == SpanSearchScopeRoot || strings.ToLower(key.Name) == SpanSearchScopeEntryPoint {
|
||||
// The actual SQL will be generated in the condition builder
|
||||
return &schema.Column{Name: key.Name, Type: schema.ColumnTypeBool}, nil
|
||||
return []*schema.Column{{Name: key.Name, Type: schema.ColumnTypeBool}}, nil
|
||||
}
|
||||
|
||||
// TODO(srikanthccv): remove this when it's safe to remove
|
||||
@@ -210,18 +211,18 @@ func (m *defaultFieldMapper) getColumn(
|
||||
if _, ok := CalculatedFieldsDeprecated[key.Name]; ok {
|
||||
// Check if we have a mapping for the deprecated calculated field
|
||||
if col, ok := indexV3Columns[oldToNew[key.Name]]; ok {
|
||||
return col, nil
|
||||
return []*schema.Column{col}, nil
|
||||
}
|
||||
}
|
||||
if _, ok := IntrinsicFieldsDeprecated[key.Name]; ok {
|
||||
// Check if we have a mapping for the deprecated intrinsic field
|
||||
if col, ok := indexV3Columns[oldToNew[key.Name]]; ok {
|
||||
return col, nil
|
||||
return []*schema.Column{col}, nil
|
||||
}
|
||||
}
|
||||
|
||||
if col, ok := indexV3Columns[key.Name]; ok {
|
||||
return col, nil
|
||||
return []*schema.Column{col}, nil
|
||||
}
|
||||
}
|
||||
return nil, qbtypes.ErrColumnNotFound
|
||||
@@ -229,15 +230,17 @@ func (m *defaultFieldMapper) getColumn(
|
||||
|
||||
func (m *defaultFieldMapper) ColumnFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (*schema.Column, error) {
|
||||
return m.getColumn(ctx, key)
|
||||
) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, startNs, endNs, key)
|
||||
}
|
||||
|
||||
// FieldFor returns the table field name for the given key if it exists
|
||||
// otherwise it returns qbtypes.ErrColumnNotFound
|
||||
func (m *defaultFieldMapper) FieldFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
// Special handling for span scope fields
|
||||
@@ -247,10 +250,14 @@ func (m *defaultFieldMapper) FieldFor(
|
||||
return key.Name, nil
|
||||
}
|
||||
|
||||
column, err := m.getColumn(ctx, key)
|
||||
columns, err := m.getColumn(ctx, startNs, endNs, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(columns) != 1 {
|
||||
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
|
||||
}
|
||||
column := columns[0]
|
||||
|
||||
switch column.Type.GetType() {
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
@@ -310,11 +317,12 @@ func (m *defaultFieldMapper) FieldFor(
|
||||
// if it exists otherwise it returns qbtypes.ErrColumnNotFound
|
||||
func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
ctx context.Context,
|
||||
startNs, endNs uint64,
|
||||
field *telemetrytypes.TelemetryFieldKey,
|
||||
keys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
) (string, error) {
|
||||
|
||||
colName, err := m.FieldFor(ctx, field)
|
||||
fieldExpression, err := m.FieldFor(ctx, startNs, endNs, field)
|
||||
if errors.Is(err, qbtypes.ErrColumnNotFound) {
|
||||
// the key didn't have the right context to be added to the query
|
||||
// we try to use the context we know of
|
||||
@@ -324,7 +332,7 @@ func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
if _, ok := indexV3Columns[field.Name]; ok {
|
||||
// if it is, attach the column name directly
|
||||
field.FieldContext = telemetrytypes.FieldContextSpan
|
||||
colName, _ = m.FieldFor(ctx, field)
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, field)
|
||||
} else {
|
||||
// - the context is not provided
|
||||
// - there are not keys for the field
|
||||
@@ -342,17 +350,17 @@ func (m *defaultFieldMapper) ColumnExpressionFor(
|
||||
}
|
||||
} else if len(keysForField) == 1 {
|
||||
// we have a single key for the field, use it
|
||||
colName, _ = m.FieldFor(ctx, keysForField[0])
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, keysForField[0])
|
||||
} else {
|
||||
// select any non-empty value from the keys
|
||||
args := []string{}
|
||||
for _, key := range keysForField {
|
||||
colName, _ = m.FieldFor(ctx, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", colName, colName))
|
||||
fieldExpression, _ = m.FieldFor(ctx, startNs, endNs, key)
|
||||
args = append(args, fmt.Sprintf("toString(%s) != '', toString(%s)", fieldExpression, fieldExpression))
|
||||
}
|
||||
colName = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
fieldExpression = fmt.Sprintf("multiIf(%s, NULL)", strings.Join(args, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(colName), field.Name), nil
|
||||
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ func TestGetFieldKeyName(t *testing.T) {
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
result, err := fm.FieldFor(ctx, &tc.key)
|
||||
result, err := fm.FieldFor(ctx, 0, 0, &tc.key)
|
||||
|
||||
if tc.expectedError != nil {
|
||||
assert.Equal(t, tc.expectedError, err)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package telemetrytraces
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
@@ -75,13 +76,16 @@ func TestSpanScopeFilterExpression(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextSpan,
|
||||
}}
|
||||
|
||||
whereClause, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
whereClause, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
FieldKeys: fieldKeys,
|
||||
Builder: sb,
|
||||
}, tt.startNs, 1761458708000000000)
|
||||
StartNs: tt.startNs,
|
||||
EndNs: 1761458708000000000,
|
||||
})
|
||||
|
||||
if tt.expectError {
|
||||
assert.Error(t, err)
|
||||
@@ -142,13 +146,16 @@ func TestSpanScopeWithResourceFilter(t *testing.T) {
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}}
|
||||
|
||||
_, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
_, err := querybuilder.PrepareWhereClause(tt.expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: context.Background(),
|
||||
Logger: instrumentationtest.New().Logger(),
|
||||
FieldMapper: fm,
|
||||
ConditionBuilder: cb,
|
||||
FieldKeys: fieldKeys,
|
||||
SkipResourceFilter: false, // This would be set by the statement builder
|
||||
}, 1761437108000000000, 1761458708000000000)
|
||||
StartNs: 1761437108000000000,
|
||||
EndNs: 1761458708000000000,
|
||||
})
|
||||
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
@@ -296,7 +296,7 @@ func (b *traceQueryStatementBuilder) buildListQuery(
|
||||
|
||||
// TODO: should we deprecate `SelectFields` and return everything from a span like we do for logs?
|
||||
for _, field := range query.SelectFields {
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &field, keys)
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &field, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -314,7 +314,7 @@ func (b *traceQueryStatementBuilder) buildListQuery(
|
||||
|
||||
// Add order by
|
||||
for _, orderBy := range query.Order {
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -498,7 +498,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
// Keep original column expressions so we can build the tuple
|
||||
fieldNames := make([]string, 0, len(query.GroupBy))
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -512,7 +512,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
|
||||
allAggChArgs := make([]any, 0)
|
||||
for i, agg := range query.Aggregations {
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, agg.Expression,
|
||||
ctx, start, end, agg.Expression,
|
||||
uint64(query.StepInterval.Seconds()),
|
||||
keys,
|
||||
)
|
||||
@@ -640,7 +640,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
|
||||
|
||||
var allGroupByArgs []any
|
||||
for _, gb := range query.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -657,7 +657,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
|
||||
for idx := range query.Aggregations {
|
||||
aggExpr := query.Aggregations[idx]
|
||||
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(
|
||||
ctx, aggExpr.Expression,
|
||||
ctx, start, end, aggExpr.Expression,
|
||||
rateInterval,
|
||||
keys,
|
||||
)
|
||||
@@ -729,7 +729,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
|
||||
|
||||
// buildFilterCondition builds SQL condition from filter expression
|
||||
func (b *traceQueryStatementBuilder) addFilterCondition(
|
||||
_ context.Context,
|
||||
ctx context.Context,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
start, end uint64,
|
||||
query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation],
|
||||
@@ -743,13 +743,16 @@ func (b *traceQueryStatementBuilder) addFilterCondition(
|
||||
if query.Filter != nil && query.Filter.Expression != "" {
|
||||
// add filter expression
|
||||
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.logger,
|
||||
FieldMapper: b.fm,
|
||||
ConditionBuilder: b.cb,
|
||||
FieldKeys: keys,
|
||||
SkipResourceFilter: true,
|
||||
Variables: variables,
|
||||
}, start, end)
|
||||
StartNs: start,
|
||||
EndNs: end,
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -234,12 +234,15 @@ func (b *traceOperatorCTEBuilder) buildQueryCTE(ctx context.Context, queryName s
|
||||
filterWhereClause, err := querybuilder.PrepareWhereClause(
|
||||
query.Filter.Expression,
|
||||
querybuilder.FilterExprVisitorOpts{
|
||||
Context: ctx,
|
||||
Logger: b.stmtBuilder.logger,
|
||||
FieldMapper: b.stmtBuilder.fm,
|
||||
ConditionBuilder: b.stmtBuilder.cb,
|
||||
FieldKeys: keys,
|
||||
SkipResourceFilter: true,
|
||||
}, b.start, b.end,
|
||||
StartNs: b.start,
|
||||
EndNs: b.end,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
b.stmtBuilder.logger.ErrorContext(ctx, "Failed to prepare where clause", errors.Attr(err), slog.String("filter", query.Filter.Expression))
|
||||
@@ -452,7 +455,7 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
|
||||
if selectedFields[field.Name] {
|
||||
continue
|
||||
}
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, &field, keys)
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.start, b.end, &field, keys)
|
||||
if err != nil {
|
||||
b.stmtBuilder.logger.WarnContext(ctx, "failed to map select field",
|
||||
slog.String("field", field.Name), errors.Attr(err))
|
||||
@@ -467,7 +470,7 @@ func (b *traceOperatorCTEBuilder) buildListQuery(ctx context.Context, selectFrom
|
||||
// Add order by support using ColumnExpressionFor
|
||||
orderApplied := false
|
||||
for _, orderBy := range b.operator.Order {
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
colExpr, err := b.stmtBuilder.fm.ColumnExpressionFor(ctx, b.start, b.end, &orderBy.Key.TelemetryFieldKey, keys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -549,6 +552,8 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
|
||||
for _, gb := range b.operator.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
&gb.TelemetryFieldKey,
|
||||
b.stmtBuilder.fm,
|
||||
b.stmtBuilder.cb,
|
||||
@@ -573,6 +578,8 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
|
||||
for i, agg := range b.operator.Aggregations {
|
||||
rewritten, chArgs, err := b.stmtBuilder.aggExprRewriter.Rewrite(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
agg.Expression,
|
||||
uint64(b.operator.StepInterval.Seconds()),
|
||||
keys,
|
||||
@@ -658,6 +665,8 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
|
||||
for _, gb := range b.operator.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
&gb.TelemetryFieldKey,
|
||||
b.stmtBuilder.fm,
|
||||
b.stmtBuilder.cb,
|
||||
@@ -684,6 +693,8 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
|
||||
for i, agg := range b.operator.Aggregations {
|
||||
rewritten, chArgs, err := b.stmtBuilder.aggExprRewriter.Rewrite(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
agg.Expression,
|
||||
rateInterval,
|
||||
keys,
|
||||
@@ -797,6 +808,8 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
|
||||
for _, gb := range b.operator.GroupBy {
|
||||
expr, args, err := querybuilder.CollisionHandledFinalExpr(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
&gb.TelemetryFieldKey,
|
||||
b.stmtBuilder.fm,
|
||||
b.stmtBuilder.cb,
|
||||
@@ -821,6 +834,8 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
|
||||
for i, agg := range b.operator.Aggregations {
|
||||
rewritten, chArgs, err := b.stmtBuilder.aggExprRewriter.Rewrite(
|
||||
ctx,
|
||||
b.start,
|
||||
b.end,
|
||||
agg.Expression,
|
||||
uint64((b.end-b.start)/querybuilder.NsToSeconds),
|
||||
keys,
|
||||
|
||||
@@ -2,12 +2,13 @@ package telemetrytraces
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type traceOperatorStatementBuilder struct {
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package cloudintegrationtypes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
@@ -33,8 +35,6 @@ type GettableAccounts struct {
|
||||
Accounts []*Account `json:"accounts" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
type GettableAccount = Account
|
||||
|
||||
type UpdatableAccount struct {
|
||||
Config *AccountConfig `json:"config" required:"true" nullable:"false"`
|
||||
}
|
||||
@@ -42,3 +42,120 @@ type UpdatableAccount struct {
|
||||
type AWSAccountConfig struct {
|
||||
Regions []string `json:"regions" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
func NewAccount(orgID valuer.UUID, provider CloudProviderType, config *AccountConfig) *Account {
|
||||
return &Account{
|
||||
Identifiable: types.Identifiable{
|
||||
ID: valuer.GenerateUUID(),
|
||||
},
|
||||
TimeAuditable: types.TimeAuditable{
|
||||
CreatedAt: time.Now(),
|
||||
UpdatedAt: time.Now(),
|
||||
},
|
||||
OrgID: orgID,
|
||||
Provider: provider,
|
||||
Config: config,
|
||||
}
|
||||
}
|
||||
|
||||
func NewAccountFromStorable(storableAccount *StorableCloudIntegration) (*Account, error) {
|
||||
// config can not be empty
|
||||
if storableAccount.Config == "" {
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "config is empty for account with id: %s", storableAccount.ID)
|
||||
}
|
||||
|
||||
account := &Account{
|
||||
Identifiable: storableAccount.Identifiable,
|
||||
TimeAuditable: storableAccount.TimeAuditable,
|
||||
ProviderAccountID: storableAccount.AccountID,
|
||||
Provider: storableAccount.Provider,
|
||||
RemovedAt: storableAccount.RemovedAt,
|
||||
OrgID: storableAccount.OrgID,
|
||||
Config: new(AccountConfig),
|
||||
}
|
||||
|
||||
switch storableAccount.Provider {
|
||||
case CloudProviderTypeAWS:
|
||||
awsConfig := new(AWSAccountConfig)
|
||||
err := json.Unmarshal([]byte(storableAccount.Config), awsConfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
account.Config.AWS = awsConfig
|
||||
}
|
||||
|
||||
if storableAccount.LastAgentReport != nil {
|
||||
account.AgentReport = &AgentReport{
|
||||
TimestampMillis: storableAccount.LastAgentReport.TimestampMillis,
|
||||
Data: storableAccount.LastAgentReport.Data,
|
||||
}
|
||||
}
|
||||
|
||||
return account, nil
|
||||
}
|
||||
|
||||
func NewAccountsFromStorables(storableAccounts []*StorableCloudIntegration) ([]*Account, error) {
|
||||
accounts := make([]*Account, 0, len(storableAccounts))
|
||||
for _, storableAccount := range storableAccounts {
|
||||
account, err := NewAccountFromStorable(storableAccount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accounts = append(accounts, account)
|
||||
}
|
||||
|
||||
return accounts, nil
|
||||
}
|
||||
|
||||
func (account *Account) Update(config *AccountConfig) error {
|
||||
if account.RemovedAt != nil {
|
||||
return errors.New(errors.TypeUnsupported, ErrCodeCloudIntegrationRemoved, "this operation is not supported for a removed cloud integration account")
|
||||
}
|
||||
account.Config = config
|
||||
account.UpdatedAt = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
func NewAccountConfigFromPostableArtifact(provider CloudProviderType, artifact *PostableConnectionArtifact) (*AccountConfig, error) {
|
||||
switch provider {
|
||||
case CloudProviderTypeAWS:
|
||||
if artifact.Aws == nil {
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "AWS artifact is nil")
|
||||
}
|
||||
return &AccountConfig{
|
||||
AWS: &AWSAccountConfig{
|
||||
Regions: artifact.Aws.Regions,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "unsupported provider type")
|
||||
}
|
||||
|
||||
func NewArtifactRequestFromPostableArtifact(provider CloudProviderType, artifact *PostableConnectionArtifact) (*ConnectionArtifactRequest, error) {
|
||||
switch provider {
|
||||
case CloudProviderTypeAWS:
|
||||
if artifact.Aws == nil {
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "AWS artifact is nil")
|
||||
}
|
||||
return &ConnectionArtifactRequest{
|
||||
Aws: &AWSConnectionArtifactRequest{
|
||||
DeploymentRegion: artifact.Aws.DeploymentRegion,
|
||||
Regions: artifact.Aws.Regions,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "unsupported provider type")
|
||||
}
|
||||
|
||||
// ToJSON return JSON bytes for the provider's config
|
||||
// thats why not naming it MarshalJSON(), as it will interfere with default JSON marshalling of AccountConfig struct.
|
||||
// NOTE: this entertains first non-null provider's config
|
||||
func (config *AccountConfig) ToJSON() ([]byte, error) {
|
||||
if config.AWS != nil {
|
||||
return json.Marshal(config.AWS)
|
||||
}
|
||||
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "no provider account config found")
|
||||
}
|
||||
|
||||
@@ -13,8 +13,12 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCodeUnsupported = errors.MustNewCode("cloud_integration_unsupported")
|
||||
ErrCodeCloudIntegrationNotFound = errors.MustNewCode("cloud_integration_not_found")
|
||||
ErrCodeCloudIntegrationAlreadyExists = errors.MustNewCode("cloud_integration_already_exists")
|
||||
ErrCodeCloudIntegrationAlreadyConnected = errors.MustNewCode("cloud_integration_already_connected")
|
||||
ErrCodeCloudIntegrationRemoved = errors.MustNewCode("cloud_integration_removed")
|
||||
ErrCodeInvalidInput = errors.MustNewCode("cloud_integration_invalid_input")
|
||||
ErrCodeCloudIntegrationServiceNotFound = errors.MustNewCode("cloud_integration_service_not_found")
|
||||
ErrCodeCloudIntegrationServiceAlreadyExists = errors.MustNewCode("cloud_integration_service_already_exists")
|
||||
)
|
||||
@@ -68,10 +72,6 @@ func (r *StorableAgentReport) Scan(src any) error {
|
||||
|
||||
// Value creates value to be stored in DB.
|
||||
func (r *StorableAgentReport) Value() (driver.Value, error) {
|
||||
if r == nil {
|
||||
return nil, errors.NewInternalf(errors.CodeInternal, "agent report is nil")
|
||||
}
|
||||
|
||||
serialized, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return nil, errors.WrapInternalf(
|
||||
@@ -81,3 +81,31 @@ func (r *StorableAgentReport) Value() (driver.Value, error) {
|
||||
// Return as string instead of []byte to ensure PostgreSQL stores as text, not bytes
|
||||
return string(serialized), nil
|
||||
}
|
||||
|
||||
func NewStorableCloudIntegration(account *Account) (*StorableCloudIntegration, error) {
|
||||
configBytes, err := account.Config.ToJSON()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &StorableCloudIntegration{
|
||||
Identifiable: account.Identifiable,
|
||||
TimeAuditable: account.TimeAuditable,
|
||||
Provider: account.Provider,
|
||||
Config: string(configBytes),
|
||||
AccountID: nil, // updated during agent check in
|
||||
OrgID: account.OrgID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewStorableCloudIntegrationService creates a new StorableCloudIntegrationService with
|
||||
// generated ID and timestamps from a CloudIntegrationService and its serialized config JSON.
|
||||
func NewStorableCloudIntegrationService(svc *CloudIntegrationService, configJSON string) *StorableCloudIntegrationService {
|
||||
return &StorableCloudIntegrationService{
|
||||
Identifiable: svc.Identifiable,
|
||||
TimeAuditable: svc.TimeAuditable,
|
||||
Type: svc.Type,
|
||||
Config: configJSON,
|
||||
CloudIntegrationID: svc.CloudIntegrationID,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package cloudintegrationtypes
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
@@ -39,3 +41,29 @@ func NewCloudProvider(provider string) (CloudProviderType, error) {
|
||||
return CloudProviderType{}, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider)
|
||||
}
|
||||
}
|
||||
|
||||
func GetCloudProviderEmail(provider CloudProviderType) (valuer.Email, error) {
|
||||
switch provider {
|
||||
case CloudProviderTypeAWS:
|
||||
return AWSIntegrationUserEmail, nil
|
||||
case CloudProviderTypeAzure:
|
||||
return AzureIntegrationUserEmail, nil
|
||||
default:
|
||||
return valuer.Email{}, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
|
||||
}
|
||||
}
|
||||
|
||||
func NewIngestionKeyName(provider CloudProviderType) string {
|
||||
return fmt.Sprintf("%s-integration", provider.StringValue())
|
||||
}
|
||||
|
||||
func NewIntegrationUserDisplayName(provider CloudProviderType) string {
|
||||
return fmt.Sprintf("%s-integration", provider.StringValue())
|
||||
}
|
||||
|
||||
// NewAPIKeyName returns API key name for cloud integration provider
|
||||
// TODO: figure out way to migrate API keys to have similar naming convention as ingestion key
|
||||
// ie. "{cloud-provider}-integration", and then remove this function.
|
||||
func NewAPIKeyName(provider CloudProviderType) string {
|
||||
return fmt.Sprintf("%s integration", provider.StringValue())
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package cloudintegrationtypes
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
@@ -33,8 +34,8 @@ type GettableAccountWithArtifact struct {
|
||||
}
|
||||
|
||||
type AgentCheckInRequest struct {
|
||||
ProviderAccountID string `json:"providerAccountId" required:"false"`
|
||||
CloudIntegrationID string `json:"cloudIntegrationId" required:"false"`
|
||||
ProviderAccountID string `json:"providerAccountId" required:"false"`
|
||||
CloudIntegrationID valuer.UUID `json:"cloudIntegrationId" required:"false"`
|
||||
|
||||
Data map[string]any `json:"data" required:"true" nullable:"true"`
|
||||
}
|
||||
@@ -67,8 +68,8 @@ type GettableAgentCheckInResponse struct {
|
||||
// IntegrationConfig older integration config struct for backward compatibility,
|
||||
// this will be eventually removed once agents are updated to use new struct.
|
||||
type IntegrationConfig struct {
|
||||
EnabledRegions []string `json:"enabled_regions" required:"true" nullable:"false"` // backward compatible
|
||||
Telemetry *AWSCollectionStrategy `json:"telemetry" required:"true" nullable:"false"` // backward compatible
|
||||
EnabledRegions []string `json:"enabled_regions" required:"true" nullable:"false"` // backward compatible
|
||||
Telemetry *OldAWSCollectionStrategy `json:"telemetry" required:"true" nullable:"false"` // backward compatible
|
||||
}
|
||||
|
||||
type ProviderIntegrationConfig struct {
|
||||
@@ -79,3 +80,47 @@ type AWSIntegrationConfig struct {
|
||||
EnabledRegions []string `json:"enabledRegions" required:"true" nullable:"false"`
|
||||
Telemetry *AWSCollectionStrategy `json:"telemetry" required:"true" nullable:"false"`
|
||||
}
|
||||
|
||||
type SignozCredentials struct {
|
||||
SigNozAPIURL string
|
||||
SigNozAPIKey string // PAT
|
||||
IngestionURL string
|
||||
IngestionKey string
|
||||
}
|
||||
|
||||
// NewGettableAgentCheckInResponse constructs a backward-compatible response from an AgentCheckInResponse.
|
||||
// It populates the old snake_case fields (account_id, cloud_account_id, integration_config, removed_at)
|
||||
// from the new camelCase fields so older agents continue to work unchanged.
|
||||
// The provider parameter controls which provider-specific block is mapped into the legacy integration_config.
|
||||
func NewGettableAgentCheckInResponse(provider CloudProviderType, resp *AgentCheckInResponse) *GettableAgentCheckInResponse {
|
||||
gettable := &GettableAgentCheckInResponse{
|
||||
AccountID: resp.CloudIntegrationID,
|
||||
CloudAccountID: resp.ProviderAccountID,
|
||||
OlderRemovedAt: resp.RemovedAt,
|
||||
AgentCheckInResponse: *resp,
|
||||
}
|
||||
|
||||
switch provider {
|
||||
case CloudProviderTypeAWS:
|
||||
gettable.OlderIntegrationConfig = awsOlderIntegrationConfig(resp.IntegrationConfig)
|
||||
}
|
||||
|
||||
return gettable
|
||||
}
|
||||
|
||||
// Validate checks that the request uses either old fields (account_id, cloud_account_id) or
|
||||
// new fields (cloudIntegrationId, providerAccountId), never a mix of both.
|
||||
func (req *PostableAgentCheckInRequest) Validate() error {
|
||||
hasOldFields := req.ID != "" || req.AccountID != ""
|
||||
hasNewFields := !req.CloudIntegrationID.IsZero() || req.ProviderAccountID != ""
|
||||
|
||||
if hasOldFields && hasNewFields {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput,
|
||||
"request must use either old fields (account_id, cloud_account_id) or new fields (cloudIntegrationId, providerAccountId), not both")
|
||||
}
|
||||
if !hasOldFields && !hasNewFields {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput,
|
||||
"request must provide either old fields (account_id, cloud_account_id) or new fields (cloudIntegrationId, providerAccountId)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 125 KiB |
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user