Compare commits

..

2 Commits

56 changed files with 170 additions and 5560 deletions

View File

@@ -17,15 +17,11 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/gateway"
"github.com/SigNoz/signoz/pkg/gateway/noopgateway"
"github.com/SigNoz/signoz/pkg/global"
"github.com/SigNoz/signoz/pkg/licensing"
"github.com/SigNoz/signoz/pkg/licensing/nooplicensing"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
"github.com/SigNoz/signoz/pkg/modules/dashboard"
"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/querier"
"github.com/SigNoz/signoz/pkg/query-service/app"
"github.com/SigNoz/signoz/pkg/queryparser"
@@ -33,7 +29,6 @@ import (
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/version"
"github.com/SigNoz/signoz/pkg/zeus"
"github.com/SigNoz/signoz/pkg/zeus/noopzeus"
@@ -101,9 +96,6 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
return querier.NewHandler(ps, q, a)
},
func(_ cloudintegrationtypes.Store, _ global.Global, _ zeus.Zeus, _ gateway.Gateway, _ licensing.Licensing, _ serviceaccount.Module, _ cloudintegration.Config) (cloudintegration.Module, error) {
return implcloudintegration.NewModule(), nil
},
)
if err != nil {
logger.ErrorContext(ctx, "failed to create signoz", errors.Attr(err))

View File

@@ -16,8 +16,6 @@ import (
"github.com/SigNoz/signoz/ee/gateway/httpgateway"
enterpriselicensing "github.com/SigNoz/signoz/ee/licensing"
"github.com/SigNoz/signoz/ee/licensing/httplicensing"
"github.com/SigNoz/signoz/ee/modules/cloudintegration/implcloudintegration"
"github.com/SigNoz/signoz/ee/modules/cloudintegration/implcloudintegration/implcloudprovider"
"github.com/SigNoz/signoz/ee/modules/dashboard/impldashboard"
eequerier "github.com/SigNoz/signoz/ee/querier"
enterpriseapp "github.com/SigNoz/signoz/ee/query-service/app"
@@ -31,14 +29,10 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/gateway"
"github.com/SigNoz/signoz/pkg/global"
"github.com/SigNoz/signoz/pkg/licensing"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
pkgcloudintegration "github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
"github.com/SigNoz/signoz/pkg/modules/dashboard"
pkgimpldashboard "github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/querier"
"github.com/SigNoz/signoz/pkg/queryparser"
"github.com/SigNoz/signoz/pkg/signoz"
@@ -46,7 +40,6 @@ import (
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstorehook"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/version"
"github.com/SigNoz/signoz/pkg/zeus"
)
@@ -132,6 +125,7 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
return nil, err
}
return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, licensing, dashboardModule), nil
},
func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing) dashboard.Module {
return impldashboard.NewModule(pkgimpldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, querier, licensing)
@@ -143,21 +137,8 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
communityHandler := querier.NewHandler(ps, q, a)
return eequerier.NewHandler(ps, q, communityHandler)
},
func(store cloudintegrationtypes.Store, global global.Global, zeus zeus.Zeus, gateway gateway.Gateway, licensing licensing.Licensing, serviceAccount serviceaccount.Module, config cloudintegration.Config) (cloudintegration.Module, error) {
defStore := pkgcloudintegration.NewServiceDefinitionStore()
awsCloudProviderModule, err := implcloudprovider.NewAWSCloudProvider(defStore)
if err != nil {
return nil, err
}
azureCloudProviderModule := implcloudprovider.NewAzureCloudProvider()
cloudProvidersMap := map[cloudintegrationtypes.CloudProviderType]cloudintegration.CloudProviderModule{
cloudintegrationtypes.CloudProviderTypeAWS: awsCloudProviderModule,
cloudintegrationtypes.CloudProviderTypeAzure: azureCloudProviderModule,
}
return implcloudintegration.NewModule(store, global, zeus, gateway, licensing, serviceAccount, cloudProvidersMap, config)
},
)
if err != nil {
logger.ErrorContext(ctx, "failed to create signoz", errors.Attr(err))
return err

View File

@@ -364,10 +364,3 @@ serviceaccount:
analytics:
# toggle service account analytics
enabled: true
##################### Cloud Integration #####################
cloudintegration:
# cloud integration agent configuration
agent:
# The version of the cloud integration agent.
version: v0.0.8

View File

@@ -1,132 +0,0 @@
package implcloudprovider
import (
"context"
"fmt"
"net/url"
"sort"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
)
type awscloudprovider struct {
serviceDefinitions cloudintegrationtypes.ServiceDefinitionStore
}
func NewAWSCloudProvider(defStore cloudintegrationtypes.ServiceDefinitionStore) (cloudintegration.CloudProviderModule, error) {
return &awscloudprovider{serviceDefinitions: defStore}, nil
}
func (provider *awscloudprovider) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.GetConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
baseURL := fmt.Sprintf(cloudintegrationtypes.CloudFormationQuickCreateBaseURL.StringValue(), req.Config.Aws.DeploymentRegion)
u, _ := url.Parse(baseURL)
q := u.Query()
q.Set("region", req.Config.Aws.DeploymentRegion)
u.Fragment = "/stacks/quickcreate"
u.RawQuery = q.Encode()
q = u.Query()
q.Set("stackName", cloudintegrationtypes.AgentCloudFormationBaseStackName.StringValue())
q.Set("templateURL", fmt.Sprintf(cloudintegrationtypes.AgentCloudFormationTemplateS3Path.StringValue(), req.Config.AgentVersion))
q.Set("param_SigNozIntegrationAgentVersion", req.Config.AgentVersion)
q.Set("param_SigNozApiUrl", req.Credentials.SigNozAPIURL)
q.Set("param_SigNozApiKey", req.Credentials.SigNozAPIKey)
q.Set("param_SigNozAccountId", account.ID.StringValue())
q.Set("param_IngestionUrl", req.Credentials.IngestionURL)
q.Set("param_IngestionKey", req.Credentials.IngestionKey)
return &cloudintegrationtypes.ConnectionArtifact{
Aws: &cloudintegrationtypes.AWSConnectionArtifact{
ConnectionURL: u.String() + "?&" + q.Encode(), // this format is required by AWS
},
}, nil
}
func (provider *awscloudprovider) ListServiceDefinitions(ctx context.Context) ([]*cloudintegrationtypes.ServiceDefinition, error) {
return provider.serviceDefinitions.List(ctx, cloudintegrationtypes.CloudProviderTypeAWS)
}
func (provider *awscloudprovider) GetServiceDefinition(ctx context.Context, serviceID cloudintegrationtypes.ServiceID) (*cloudintegrationtypes.ServiceDefinition, error) {
serviceDef, err := provider.serviceDefinitions.Get(ctx, cloudintegrationtypes.CloudProviderTypeAWS, serviceID)
if err != nil {
return nil, err
}
// override cloud integration dashboard id
for index, dashboard := range serviceDef.Assets.Dashboards {
serviceDef.Assets.Dashboards[index].ID = cloudintegrationtypes.GetCloudIntegrationDashboardID(cloudintegrationtypes.CloudProviderTypeAWS, serviceID.StringValue(), dashboard.ID)
}
return serviceDef, nil
}
func (provider *awscloudprovider) BuildIntegrationConfig(
ctx context.Context,
account *cloudintegrationtypes.Account,
services []*cloudintegrationtypes.StorableCloudIntegrationService,
) (*cloudintegrationtypes.ProviderIntegrationConfig, error) {
// Sort services for deterministic output
sort.Slice(services, func(i, j int) bool {
return services[i].Type.StringValue() < services[j].Type.StringValue()
})
compiledMetrics := new(cloudintegrationtypes.AWSMetricsCollectionStrategy)
compiledLogs := new(cloudintegrationtypes.AWSLogsCollectionStrategy)
var compiledS3Buckets map[string][]string
for _, storedSvc := range services {
svcCfg, err := cloudintegrationtypes.NewServiceConfigFromJSON(cloudintegrationtypes.CloudProviderTypeAWS, storedSvc.Config)
if err != nil {
return nil, err
}
svcDef, err := provider.GetServiceDefinition(ctx, storedSvc.Type)
if err != nil {
return nil, err
}
strategy := svcDef.TelemetryCollectionStrategy.AWS
logsEnabled := svcCfg.IsLogsEnabled(cloudintegrationtypes.CloudProviderTypeAWS)
// S3Sync: logs come directly from configured S3 buckets, not CloudWatch subscriptions
if storedSvc.Type == cloudintegrationtypes.AWSServiceS3Sync {
if logsEnabled && svcCfg.AWS.Logs.S3Buckets != nil {
compiledS3Buckets = svcCfg.AWS.Logs.S3Buckets
}
// no need to go ahead as the code block specifically checks for the S3Sync service
continue
}
if logsEnabled && strategy.Logs != nil {
compiledLogs.Subscriptions = append(compiledLogs.Subscriptions, strategy.Logs.Subscriptions...)
}
metricsEnabled := svcCfg.IsMetricsEnabled(cloudintegrationtypes.CloudProviderTypeAWS)
if metricsEnabled && strategy.Metrics != nil {
compiledMetrics.StreamFilters = append(compiledMetrics.StreamFilters, strategy.Metrics.StreamFilters...)
}
}
collectionStrategy := new(cloudintegrationtypes.AWSTelemetryCollectionStrategy)
if len(compiledMetrics.StreamFilters) > 0 {
collectionStrategy.Metrics = compiledMetrics
}
if len(compiledLogs.Subscriptions) > 0 {
collectionStrategy.Logs = compiledLogs
}
if compiledS3Buckets != nil {
collectionStrategy.S3Buckets = compiledS3Buckets
}
return &cloudintegrationtypes.ProviderIntegrationConfig{
AWS: &cloudintegrationtypes.AWSIntegrationConfig{
EnabledRegions: account.Config.AWS.Regions,
TelemetryCollectionStrategy: collectionStrategy,
},
}, nil
}

View File

@@ -1,34 +0,0 @@
package implcloudprovider
import (
"context"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
)
type azurecloudprovider struct{}
func NewAzureCloudProvider() cloudintegration.CloudProviderModule {
return &azurecloudprovider{}
}
func (provider *azurecloudprovider) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.GetConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
panic("implement me")
}
func (provider *azurecloudprovider) ListServiceDefinitions(ctx context.Context) ([]*cloudintegrationtypes.ServiceDefinition, error) {
panic("implement me")
}
func (provider *azurecloudprovider) GetServiceDefinition(ctx context.Context, serviceID cloudintegrationtypes.ServiceID) (*cloudintegrationtypes.ServiceDefinition, error) {
panic("implement me")
}
func (provider *azurecloudprovider) BuildIntegrationConfig(
ctx context.Context,
account *cloudintegrationtypes.Account,
services []*cloudintegrationtypes.StorableCloudIntegrationService,
) (*cloudintegrationtypes.ProviderIntegrationConfig, error) {
panic("implement me")
}

View File

@@ -1,513 +0,0 @@
package implcloudintegration
import (
"context"
"fmt"
"sort"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/gateway"
"github.com/SigNoz/signoz/pkg/global"
"github.com/SigNoz/signoz/pkg/licensing"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
"github.com/SigNoz/signoz/pkg/types/serviceaccounttypes"
"github.com/SigNoz/signoz/pkg/types/zeustypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/SigNoz/signoz/pkg/zeus"
)
type module struct {
store cloudintegrationtypes.Store
gateway gateway.Gateway
zeus zeus.Zeus
licensing licensing.Licensing
global global.Global
serviceAccount serviceaccount.Module
cloudProvidersMap map[cloudintegrationtypes.CloudProviderType]cloudintegration.CloudProviderModule
config cloudintegration.Config
}
func NewModule(
store cloudintegrationtypes.Store,
global global.Global,
zeus zeus.Zeus,
gateway gateway.Gateway,
licensing licensing.Licensing,
serviceAccount serviceaccount.Module,
cloudProvidersMap map[cloudintegrationtypes.CloudProviderType]cloudintegration.CloudProviderModule,
config cloudintegration.Config,
) (cloudintegration.Module, error) {
return &module{
store: store,
global: global,
zeus: zeus,
gateway: gateway,
licensing: licensing,
serviceAccount: serviceAccount,
cloudProvidersMap: cloudProvidersMap,
config: config,
}, nil
}
// GetConnectionCredentials returns credentials required to generate connection artifact. eg. apiKey, ingestionKey etc.
// It will return creds it can deduce and return empty value for others.
func (module *module) GetConnectionCredentials(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Credentials, error) {
// get license to get the deployment details
license, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
var ingestionURL string
globalConfig := module.global.GetConfig(ctx)
if globalConfig != nil {
ingestionURL = globalConfig.IngestionURL
}
// get deployment details from zeus, ignore error
respBytes, _ := module.zeus.GetDeployment(ctx, license.Key)
var signozAPIURL string
if len(respBytes) > 0 {
// parse deployment details, ignore error, if client is asking api url every time check for possible error
deployment, _ := zeustypes.NewGettableDeployment(respBytes)
if deployment != nil {
signozAPIURL = deployment.SignozAPIUrl
}
}
// ignore error
apiKey, _ := module.getOrCreateAPIKey(ctx, orgID, provider)
// ignore error
ingestionKey, _ := module.getOrCreateIngestionKey(ctx, orgID, provider)
return &cloudintegrationtypes.Credentials{
SigNozAPIURL: signozAPIURL,
SigNozAPIKey: apiKey,
IngestionURL: ingestionURL,
IngestionKey: ingestionKey,
}, nil
}
func (module *module) CreateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
_, err := module.licensing.GetActive(ctx, account.OrgID)
if err != nil {
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
storableCloudIntegration, err := cloudintegrationtypes.NewStorableCloudIntegration(account)
if err != nil {
return err
}
return module.store.CreateAccount(ctx, storableCloudIntegration)
}
func (module *module) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.GetConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
cloudProviderModule, err := module.getCloudProvider(account.Provider)
if err != nil {
return nil, err
}
req.Config.AddAgentVersion(module.config.Agent.Version)
return cloudProviderModule.GetConnectionArtifact(ctx, account, req)
}
func (module *module) GetAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Account, error) {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
storableAccount, err := module.store.GetAccountByID(ctx, orgID, accountID, provider)
if err != nil {
return nil, err
}
return cloudintegrationtypes.NewAccountFromStorable(storableAccount)
}
// ListAccounts return only agent connected accounts.
func (module *module) ListAccounts(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) ([]*cloudintegrationtypes.Account, error) {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
storableAccounts, err := module.store.ListConnectedAccounts(ctx, orgID, provider)
if err != nil {
return nil, err
}
return cloudintegrationtypes.NewAccountsFromStorables(storableAccounts)
}
func (module *module) AgentCheckIn(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, req *cloudintegrationtypes.AgentCheckInRequest) (*cloudintegrationtypes.AgentCheckInResponse, error) {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
connectedAccount, err := module.store.GetConnectedAccount(ctx, orgID, provider, req.ProviderAccountID)
if err != nil && !errors.Ast(err, errors.TypeNotFound) {
return nil, err
}
// If a different integration is already connected to this provider account ID, reject the check-in.
// Allow re-check-in from the same integration (e.g. agent restarting).
if connectedAccount != nil && connectedAccount.ID != req.CloudIntegrationID {
errMessage := fmt.Sprintf("provider account id %s is already connected to cloud integration id %s", req.ProviderAccountID, connectedAccount.ID)
return nil, errors.New(errors.TypeAlreadyExists, cloudintegrationtypes.ErrCodeCloudIntegrationAlreadyConnected, errMessage)
}
account, err := module.store.GetAccountByID(ctx, orgID, req.CloudIntegrationID, provider)
if err != nil {
return nil, err
}
// update account with cloud provider account id and agent report (heartbeat)
account.Update(&req.ProviderAccountID, cloudintegrationtypes.NewAgentReport(req.Data))
err = module.store.UpdateAccount(ctx, account)
if err != nil {
return nil, err
}
// If account has been removed (disconnected), return a minimal response with empty integration config.
// The agent doesn't act on config for removed accounts.
if account.RemovedAt != nil {
return &cloudintegrationtypes.AgentCheckInResponse{
CloudIntegrationID: account.ID.StringValue(),
ProviderAccountID: req.ProviderAccountID,
IntegrationConfig: new(cloudintegrationtypes.ProviderIntegrationConfig),
RemovedAt: account.RemovedAt,
}, nil
}
// Get account as domain object for config access (enabled regions, etc.)
domainAccount, err := cloudintegrationtypes.NewAccountFromStorable(account)
if err != nil {
return nil, err
}
cloudProvider, err := module.getCloudProvider(provider)
if err != nil {
return nil, err
}
storedServices, err := module.store.ListServices(ctx, req.CloudIntegrationID)
if err != nil {
return nil, err
}
// Delegate integration config building entirely to the provider module
integrationConfig, err := cloudProvider.BuildIntegrationConfig(ctx, domainAccount, storedServices)
if err != nil {
return nil, err
}
return &cloudintegrationtypes.AgentCheckInResponse{
CloudIntegrationID: account.ID.StringValue(),
ProviderAccountID: req.ProviderAccountID,
IntegrationConfig: integrationConfig,
RemovedAt: account.RemovedAt,
}, nil
}
func (module *module) UpdateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
_, err := module.licensing.GetActive(ctx, account.OrgID)
if err != nil {
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
storableAccount, err := cloudintegrationtypes.NewStorableCloudIntegration(account)
if err != nil {
return err
}
return module.store.UpdateAccount(ctx, storableAccount)
}
func (module *module) DisconnectAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) error {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
return module.store.RemoveAccount(ctx, orgID, accountID, provider)
}
func (module *module) ListServicesMetadata(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, integrationID *valuer.UUID) ([]*cloudintegrationtypes.ServiceMetadata, error) {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
cloudProvider, err := module.getCloudProvider(provider)
if err != nil {
return nil, err
}
serviceDefinitions, err := cloudProvider.ListServiceDefinitions(ctx)
if err != nil {
return nil, err
}
enabledServiceIDs := map[string]bool{}
if integrationID != nil {
storedServices, err := module.store.ListServices(ctx, *integrationID)
if err != nil {
return nil, err
}
for _, svc := range storedServices {
serviceConfig, err := cloudintegrationtypes.NewServiceConfigFromJSON(provider, svc.Config)
if err != nil {
return nil, err
}
if serviceConfig.IsServiceEnabled(provider) {
enabledServiceIDs[svc.Type.StringValue()] = true
}
}
}
resp := make([]*cloudintegrationtypes.ServiceMetadata, 0, len(serviceDefinitions))
for _, serviceDefinition := range serviceDefinitions {
resp = append(resp, cloudintegrationtypes.NewServiceMetadata(*serviceDefinition, enabledServiceIDs[serviceDefinition.ID]))
}
return resp, nil
}
func (module *module) GetService(ctx context.Context, orgID valuer.UUID, integrationID *valuer.UUID, serviceID cloudintegrationtypes.ServiceID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Service, error) {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
cloudProvider, err := module.getCloudProvider(provider)
if err != nil {
return nil, err
}
serviceDefinition, err := cloudProvider.GetServiceDefinition(ctx, serviceID)
if err != nil {
return nil, err
}
var integrationService *cloudintegrationtypes.CloudIntegrationService
if integrationID != nil {
storedService, err := module.store.GetServiceByServiceID(ctx, *integrationID, serviceID)
if err != nil && !errors.Ast(err, errors.TypeNotFound) {
return nil, err
}
if storedService != nil {
serviceConfig, err := cloudintegrationtypes.NewServiceConfigFromJSON(provider, storedService.Config)
if err != nil {
return nil, err
}
integrationService = cloudintegrationtypes.NewCloudIntegrationServiceFromStorable(storedService, serviceConfig)
}
}
return cloudintegrationtypes.NewService(*serviceDefinition, integrationService), nil
}
func (module *module) CreateService(ctx context.Context, orgID valuer.UUID, service *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
cloudProvider, err := module.getCloudProvider(provider)
if err != nil {
return err
}
serviceDefinition, err := cloudProvider.GetServiceDefinition(ctx, service.Type)
if err != nil {
return err
}
configJSON, err := service.Config.ToJSON(provider, service.Type, &serviceDefinition.SupportedSignals)
if err != nil {
return err
}
return module.store.CreateService(ctx, cloudintegrationtypes.NewStorableCloudIntegrationService(service, string(configJSON)))
}
func (module *module) UpdateService(ctx context.Context, orgID valuer.UUID, integrationService *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
cloudProvider, err := module.getCloudProvider(provider)
if err != nil {
return err
}
serviceDefinition, err := cloudProvider.GetServiceDefinition(ctx, integrationService.Type)
if err != nil {
return err
}
configJSON, err := integrationService.Config.ToJSON(provider, integrationService.Type, &serviceDefinition.SupportedSignals)
if err != nil {
return err
}
storableService := cloudintegrationtypes.NewStorableCloudIntegrationService(integrationService, string(configJSON))
return module.store.UpdateService(ctx, storableService)
}
func (module *module) GetDashboardByID(ctx context.Context, orgID valuer.UUID, id string) (*dashboardtypes.Dashboard, error) {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
_, _, _, err = cloudintegrationtypes.ParseCloudIntegrationDashboardID(id)
if err != nil {
return nil, err
}
allDashboards, err := module.listDashboards(ctx, orgID)
if err != nil {
return nil, err
}
for _, d := range allDashboards {
if d.ID == id {
return d, nil
}
}
return nil, errors.New(errors.TypeNotFound, cloudintegrationtypes.ErrCodeCloudIntegrationNotFound, "cloud integration dashboard not found")
}
func (module *module) ListDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error) {
_, err := module.licensing.GetActive(ctx, orgID)
if err != nil {
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
return module.listDashboards(ctx, orgID)
}
func (module *module) getCloudProvider(provider cloudintegrationtypes.CloudProviderType) (cloudintegration.CloudProviderModule, error) {
if cloudProviderModule, ok := module.cloudProvidersMap[provider]; ok {
return cloudProviderModule, nil
}
return nil, errors.NewInvalidInputf(cloudintegrationtypes.ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
}
func (module *module) getOrCreateIngestionKey(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (string, error) {
keyName := cloudintegrationtypes.NewIngestionKeyName(provider)
result, err := module.gateway.SearchIngestionKeysByName(ctx, orgID, keyName, 1, 10)
if err != nil {
return "", errors.WrapInternalf(err, errors.CodeInternal, "couldn't search ingestion keys")
}
// ideally there should be only one key per cloud integration provider
if len(result.Keys) > 0 {
return result.Keys[0].Value, nil
}
createdIngestionKey, err := module.gateway.CreateIngestionKey(ctx, orgID, keyName, []string{"integration"}, time.Time{})
if err != nil {
return "", errors.WrapInternalf(err, errors.CodeInternal, "couldn't create ingestion key")
}
return createdIngestionKey.Value, nil
}
func (module *module) getOrCreateAPIKey(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (string, error) {
domain := module.serviceAccount.Config().Email.Domain
serviceAccount := serviceaccounttypes.NewServiceAccount("integration", domain, serviceaccounttypes.ServiceAccountStatusActive, orgID)
serviceAccount, err := module.serviceAccount.GetOrCreate(ctx, orgID, serviceAccount)
if err != nil {
return "", err
}
err = module.serviceAccount.SetRoleByName(ctx, orgID, serviceAccount.ID, authtypes.SigNozViewerRoleName)
if err != nil {
return "", err
}
factorAPIKey, err := serviceAccount.NewFactorAPIKey(provider.StringValue(), 0)
if err != nil {
return "", err
}
factorAPIKey, err = module.serviceAccount.GetOrCreateFactorAPIKey(ctx, factorAPIKey)
if err != nil {
return "", err
}
return factorAPIKey.Key, nil
}
func (module *module) listDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error) {
var allDashboards []*dashboardtypes.Dashboard
for provider := range module.cloudProvidersMap {
cloudProvider, err := module.getCloudProvider(provider)
if err != nil {
return nil, err
}
connectedAccounts, err := module.store.ListConnectedAccounts(ctx, orgID, provider)
if err != nil {
return nil, err
}
for _, storableAccount := range connectedAccounts {
storedServices, err := module.store.ListServices(ctx, storableAccount.ID)
if err != nil {
return nil, err
}
for _, storedSvc := range storedServices {
serviceConfig, err := cloudintegrationtypes.NewServiceConfigFromJSON(provider, storedSvc.Config)
if err != nil || !serviceConfig.IsMetricsEnabled(provider) {
continue
}
svcDef, err := cloudProvider.GetServiceDefinition(ctx, storedSvc.Type)
if err != nil || svcDef == nil {
continue
}
dashboards := cloudintegrationtypes.GetDashboardsFromAssets(
storedSvc.Type.StringValue(),
orgID,
provider,
storableAccount.CreatedAt,
svcDef.Assets,
)
allDashboards = append(allDashboards, dashboards...)
}
}
}
sort.Slice(allDashboards, func(i, j int) bool {
return allDashboards[i].ID < allDashboards[j].ID
})
return allDashboards, nil
}

View File

@@ -1,32 +0,0 @@
package cloudintegration
import (
"github.com/SigNoz/signoz/pkg/factory"
)
type Config struct {
// Agent config for cloud integration
Agent AgentConfig `mapstructure:"agent"`
}
type AgentConfig struct {
Version string `mapstructure:"version"`
}
func NewConfigFactory() factory.ConfigFactory {
return factory.NewConfigFactory(factory.MustNewName("cloudintegration"), newConfig)
}
func newConfig() factory.Config {
return &Config{
Agent: AgentConfig{
// we will maintain the latest version of cloud integration agent from here,
// till we automate it externally or figure out a way to validate it.
Version: "v0.0.8",
},
}
}
func (c Config) Validate() error {
return nil
}

View File

@@ -1,176 +1,21 @@
package implcloudintegration
import (
"bytes"
"context"
"embed"
"encoding/base64"
"encoding/json"
"fmt"
"io/fs"
"path"
"sort"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
citypes "github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
)
const definitionsRoot = "fs/definitions"
//go:embed fs/definitions/*
var definitionFiles embed.FS
type definitionStore struct{}
// NewServiceDefinitionStore creates a new ServiceDefinitionStore backed by the embedded filesystem.
func NewServiceDefinitionStore() citypes.ServiceDefinitionStore {
func NewDefinitionStore() citypes.ServiceDefinitionStore {
return &definitionStore{}
}
// Get reads and hydrates the service definition for the given provider and service ID.
func (s *definitionStore) Get(ctx context.Context, provider citypes.CloudProviderType, serviceID citypes.ServiceID) (*citypes.ServiceDefinition, error) {
svcDir := path.Join(definitionsRoot, provider.StringValue(), serviceID.StringValue())
def, err := readServiceDefinition(svcDir)
if err != nil {
return nil, errors.New(errors.TypeNotFound, citypes.ErrCodeServiceDefinitionNotFound, fmt.Sprintf("service definition not found for service id %q", serviceID.StringValue()))
}
return def, nil
func (d *definitionStore) Get(ctx context.Context, provider citypes.CloudProviderType, serviceID citypes.ServiceID) (*citypes.ServiceDefinition, error) {
panic("unimplemented")
}
// List reads and hydrates all service definitions for the given provider, sorted by ID.
func (s *definitionStore) List(ctx context.Context, provider citypes.CloudProviderType) ([]*citypes.ServiceDefinition, error) {
providerDir := path.Join(definitionsRoot, provider.StringValue())
entries, err := fs.ReadDir(definitionFiles, providerDir)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read service definition dirs for %s", provider.StringValue())
}
var result []*citypes.ServiceDefinition
for _, entry := range entries {
if !entry.IsDir() {
continue
}
svcDir := path.Join(providerDir, entry.Name())
def, err := readServiceDefinition(svcDir)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read service definition for %s/%s", provider.StringValue(), entry.Name())
}
result = append(result, def)
}
sort.Slice(result, func(i, j int) bool {
return result[i].ID < result[j].ID
})
return result, nil
}
// following are helper functions for reading and hydrating service definitions,
// not keeping this in types as this is an implementation detail of the definition store.
func readServiceDefinition(svcDir string) (*citypes.ServiceDefinition, error) {
integrationJSONPath := path.Join(svcDir, "integration.json")
raw, err := definitionFiles.ReadFile(integrationJSONPath)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read %s", integrationJSONPath)
}
var specMap map[string]any
if err := json.Unmarshal(raw, &specMap); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't parse %s", integrationJSONPath)
}
hydrated, err := hydrateFileURIs(specMap, definitionFiles, svcDir)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't hydrate file URIs in %s", integrationJSONPath)
}
reEncoded, err := json.Marshal(hydrated)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't re-encode hydrated spec from %s", integrationJSONPath)
}
var def citypes.ServiceDefinition
decoder := json.NewDecoder(bytes.NewReader(reEncoded))
decoder.DisallowUnknownFields()
if err := decoder.Decode(&def); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't decode service definition from %s", integrationJSONPath)
}
if err := validateServiceDefinition(&def); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "invalid service definition in %s", svcDir)
}
return &def, nil
}
func validateServiceDefinition(def *citypes.ServiceDefinition) error {
if def.TelemetryCollectionStrategy == nil {
return errors.NewInternalf(errors.CodeInternal, "telemetryCollectionStrategy is required")
}
seenDashboardIDs := map[string]struct{}{}
for _, d := range def.Assets.Dashboards {
if _, seen := seenDashboardIDs[d.ID]; seen {
return errors.NewInternalf(errors.CodeInternal, "duplicate dashboard id %q", d.ID)
}
seenDashboardIDs[d.ID] = struct{}{}
}
return nil
}
// hydrateFileURIs walks a JSON-decoded value and replaces any "file://<path>" strings
// with the actual file contents (text for .md, base64 data URI for .svg, parsed JSON for .json).
func hydrateFileURIs(v any, embeddedFS embed.FS, basedir string) (any, error) {
switch val := v.(type) {
case map[string]any:
result := make(map[string]any, len(val))
for k, child := range val {
hydrated, err := hydrateFileURIs(child, embeddedFS, basedir)
if err != nil {
return nil, err
}
result[k] = hydrated
}
return result, nil
case []any:
result := make([]any, len(val))
for i, child := range val {
hydrated, err := hydrateFileURIs(child, embeddedFS, basedir)
if err != nil {
return nil, err
}
result[i] = hydrated
}
return result, nil
case string:
if !strings.HasPrefix(val, "file://") {
return val, nil
}
return readEmbeddedFile(embeddedFS, path.Join(basedir, val[len("file://"):]))
}
return v, nil
}
func readEmbeddedFile(embeddedFS embed.FS, filePath string) (any, error) {
contents, err := embeddedFS.ReadFile(filePath)
if err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't read embedded file %s", filePath)
}
switch {
case strings.HasSuffix(filePath, ".md"):
return string(contents), nil
case strings.HasSuffix(filePath, ".svg"):
return fmt.Sprintf("data:image/svg+xml;base64,%s", base64.StdEncoding.EncodeToString(contents)), nil
case strings.HasSuffix(filePath, ".json"):
var parsed any
if err := json.Unmarshal(contents, &parsed); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't parse JSON file %s", filePath)
}
return parsed, nil
default:
return nil, errors.NewInternalf(errors.CodeInternal, "unsupported file type for embedded reference: %s", filePath)
}
func (d *definitionStore) List(ctx context.Context, provider citypes.CloudProviderType) ([]*citypes.ServiceDefinition, error) {
panic("unimplemented")
}

View File

@@ -1,493 +1,62 @@
package implcloudintegration
import (
"context"
"net/http"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/http/binding"
"github.com/SigNoz/signoz/pkg/http/render"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/gorilla/mux"
)
type handler struct {
module cloudintegration.Module
type handler struct{}
func NewHandler() cloudintegration.Handler {
return &handler{}
}
func NewHandler(module cloudintegration.Module) cloudintegration.Handler {
return &handler{
module: module,
}
func (handler *handler) GetConnectionCredentials(http.ResponseWriter, *http.Request) {
panic("unimplemented")
}
func (handler *handler) GetConnectionCredentials(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
creds, err := handler.module.GetConnectionCredentials(ctx, valuer.MustNewUUID(claims.OrgID), provider)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, creds)
func (handler *handler) CreateAccount(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) CreateAccount(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
postableAccount := new(cloudintegrationtypes.PostableAccount)
err = binding.JSON.BindBody(r.Body, postableAccount)
if err != nil {
render.Error(rw, err)
return
}
if err := postableAccount.Validate(provider); err != nil {
render.Error(rw, err)
return
}
accountConfig, err := cloudintegrationtypes.NewAccountConfigFromPostable(provider, postableAccount.Config)
if err != nil {
render.Error(rw, err)
return
}
account := cloudintegrationtypes.NewAccount(valuer.MustNewUUID(claims.OrgID), provider, accountConfig)
err = handler.module.CreateAccount(ctx, account)
if err != nil {
render.Error(rw, err)
return
}
connectionArtifact, err := handler.module.GetConnectionArtifact(ctx, account, postableAccount)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, &cloudintegrationtypes.GettableAccountWithConnectionArtifact{
ID: account.ID,
ConnectionArtifact: connectionArtifact,
})
func (handler *handler) ListAccounts(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) GetAccount(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
accountIDString := mux.Vars(r)["id"]
accountID, err := valuer.NewUUID(accountIDString)
if err != nil {
render.Error(rw, err)
return
}
account, err := handler.module.GetAccount(ctx, valuer.MustNewUUID(claims.OrgID), accountID, provider)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, account)
func (handler *handler) GetAccount(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) ListAccounts(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
accounts, err := handler.module.ListAccounts(ctx, valuer.MustNewUUID(claims.OrgID), provider)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, &cloudintegrationtypes.GettableAccounts{
Accounts: accounts,
})
func (handler *handler) UpdateAccount(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) UpdateAccount(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
id := mux.Vars(r)["id"]
cloudIntegrationID, err := valuer.NewUUID(id)
if err != nil {
render.Error(rw, err)
return
}
req := new(cloudintegrationtypes.UpdatableAccount)
if err := binding.JSON.BindBody(r.Body, req); err != nil {
render.Error(rw, err)
return
}
if err := req.Validate(provider); err != nil {
render.Error(rw, err)
return
}
account, err := handler.module.GetAccount(ctx, valuer.MustNewUUID(claims.OrgID), cloudIntegrationID, provider)
if err != nil {
render.Error(rw, err)
return
}
if err := account.Update(req.Config); err != nil {
render.Error(rw, err)
return
}
err = handler.module.UpdateAccount(ctx, account)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusNoContent, nil)
func (handler *handler) DisconnectAccount(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) DisconnectAccount(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
id := mux.Vars(r)["id"]
cloudIntegrationID, err := valuer.NewUUID(id)
if err != nil {
render.Error(rw, err)
return
}
err = handler.module.DisconnectAccount(ctx, valuer.MustNewUUID(claims.OrgID), cloudIntegrationID, provider)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusNoContent, nil)
func (handler *handler) ListServicesMetadata(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) ListServicesMetadata(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
var integrationID *valuer.UUID
if idStr := r.URL.Query().Get("cloud_integration_id"); idStr != "" {
id, err := valuer.NewUUID(idStr)
if err != nil {
render.Error(rw, err)
return
}
integrationID = &id
}
orgID := valuer.MustNewUUID(claims.OrgID)
// check if integration account exists and is not removed.
if integrationID != nil {
account, err := handler.module.GetAccount(ctx, orgID, *integrationID, provider)
if err != nil {
render.Error(rw, err)
return
}
if account.IsRemoved() {
render.Error(rw, errors.New(errors.TypeNotFound, cloudintegrationtypes.ErrCodeCloudIntegrationNotFound, "cloud integration account is removed"))
return
}
}
services, err := handler.module.ListServicesMetadata(ctx, orgID, provider, integrationID)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, &cloudintegrationtypes.GettableServicesMetadata{
Services: services,
})
func (handler *handler) GetService(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) GetService(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
serviceIDString := mux.Vars(r)["service_id"]
serviceID, err := cloudintegrationtypes.NewServiceID(provider, serviceIDString)
if err != nil {
render.Error(rw, err)
return
}
var integrationID *valuer.UUID
if idStr := r.URL.Query().Get("cloud_integration_id"); idStr != "" {
id, err := valuer.NewUUID(idStr)
if err != nil {
render.Error(rw, err)
return
}
integrationID = &id
}
// check if integration account exists and is not removed.
if integrationID != nil {
account, err := handler.module.GetAccount(ctx, valuer.MustNewUUID(claims.OrgID), *integrationID, provider)
if err != nil {
render.Error(rw, err)
return
}
if account.IsRemoved() {
render.Error(rw, errors.New(errors.TypeNotFound, cloudintegrationtypes.ErrCodeCloudIntegrationNotFound, "cloud integration account is removed"))
return
}
}
svc, err := handler.module.GetService(ctx, valuer.MustNewUUID(claims.OrgID), integrationID, serviceID, provider)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, svc)
func (handler *handler) UpdateService(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}
func (handler *handler) UpdateService(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
serviceIDString := mux.Vars(r)["service_id"]
serviceID, err := cloudintegrationtypes.NewServiceID(provider, serviceIDString)
if err != nil {
render.Error(rw, err)
return
}
req := new(cloudintegrationtypes.UpdatableService)
if err := binding.JSON.BindBody(r.Body, req); err != nil {
render.Error(rw, err)
return
}
if err := req.Validate(provider, serviceID); err != nil {
render.Error(rw, err)
return
}
cloudIntegrationID, err := valuer.NewUUID(mux.Vars(r)["id"])
if err != nil {
render.Error(rw, err)
return
}
orgID := valuer.MustNewUUID(claims.OrgID)
// check if integration account exists and is not removed.
account, err := handler.module.GetAccount(ctx, orgID, cloudIntegrationID, provider)
if err != nil {
render.Error(rw, err)
return
}
if account.IsRemoved() {
render.Error(rw, errors.New(errors.TypeNotFound, cloudintegrationtypes.ErrCodeCloudIntegrationNotFound, "cloud integration account is removed"))
return
}
svc, err := handler.module.GetService(ctx, orgID, &cloudIntegrationID, serviceID, provider)
if err != nil {
render.Error(rw, err)
return
}
if svc.CloudIntegrationService == nil {
cloudIntegrationService := cloudintegrationtypes.NewCloudIntegrationService(serviceID, cloudIntegrationID, req.Config)
err = handler.module.CreateService(ctx, orgID, cloudIntegrationService, provider)
} else {
svc.CloudIntegrationService.Update(req.Config)
err = handler.module.UpdateService(ctx, orgID, svc.CloudIntegrationService, provider)
}
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusNoContent, nil)
}
func (handler *handler) AgentCheckIn(rw http.ResponseWriter, r *http.Request) {
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second)
defer cancel()
claims, err := authtypes.ClaimsFromContext(ctx)
if err != nil {
render.Error(rw, err)
return
}
providerString := mux.Vars(r)["cloud_provider"]
provider, err := cloudintegrationtypes.NewCloudProvider(providerString)
if err != nil {
render.Error(rw, err)
return
}
req := new(cloudintegrationtypes.PostableAgentCheckIn)
if err := binding.JSON.BindBody(r.Body, req); err != nil {
render.Error(rw, err)
return
}
if err := req.Validate(); err != nil {
render.Error(rw, err)
return
}
// Map old fields → new fields for backward compatibility with old agents
// Old agents send account_id (=> cloudIntegrationId) and cloud_account_id (=> providerAccountId)
if req.ID != "" {
id, err := valuer.NewUUID(req.ID)
if err != nil {
render.Error(rw, err)
return
}
req.CloudIntegrationID = id
req.ProviderAccountID = req.AccountID
}
orgID := valuer.MustNewUUID(claims.OrgID)
resp, err := handler.module.AgentCheckIn(ctx, orgID, provider, &req.AgentCheckInRequest)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, cloudintegrationtypes.NewGettableAgentCheckIn(provider, resp))
func (handler *handler) AgentCheckIn(writer http.ResponseWriter, request *http.Request) {
// TODO implement me
panic("implement me")
}

View File

@@ -1,73 +0,0 @@
package implcloudintegration
import (
"context"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
type module struct{}
func NewModule() cloudintegration.Module {
return &module{}
}
func (module *module) GetConnectionCredentials(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Credentials, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get connection credentials is not supported")
}
func (module *module) CreateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "create account is not supported")
}
func (module *module) GetAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Account, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get account is not supported")
}
func (module *module) ListAccounts(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) ([]*cloudintegrationtypes.Account, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "list accounts is not supported")
}
func (module *module) UpdateAccount(ctx context.Context, account *cloudintegrationtypes.Account) error {
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "update account is not supported")
}
func (module *module) DisconnectAccount(ctx context.Context, orgID valuer.UUID, accountID valuer.UUID, provider cloudintegrationtypes.CloudProviderType) error {
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "disconnect account is not supported")
}
func (module *module) CreateService(ctx context.Context, orgID valuer.UUID, service *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "create service is not supported")
}
func (module *module) GetService(ctx context.Context, orgID valuer.UUID, integrationID *valuer.UUID, serviceID cloudintegrationtypes.ServiceID, provider cloudintegrationtypes.CloudProviderType) (*cloudintegrationtypes.Service, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get service is not supported")
}
func (module *module) ListServicesMetadata(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, integrationID *valuer.UUID) ([]*cloudintegrationtypes.ServiceMetadata, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "list services metadata is not supported")
}
func (module *module) UpdateService(ctx context.Context, orgID valuer.UUID, service *cloudintegrationtypes.CloudIntegrationService, provider cloudintegrationtypes.CloudProviderType) error {
return errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "update service is not supported")
}
func (module *module) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.GetConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get connection artifact is not supported")
}
func (module *module) AgentCheckIn(ctx context.Context, orgID valuer.UUID, provider cloudintegrationtypes.CloudProviderType, req *cloudintegrationtypes.AgentCheckInRequest) (*cloudintegrationtypes.AgentCheckInResponse, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "agent check-in is not supported")
}
func (module *module) GetDashboardByID(ctx context.Context, orgID valuer.UUID, id string) (*dashboardtypes.Dashboard, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "get dashboard by ID is not supported")
}
func (module *module) ListDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error) {
return nil, errors.New(errors.TypeUnsupported, cloudintegrationtypes.ErrCodeUnsupported, "list dashboards is not supported")
}

View File

@@ -138,7 +138,14 @@ func (m *module) listMeterMetrics(ctx context.Context, params *metricsexplorerty
func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *metricsexplorertypes.ListMetricsParams) (*metricsexplorertypes.ListMetricsResponse, error) {
sb := sqlbuilder.NewSelectBuilder()
sb.Select("DISTINCT metric_name")
sb.Select(
"metric_name",
"anyLast(description) AS description",
"anyLast(type) AS metric_type",
"argMax(unit, unix_milli) AS metric_unit",
"anyLast(temporality) AS temporality",
"anyLast(is_monotonic) AS is_monotonic",
)
if params.Start != nil && params.End != nil {
start, end, distributedTsTable, _ := telemetrymetrics.WhichTSTableToUse(uint64(*params.Start), uint64(*params.End), nil)
@@ -157,6 +164,7 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
sb.Where(sb.Like("lower(metric_name)", fmt.Sprintf("%%%s%%", searchLower)))
}
sb.GroupBy("metric_name")
sb.OrderBy("metric_name ASC")
sb.Limit(params.Limit)
@@ -170,43 +178,47 @@ func (m *module) listMetrics(ctx context.Context, orgID valuer.UUID, params *met
}
defer rows.Close()
metricNames := make([]string, 0)
var metrics []metricsexplorertypes.ListMetric
var metricNames []string
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan metric name")
var metric metricsexplorertypes.ListMetric
if err := rows.Scan(
&metric.MetricName,
&metric.Description,
&metric.MetricType,
&metric.MetricUnit,
&metric.Temporality,
&metric.IsMonotonic,
); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan metric")
}
metricNames = append(metricNames, name)
metrics = append(metrics, metric)
metricNames = append(metricNames, metric.MetricName)
}
if err := rows.Err(); err != nil {
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error iterating metric names")
return nil, errors.WrapInternalf(err, errors.CodeInternal, "error iterating metrics")
}
if len(metricNames) == 0 {
if len(metrics) == 0 {
return &metricsexplorertypes.ListMetricsResponse{
Metrics: []metricsexplorertypes.ListMetric{},
}, nil
}
metadata, err := m.GetMetricMetadataMulti(ctx, orgID, metricNames)
// Overlay any user-updated metadata on top of the timeseries metadata.
updatedMetadata, err := m.fetchUpdatedMetadata(ctx, orgID, metricNames)
if err != nil {
return nil, err
}
metrics := make([]metricsexplorertypes.ListMetric, 0, len(metricNames))
for _, name := range metricNames {
metric := metricsexplorertypes.ListMetric{
MetricName: name,
for i := range metrics {
if meta, ok := updatedMetadata[metrics[i].MetricName]; ok && meta != nil {
metrics[i].Description = meta.Description
metrics[i].MetricType = meta.MetricType
metrics[i].MetricUnit = meta.MetricUnit
metrics[i].Temporality = meta.Temporality
metrics[i].IsMonotonic = meta.IsMonotonic
}
if meta, ok := metadata[name]; ok && meta != nil {
metric.Description = meta.Description
metric.MetricType = meta.MetricType
metric.MetricUnit = meta.MetricUnit
metric.Temporality = meta.Temporality
metric.IsMonotonic = meta.IsMonotonic
}
metrics = append(metrics, metric)
}
return &metricsexplorertypes.ListMetricsResponse{
@@ -243,19 +255,18 @@ func (m *module) GetStats(ctx context.Context, orgID valuer.UUID, req *metricsex
}, nil
}
// Get metadata for all metrics
// Overlay any user-updated metadata on top of the timeseries metadata
// that was already fetched in the combined query.
metricNames := make([]string, len(metricStats))
for i := range metricStats {
metricNames[i] = metricStats[i].MetricName
}
metadata, err := m.GetMetricMetadataMulti(ctx, orgID, metricNames)
updatedMetadata, err := m.fetchUpdatedMetadata(ctx, orgID, metricNames)
if err != nil {
return nil, err
}
// Enrich stats with metadata
enrichStatsWithMetadata(metricStats, metadata)
enrichStatsWithMetadata(metricStats, updatedMetadata)
return &metricsexplorertypes.StatsResponse{
Metrics: metricStats,
@@ -984,11 +995,14 @@ func (m *module) fetchMetricsStatsWithSamples(
samplesTable := telemetrymetrics.WhichSamplesTableToUse(uint64(req.Start), uint64(req.End), metrictypes.UnspecifiedType, metrictypes.TimeAggregationUnspecified, nil)
countExp := telemetrymetrics.CountExpressionForSamplesTable(samplesTable)
// Timeseries counts per metric
// Timeseries counts and metadata per metric.
tsSB := sqlbuilder.NewSelectBuilder()
tsSB.Select(
"metric_name",
"uniq(fingerprint) AS timeseries",
"anyLast(description) AS description",
"anyLast(type) AS metric_type",
"argMax(unit, unix_milli) AS metric_unit",
)
tsSB.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, distributedTsTable))
tsSB.Where(tsSB.Between("unix_milli", start, end))
@@ -1036,6 +1050,9 @@ func (m *module) fetchMetricsStatsWithSamples(
"COALESCE(ts.timeseries, 0) AS timeseries",
"COALESCE(s.samples, 0) AS samples",
"COUNT(*) OVER() AS total",
"ts.description AS description",
"ts.metric_type AS metric_type",
"ts.metric_unit AS metric_unit",
)
finalSB.From("__time_series_counts ts")
finalSB.JoinWithOption(sqlbuilder.FullOuterJoin, "__sample_counts s", "ts.metric_name = s.metric_name")
@@ -1071,7 +1088,7 @@ func (m *module) fetchMetricsStatsWithSamples(
metricStat metricsexplorertypes.Stat
rowTotal uint64
)
if err := rows.Scan(&metricStat.MetricName, &metricStat.TimeSeries, &metricStat.Samples, &rowTotal); err != nil {
if err := rows.Scan(&metricStat.MetricName, &metricStat.TimeSeries, &metricStat.Samples, &rowTotal, &metricStat.Description, &metricStat.MetricType, &metricStat.MetricUnit); err != nil {
return nil, 0, errors.WrapInternalf(err, errors.CodeInternal, "failed to scan metrics stats row")
}
metricStats = append(metricStats, metricStat)

View File

@@ -40,7 +40,6 @@ type querier struct {
promEngine prometheus.Prometheus
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation]
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation]
meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation]
traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder
@@ -57,7 +56,6 @@ func New(
promEngine prometheus.Prometheus,
traceStmtBuilder qbtypes.StatementBuilder[qbtypes.TraceAggregation],
logStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
auditStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
metricStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
meterStmtBuilder qbtypes.StatementBuilder[qbtypes.MetricAggregation],
traceOperatorStmtBuilder qbtypes.TraceOperatorStatementBuilder,
@@ -71,7 +69,6 @@ func New(
promEngine: promEngine,
traceStmtBuilder: traceStmtBuilder,
logStmtBuilder: logStmtBuilder,
auditStmtBuilder: auditStmtBuilder,
metricStmtBuilder: metricStmtBuilder,
meterStmtBuilder: meterStmtBuilder,
traceOperatorStmtBuilder: traceOperatorStmtBuilder,
@@ -364,11 +361,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
spec.ShiftBy = extractShiftFromBuilderQuery(spec)
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
stmtBuilder := q.logStmtBuilder
if spec.Source == telemetrytypes.SourceAudit {
stmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, stmtBuilder, spec, timeRange, req.RequestType, tmplVars)
bq := newBuilderQuery(q.logger, q.telemetryStore, q.logStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
queries[spec.Name] = bq
steps[spec.Name] = spec.StepInterval
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
@@ -557,11 +550,7 @@ func (q *querier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qb
case <-tick:
// timestamp end is not specified here
timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: tsStart}, req.RequestType)
liveTailStmtBuilder := q.logStmtBuilder
if spec.Source == telemetrytypes.SourceAudit {
liveTailStmtBuilder = q.auditStmtBuilder
}
bq := newBuilderQuery(q.logger, q.telemetryStore, liveTailStmtBuilder, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
bq := newBuilderQuery(q.logger, q.telemetryStore, q.logStmtBuilder, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
"id": {
Value: updatedLogID,
},
@@ -861,11 +850,7 @@ func (q *querier) createRangedQuery(originalQuery qbtypes.Query, timeRange qbtyp
specCopy := qt.spec.Copy()
specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
shiftStmtBuilder := q.logStmtBuilder
if qt.spec.Source == telemetrytypes.SourceAudit {
shiftStmtBuilder = q.auditStmtBuilder
}
return newBuilderQuery(q.logger, q.telemetryStore, shiftStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
return newBuilderQuery(q.logger, q.telemetryStore, q.logStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
case *builderQuery[qbtypes.MetricAggregation]:
specCopy := qt.spec.Copy()

View File

@@ -47,7 +47,6 @@ func TestQueryRange_MetricTypeMissing(t *testing.T) {
nil, // prometheus
nil, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
nil, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
@@ -111,7 +110,6 @@ func TestQueryRange_MetricTypeFromStore(t *testing.T) {
nil, // prometheus
nil, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
&mockMetricStmtBuilder{}, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder

View File

@@ -9,7 +9,6 @@ import (
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/SigNoz/signoz/pkg/querier"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
@@ -64,11 +63,6 @@ func newProvider(
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
telemetrymetadata.DBName,
telemetrymetadata.AttributesMetadataLocalTableName,
telemetrymetadata.ColumnEvolutionMetadataTableName,
@@ -88,13 +82,13 @@ func newProvider(
telemetryStore,
)
// Create trace operator statement builder
// ADD: Create trace operator statement builder
traceOperatorStmtBuilder := telemetrytraces.NewTraceOperatorStatementBuilder(
settings,
telemetryMetadataStore,
traceFieldMapper,
traceConditionBuilder,
traceStmtBuilder,
traceStmtBuilder, // Pass the regular trace statement builder
traceAggExprRewriter,
)
@@ -118,26 +112,6 @@ func newProvider(
telemetrylogs.GetBodyJSONKey,
)
// Create audit statement builder
auditFieldMapper := telemetryaudit.NewFieldMapper()
auditConditionBuilder := telemetryaudit.NewConditionBuilder(auditFieldMapper)
auditAggExprRewriter := querybuilder.NewAggExprRewriter(
settings,
telemetryaudit.DefaultFullTextColumn,
auditFieldMapper,
auditConditionBuilder,
nil,
)
auditStmtBuilder := telemetryaudit.NewAuditQueryStatementBuilder(
settings,
telemetryMetadataStore,
auditFieldMapper,
auditConditionBuilder,
auditAggExprRewriter,
telemetryaudit.DefaultFullTextColumn,
nil,
)
// Create metric statement builder
metricFieldMapper := telemetrymetrics.NewFieldMapper()
metricConditionBuilder := telemetrymetrics.NewConditionBuilder(metricFieldMapper)
@@ -174,7 +148,6 @@ func newProvider(
prometheus,
traceStmtBuilder,
logStmtBuilder,
auditStmtBuilder,
metricStmtBuilder,
meterStmtBuilder,
traceOperatorStmtBuilder,

View File

@@ -63,7 +63,6 @@ import (
"github.com/SigNoz/signoz/pkg/query-service/postprocess"
"github.com/SigNoz/signoz/pkg/types"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
@@ -1138,19 +1137,12 @@ func (aH *APIHandler) Get(rw http.ResponseWriter, r *http.Request) {
}
dashboard := new(dashboardtypes.Dashboard)
if _, _, _, err := cloudintegrationtypes.ParseCloudIntegrationDashboardID(id); err == nil {
cloudIntegrationDashboard, err := aH.Signoz.Modules.CloudIntegration.GetDashboardByID(ctx, orgID, id)
if err != nil && !errorsV2.Ast(err, errorsV2.TypeLicenseUnavailable) {
render.Error(rw, errorsV2.Wrapf(err, errorsV2.TypeInternal, errorsV2.CodeInternal, "failed to get dashboard"))
if aH.CloudIntegrationsController.IsCloudIntegrationDashboardUuid(id) {
cloudIntegrationDashboard, apiErr := aH.CloudIntegrationsController.GetDashboardById(ctx, orgID, id)
if apiErr != nil {
render.Error(rw, errorsV2.Wrapf(apiErr, errorsV2.TypeInternal, errorsV2.CodeInternal, "failed to get dashboard"))
return
}
if cloudIntegrationDashboard == nil {
render.Error(rw, errorsV2.Newf(errorsV2.TypeNotFound, errorsV2.CodeNotFound, "dashboard not found"))
return
}
dashboard = cloudIntegrationDashboard
} else if aH.IntegrationsController.IsInstalledIntegrationDashboardID(id) {
integrationDashboard, apiErr := aH.IntegrationsController.GetInstalledIntegrationDashboardById(ctx, orgID, id)
@@ -1215,11 +1207,9 @@ func (aH *APIHandler) List(rw http.ResponseWriter, r *http.Request) {
dashboards = append(dashboards, installedIntegrationDashboards...)
}
cloudIntegrationDashboards, err := aH.Signoz.Modules.CloudIntegration.ListDashboards(ctx, orgID)
if err != nil {
if !errors.Ast(err, errorsV2.TypeLicenseUnavailable) {
aH.logger.ErrorContext(ctx, "failed to get dashboards for cloud integrations", errors.Attr(err))
}
cloudIntegrationDashboards, apiErr := aH.CloudIntegrationsController.AvailableDashboards(ctx, orgID)
if apiErr != nil {
aH.logger.ErrorContext(ctx, "failed to get dashboards for cloud integrations", errors.Attr(apiErr))
} else {
dashboards = append(dashboards, cloudIntegrationDashboards...)
}

View File

@@ -46,7 +46,6 @@ func prepareQuerierForMetrics(t *testing.T, telemetryStore telemetrystore.Teleme
nil, // prometheus
nil, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
metricStmtBuilder,
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
@@ -92,7 +91,6 @@ func prepareQuerierForLogs(telemetryStore telemetrystore.TelemetryStore, keysMap
nil, // prometheus
nil, // traceStmtBuilder
logStmtBuilder, // logStmtBuilder
nil, // auditStmtBuilder
nil, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder
@@ -133,7 +131,6 @@ func prepareQuerierForTraces(telemetryStore telemetrystore.TelemetryStore, keysM
nil, // prometheus
traceStmtBuilder, // traceStmtBuilder
nil, // logStmtBuilder
nil, // auditStmtBuilder
nil, // metricStmtBuilder
nil, // meterStmtBuilder
nil, // traceOperatorStmtBuilder

View File

@@ -21,7 +21,6 @@ import (
"github.com/SigNoz/signoz/pkg/global"
"github.com/SigNoz/signoz/pkg/identn"
"github.com/SigNoz/signoz/pkg/instrumentation"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/user"
@@ -124,9 +123,6 @@ type Config struct {
// ServiceAccount config
ServiceAccount serviceaccount.Config `mapstructure:"serviceaccount"`
// CloudIntegration config
CloudIntegration cloudintegration.Config `mapstructure:"cloudintegration"`
}
func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.ResolverConfig) (Config, error) {
@@ -157,7 +153,6 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
user.NewConfigFactory(),
identn.NewConfigFactory(),
serviceaccount.NewConfigFactory(),
cloudintegration.NewConfigFactory(),
}
conf, err := config.New(ctx, resolverConfig, configFactories)

View File

@@ -97,7 +97,7 @@ func NewHandlers(
QuerierHandler: querierHandler,
ServiceAccountHandler: implserviceaccount.NewHandler(modules.ServiceAccount),
RegistryHandler: registryHandler,
CloudIntegrationHandler: implcloudintegration.NewHandler(),
RuleStateHistory: implrulestatehistory.NewHandler(modules.RuleStateHistory),
CloudIntegrationHandler: implcloudintegration.NewHandler(modules.CloudIntegration),
}
}

View File

@@ -52,7 +52,8 @@ func TestNewHandlers(t *testing.T) {
userRoleStore := impluser.NewUserRoleStore(sqlstore, providerSettings)
userGetter := impluser.NewGetter(impluser.NewStore(sqlstore, providerSettings), userRoleStore, flagger)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, nil, nil)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore)
querierHandler := querier.NewHandler(providerSettings, nil, nil)
registryHandler := factory.NewHandler(nil)

View File

@@ -12,7 +12,6 @@ import (
"github.com/SigNoz/signoz/pkg/modules/apdex/implapdex"
"github.com/SigNoz/signoz/pkg/modules/authdomain"
"github.com/SigNoz/signoz/pkg/modules/authdomain/implauthdomain"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
"github.com/SigNoz/signoz/pkg/modules/dashboard"
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer/implmetricsexplorer"
@@ -31,6 +30,7 @@ import (
"github.com/SigNoz/signoz/pkg/modules/savedview"
"github.com/SigNoz/signoz/pkg/modules/savedview/implsavedview"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount/implserviceaccount"
"github.com/SigNoz/signoz/pkg/modules/services"
"github.com/SigNoz/signoz/pkg/modules/services/implservices"
"github.com/SigNoz/signoz/pkg/modules/session"
@@ -71,7 +71,6 @@ type Modules struct {
MetricsExplorer metricsexplorer.Module
Promote promote.Module
ServiceAccount serviceaccount.Module
CloudIntegration cloudintegration.Module
RuleStateHistory rulestatehistory.Module
}
@@ -94,8 +93,6 @@ func NewModules(
dashboard dashboard.Module,
userGetter user.Getter,
userRoleStore authtypes.UserRoleStore,
serviceAccount serviceaccount.Module,
cloudIntegrationModule cloudintegration.Module,
) Modules {
quickfilter := implquickfilter.NewModule(implquickfilter.NewStore(sqlstore))
orgSetter := implorganization.NewSetter(implorganization.NewStore(sqlstore), alertmanager, quickfilter)
@@ -120,8 +117,7 @@ func NewModules(
Services: implservices.NewModule(querier, telemetryStore),
MetricsExplorer: implmetricsexplorer.NewModule(telemetryStore, telemetryMetadataStore, cache, ruleStore, dashboard, providerSettings, config.MetricsExplorer),
Promote: implpromote.NewModule(telemetryMetadataStore, telemetryStore),
ServiceAccount: serviceAccount,
ServiceAccount: implserviceaccount.NewModule(implserviceaccount.NewStore(sqlstore), authz, cache, analytics, providerSettings, config.ServiceAccount),
RuleStateHistory: implrulestatehistory.NewModule(implrulestatehistory.NewStore(telemetryStore, telemetryMetadataStore, providerSettings.Logger)),
CloudIntegration: cloudIntegrationModule,
}
}

View File

@@ -13,11 +13,8 @@ import (
"github.com/SigNoz/signoz/pkg/factory/factorytest"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount/implserviceaccount"
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
"github.com/SigNoz/signoz/pkg/queryparser"
"github.com/SigNoz/signoz/pkg/sharder"
@@ -54,9 +51,7 @@ func TestNewModules(t *testing.T) {
userGetter := impluser.NewGetter(impluser.NewStore(sqlstore, providerSettings), userRoleStore, flagger)
serviceAccount := implserviceaccount.NewModule(implserviceaccount.NewStore(sqlstore), nil, nil, nil, providerSettings, serviceaccount.Config{})
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore, serviceAccount, implcloudintegration.NewModule())
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, nil, nil, nil, nil, nil, nil, nil, queryParser, Config{}, dashboardModule, userGetter, userRoleStore)
reflectVal := reflect.ValueOf(modules)
for i := 0; i < reflectVal.NumField(); i++ {

View File

@@ -17,17 +17,12 @@ import (
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/gateway"
"github.com/SigNoz/signoz/pkg/global"
"github.com/SigNoz/signoz/pkg/identn"
"github.com/SigNoz/signoz/pkg/instrumentation"
"github.com/SigNoz/signoz/pkg/licensing"
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
pkgimplcloudintegration "github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
"github.com/SigNoz/signoz/pkg/modules/dashboard"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
"github.com/SigNoz/signoz/pkg/modules/serviceaccount/implserviceaccount"
"github.com/SigNoz/signoz/pkg/modules/user/impluser"
"github.com/SigNoz/signoz/pkg/prometheus"
"github.com/SigNoz/signoz/pkg/querier"
@@ -38,7 +33,6 @@ import (
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/statsreporter"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetadata"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
@@ -47,7 +41,6 @@ import (
"github.com/SigNoz/signoz/pkg/telemetrytraces"
pkgtokenizer "github.com/SigNoz/signoz/pkg/tokenizer"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/version"
"github.com/SigNoz/signoz/pkg/zeus"
@@ -101,7 +94,6 @@ func New(
dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing) dashboard.Module,
gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config],
querierHandlerCallback func(factory.ProviderSettings, querier.Querier, analytics.Analytics) querier.Handler,
cloudIntegrationCallback func(cloudintegrationtypes.Store, global.Global, zeus.Zeus, gateway.Gateway, licensing.Licensing, serviceaccount.Module, cloudintegration.Config) (cloudintegration.Module, error),
) (*SigNoz, error) {
// Initialize instrumentation
instrumentation, err := instrumentation.New(ctx, config.Instrumentation, version.Info, "signoz")
@@ -403,11 +395,6 @@ func New(
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
telemetrymetadata.DBName,
telemetrymetadata.AttributesMetadataLocalTableName,
telemetrymetadata.ColumnEvolutionMetadataTableName,
@@ -424,19 +411,11 @@ func New(
return nil, err
}
serviceAccount := implserviceaccount.NewModule(implserviceaccount.NewStore(sqlstore), authz, cache, analytics, providerSettings, config.ServiceAccount)
cloudIntegrationStore := pkgimplcloudintegration.NewStore(sqlstore)
cloudIntegrationModule, err := cloudIntegrationCallback(cloudIntegrationStore, global, zeus, gateway, licensing, serviceAccount, config.CloudIntegration)
if err != nil {
return nil, err
}
// Initialize all modules
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter, userRoleStore, serviceAccount, cloudIntegrationModule)
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter, userRoleStore)
// Initialize identN resolver
identNFactories := NewIdentNProviderFactories(tokenizer, serviceAccount, orgGetter, userGetter, config.User)
identNFactories := NewIdentNProviderFactories(tokenizer, modules.ServiceAccount, orgGetter, userGetter, config.User)
identNResolver, err := identn.NewIdentNResolver(ctx, providerSettings, config.IdentN, identNFactories)
if err != nil {
return nil, err

View File

@@ -1,200 +0,0 @@
package telemetryaudit
import (
"context"
"fmt"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/huandu/go-sqlbuilder"
)
type conditionBuilder struct {
fm qbtypes.FieldMapper
}
func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
return &conditionBuilder{fm: fm}
}
func (c *conditionBuilder) conditionFor(
ctx context.Context,
startNs, endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
operator qbtypes.FilterOperator,
value any,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
columns, err := c.fm.ColumnFor(ctx, startNs, endNs, key)
if err != nil {
return "", err
}
if operator.IsStringSearchOperator() {
value = querybuilder.FormatValueForContains(value)
}
fieldExpression, err := c.fm.FieldFor(ctx, startNs, endNs, key)
if err != nil {
return "", err
}
fieldExpression, value = querybuilder.DataTypeCollisionHandledFieldName(key, value, fieldExpression, operator)
switch operator {
case qbtypes.FilterOperatorEqual:
return sb.E(fieldExpression, value), nil
case qbtypes.FilterOperatorNotEqual:
return sb.NE(fieldExpression, value), nil
case qbtypes.FilterOperatorGreaterThan:
return sb.G(fieldExpression, value), nil
case qbtypes.FilterOperatorGreaterThanOrEq:
return sb.GE(fieldExpression, value), nil
case qbtypes.FilterOperatorLessThan:
return sb.LT(fieldExpression, value), nil
case qbtypes.FilterOperatorLessThanOrEq:
return sb.LE(fieldExpression, value), nil
case qbtypes.FilterOperatorLike:
return sb.Like(fieldExpression, value), nil
case qbtypes.FilterOperatorNotLike:
return sb.NotLike(fieldExpression, value), nil
case qbtypes.FilterOperatorILike:
return sb.ILike(fieldExpression, value), nil
case qbtypes.FilterOperatorNotILike:
return sb.NotILike(fieldExpression, value), nil
case qbtypes.FilterOperatorContains:
return sb.ILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
case qbtypes.FilterOperatorNotContains:
return sb.NotILike(fieldExpression, fmt.Sprintf("%%%s%%", value)), nil
case qbtypes.FilterOperatorRegexp:
return fmt.Sprintf(`match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
case qbtypes.FilterOperatorNotRegexp:
return fmt.Sprintf(`NOT match(%s, %s)`, sqlbuilder.Escape(fieldExpression), sb.Var(value)), nil
case qbtypes.FilterOperatorBetween:
values, ok := value.([]any)
if !ok {
return "", qbtypes.ErrBetweenValues
}
if len(values) != 2 {
return "", qbtypes.ErrBetweenValues
}
return sb.Between(fieldExpression, values[0], values[1]), nil
case qbtypes.FilterOperatorNotBetween:
values, ok := value.([]any)
if !ok {
return "", qbtypes.ErrBetweenValues
}
if len(values) != 2 {
return "", qbtypes.ErrBetweenValues
}
return sb.NotBetween(fieldExpression, values[0], values[1]), nil
case qbtypes.FilterOperatorIn:
values, ok := value.([]any)
if !ok {
return "", qbtypes.ErrInValues
}
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.E(fieldExpression, value))
}
return sb.Or(conditions...), nil
case qbtypes.FilterOperatorNotIn:
values, ok := value.([]any)
if !ok {
return "", qbtypes.ErrInValues
}
conditions := []string{}
for _, value := range values {
conditions = append(conditions, sb.NE(fieldExpression, value))
}
return sb.And(conditions...), nil
case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
var value any
column := columns[0]
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
if operator == qbtypes.FilterOperatorExists {
return sb.IsNotNull(fieldExpression), nil
}
return sb.IsNull(fieldExpression), nil
case schema.ColumnTypeEnumLowCardinality:
switch elementType := column.Type.(schema.LowCardinalityColumnType).ElementType; elementType.GetType() {
case schema.ColumnTypeEnumString:
value = ""
if operator == qbtypes.FilterOperatorExists {
return sb.NE(fieldExpression, value), nil
}
return sb.E(fieldExpression, value), nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for low cardinality column type %s", elementType)
}
case schema.ColumnTypeEnumString:
value = ""
if operator == qbtypes.FilterOperatorExists {
return sb.NE(fieldExpression, value), nil
}
return sb.E(fieldExpression, value), nil
case schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
value = 0
if operator == qbtypes.FilterOperatorExists {
return sb.NE(fieldExpression, value), nil
}
return sb.E(fieldExpression, value), nil
case schema.ColumnTypeEnumMap:
keyType := column.Type.(schema.MapColumnType).KeyType
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
}
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
if key.Materialized {
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
}
if operator == qbtypes.FilterOperatorExists {
return sb.E(leftOperand, true), nil
}
return sb.NE(leftOperand, true), nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
}
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for column type %s", column.Type)
}
}
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
}
func (c *conditionBuilder) ConditionFor(
ctx context.Context,
startNs uint64,
endNs uint64,
key *telemetrytypes.TelemetryFieldKey,
operator qbtypes.FilterOperator,
value any,
sb *sqlbuilder.SelectBuilder,
) (string, error) {
condition, err := c.conditionFor(ctx, startNs, endNs, key, operator, value, sb)
if err != nil {
return "", err
}
if key.FieldContext == telemetrytypes.FieldContextLog || key.FieldContext == telemetrytypes.FieldContextScope {
return condition, nil
}
if operator.AddDefaultExistsFilter() {
existsCondition, err := c.conditionFor(ctx, startNs, endNs, key, qbtypes.FilterOperatorExists, nil, sb)
if err != nil {
return "", err
}
return sb.And(condition, existsCondition), nil
}
return condition, nil
}

View File

@@ -1,129 +0,0 @@
package telemetryaudit
import (
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
const (
// Internal Columns.
IDColumn = "id"
TimestampBucketStartColumn = "ts_bucket_start"
ResourceFingerPrintColumn = "resource_fingerprint"
// Intrinsic Columns.
TimestampColumn = "timestamp"
ObservedTimestampColumn = "observed_timestamp"
BodyColumn = "body"
EventNameColumn = "event_name"
TraceIDColumn = "trace_id"
SpanIDColumn = "span_id"
TraceFlagsColumn = "trace_flags"
SeverityTextColumn = "severity_text"
SeverityNumberColumn = "severity_number"
ScopeNameColumn = "scope_name"
ScopeVersionColumn = "scope_version"
// Contextual Columns.
AttributesStringColumn = "attributes_string"
AttributesNumberColumn = "attributes_number"
AttributesBoolColumn = "attributes_bool"
ResourceColumn = "resource"
ScopeStringColumn = "scope_string"
)
var (
DefaultFullTextColumn = &telemetrytypes.TelemetryFieldKey{
Name: "body",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
}
IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{
"body": {
Name: "body",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"trace_id": {
Name: "trace_id",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"span_id": {
Name: "span_id",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"trace_flags": {
Name: "trace_flags",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
},
"severity_text": {
Name: "severity_text",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
"severity_number": {
Name: "severity_number",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
},
"event_name": {
Name: "event_name",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
}
DefaultSortingOrder = []qbtypes.OrderBy{
{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: TimestampColumn,
},
},
Direction: qbtypes.OrderDirectionDesc,
},
{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: IDColumn,
},
},
Direction: qbtypes.OrderDirectionDesc,
},
}
)
var auditLogColumns = map[string]*schema.Column{
"ts_bucket_start": {Name: "ts_bucket_start", Type: schema.ColumnTypeUInt64},
"resource_fingerprint": {Name: "resource_fingerprint", Type: schema.ColumnTypeString},
"timestamp": {Name: "timestamp", Type: schema.ColumnTypeUInt64},
"observed_timestamp": {Name: "observed_timestamp", Type: schema.ColumnTypeUInt64},
"id": {Name: "id", Type: schema.ColumnTypeString},
"trace_id": {Name: "trace_id", Type: schema.ColumnTypeString},
"span_id": {Name: "span_id", Type: schema.ColumnTypeString},
"trace_flags": {Name: "trace_flags", Type: schema.ColumnTypeUInt32},
"severity_text": {Name: "severity_text", Type: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}},
"severity_number": {Name: "severity_number", Type: schema.ColumnTypeUInt8},
"body": {Name: "body", Type: schema.ColumnTypeString},
"attributes_string": {Name: "attributes_string", Type: schema.MapColumnType{KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, ValueType: schema.ColumnTypeString}},
"attributes_number": {Name: "attributes_number", Type: schema.MapColumnType{KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, ValueType: schema.ColumnTypeFloat64}},
"attributes_bool": {Name: "attributes_bool", Type: schema.MapColumnType{KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, ValueType: schema.ColumnTypeBool}},
"resource": {Name: "resource", Type: schema.JSONColumnType{}},
"event_name": {Name: "event_name", Type: schema.ColumnTypeString},
"scope_name": {Name: "scope_name", Type: schema.ColumnTypeString},
"scope_version": {Name: "scope_version", Type: schema.ColumnTypeString},
"scope_string": {Name: "scope_string", Type: schema.MapColumnType{KeyType: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}, ValueType: schema.ColumnTypeString}},
}

View File

@@ -1,124 +0,0 @@
package telemetryaudit
import (
"context"
"fmt"
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/errors"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/huandu/go-sqlbuilder"
"golang.org/x/exp/maps"
)
type fieldMapper struct{}
func NewFieldMapper() qbtypes.FieldMapper {
return &fieldMapper{}
}
func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
switch key.FieldContext {
case telemetrytypes.FieldContextResource:
return []*schema.Column{auditLogColumns["resource"]}, nil
case telemetrytypes.FieldContextScope:
switch key.Name {
case "name", "scope.name", "scope_name":
return []*schema.Column{auditLogColumns["scope_name"]}, nil
case "version", "scope.version", "scope_version":
return []*schema.Column{auditLogColumns["scope_version"]}, nil
}
return []*schema.Column{auditLogColumns["scope_string"]}, nil
case telemetrytypes.FieldContextAttribute:
switch key.FieldDataType {
case telemetrytypes.FieldDataTypeString:
return []*schema.Column{auditLogColumns["attributes_string"]}, nil
case telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeFloat64, telemetrytypes.FieldDataTypeNumber:
return []*schema.Column{auditLogColumns["attributes_number"]}, nil
case telemetrytypes.FieldDataTypeBool:
return []*schema.Column{auditLogColumns["attributes_bool"]}, nil
}
case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextUnspecified:
col, ok := auditLogColumns[key.Name]
if !ok {
return nil, qbtypes.ErrColumnNotFound
}
return []*schema.Column{col}, nil
}
return nil, qbtypes.ErrColumnNotFound
}
func (m *fieldMapper) FieldFor(ctx context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) (string, error) {
columns, err := m.getColumn(ctx, key)
if err != nil {
return "", err
}
if len(columns) != 1 {
return "", errors.Newf(errors.TypeInternal, errors.CodeInternal, "expected exactly 1 column, got %d", len(columns))
}
column := columns[0]
switch column.Type.GetType() {
case schema.ColumnTypeEnumJSON:
if key.FieldContext != telemetrytypes.FieldContextResource {
return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns in audit, got %s", key.FieldContext.String)
}
return fmt.Sprintf("%s.`%s`::String", column.Name, key.Name), nil
case schema.ColumnTypeEnumLowCardinality:
return column.Name, nil
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumUInt64, schema.ColumnTypeEnumUInt32, schema.ColumnTypeEnumUInt8:
return column.Name, nil
case schema.ColumnTypeEnumMap:
keyType := column.Type.(schema.MapColumnType).KeyType
if _, ok := keyType.(schema.LowCardinalityColumnType); !ok {
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "key type %s is not supported for map column type %s", keyType, column.Type)
}
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
if key.Materialized {
return telemetrytypes.FieldKeyToMaterializedColumnName(key), nil
}
return fmt.Sprintf("%s['%s']", column.Name, key.Name), nil
default:
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported map value type %s", valueType)
}
}
return column.Name, nil
}
func (m *fieldMapper) ColumnFor(ctx context.Context, _, _ uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
return m.getColumn(ctx, key)
}
func (m *fieldMapper) ColumnExpressionFor(
ctx context.Context,
tsStart, tsEnd uint64,
field *telemetrytypes.TelemetryFieldKey,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
) (string, error) {
fieldExpression, err := m.FieldFor(ctx, tsStart, tsEnd, field)
if errors.Is(err, qbtypes.ErrColumnNotFound) {
keysForField := keys[field.Name]
if len(keysForField) == 0 {
if _, ok := auditLogColumns[field.Name]; ok {
field.FieldContext = telemetrytypes.FieldContextLog
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, field)
} else {
correction, found := telemetrytypes.SuggestCorrection(field.Name, maps.Keys(keys))
if found {
return "", errors.Wrap(err, errors.TypeInvalidInput, errors.CodeInvalidInput, correction)
}
return "", errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name)
}
} else {
fieldExpression, _ = m.FieldFor(ctx, tsStart, tsEnd, keysForField[0])
}
}
return fmt.Sprintf("%s AS `%s`", sqlbuilder.Escape(fieldExpression), field.Name), nil
}

View File

@@ -1,612 +0,0 @@
package telemetryaudit
import (
"context"
"fmt"
"log/slog"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryresourcefilter"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/huandu/go-sqlbuilder"
)
type auditQueryStatementBuilder struct {
logger *slog.Logger
metadataStore telemetrytypes.MetadataStore
fm qbtypes.FieldMapper
cb qbtypes.ConditionBuilder
resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation]
aggExprRewriter qbtypes.AggExprRewriter
fullTextColumn *telemetrytypes.TelemetryFieldKey
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
}
var _ qbtypes.StatementBuilder[qbtypes.LogAggregation] = (*auditQueryStatementBuilder)(nil)
func NewAuditQueryStatementBuilder(
settings factory.ProviderSettings,
metadataStore telemetrytypes.MetadataStore,
fieldMapper qbtypes.FieldMapper,
conditionBuilder qbtypes.ConditionBuilder,
aggExprRewriter qbtypes.AggExprRewriter,
fullTextColumn *telemetrytypes.TelemetryFieldKey,
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
) *auditQueryStatementBuilder {
auditSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetryaudit")
resourceFilterStmtBuilder := telemetryresourcefilter.New[qbtypes.LogAggregation](
settings,
DBName,
LogsResourceTableName,
telemetrytypes.SignalLogs,
telemetrytypes.SourceAudit,
metadataStore,
fullTextColumn,
jsonKeyToKey,
)
return &auditQueryStatementBuilder{
logger: auditSettings.Logger(),
metadataStore: metadataStore,
fm: fieldMapper,
cb: conditionBuilder,
resourceFilterStmtBuilder: resourceFilterStmtBuilder,
aggExprRewriter: aggExprRewriter,
fullTextColumn: fullTextColumn,
jsonKeyToKey: jsonKeyToKey,
}
}
func (b *auditQueryStatementBuilder) Build(
ctx context.Context,
start uint64,
end uint64,
requestType qbtypes.RequestType,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
start = querybuilder.ToNanoSecs(start)
end = querybuilder.ToNanoSecs(end)
keySelectors := getKeySelectors(query)
keys, _, err := b.metadataStore.GetKeysMulti(ctx, keySelectors)
if err != nil {
return nil, err
}
query = b.adjustKeys(ctx, keys, query, requestType)
q := sqlbuilder.NewSelectBuilder()
var stmt *qbtypes.Statement
switch requestType {
case qbtypes.RequestTypeRaw, qbtypes.RequestTypeRawStream:
stmt, err = b.buildListQuery(ctx, q, query, start, end, keys, variables)
case qbtypes.RequestTypeTimeSeries:
stmt, err = b.buildTimeSeriesQuery(ctx, q, query, start, end, keys, variables)
case qbtypes.RequestTypeScalar:
stmt, err = b.buildScalarQuery(ctx, q, query, start, end, keys, false, variables)
default:
return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported request type: %s", requestType)
}
if err != nil {
return nil, err
}
return stmt, nil
}
func getKeySelectors(query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) []*telemetrytypes.FieldKeySelector {
var keySelectors []*telemetrytypes.FieldKeySelector
for idx := range query.Aggregations {
aggExpr := query.Aggregations[idx]
selectors := querybuilder.QueryStringToKeysSelectors(aggExpr.Expression)
keySelectors = append(keySelectors, selectors...)
}
if query.Filter != nil && query.Filter.Expression != "" {
whereClauseSelectors := querybuilder.QueryStringToKeysSelectors(query.Filter.Expression)
keySelectors = append(keySelectors, whereClauseSelectors...)
}
for idx := range query.GroupBy {
groupBy := query.GroupBy[idx]
keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{
Name: groupBy.Name,
Signal: telemetrytypes.SignalLogs,
FieldContext: groupBy.FieldContext,
FieldDataType: groupBy.FieldDataType,
})
}
for idx := range query.SelectFields {
selectField := query.SelectFields[idx]
keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{
Name: selectField.Name,
Signal: telemetrytypes.SignalLogs,
FieldContext: selectField.FieldContext,
FieldDataType: selectField.FieldDataType,
})
}
for idx := range query.Order {
keySelectors = append(keySelectors, &telemetrytypes.FieldKeySelector{
Name: query.Order[idx].Key.Name,
Signal: telemetrytypes.SignalLogs,
FieldContext: query.Order[idx].Key.FieldContext,
FieldDataType: query.Order[idx].Key.FieldDataType,
})
}
for idx := range keySelectors {
keySelectors[idx].Signal = telemetrytypes.SignalLogs
keySelectors[idx].Source = telemetrytypes.SourceAudit
keySelectors[idx].SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact
}
return keySelectors
}
func (b *auditQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[string][]*telemetrytypes.TelemetryFieldKey, query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation], requestType qbtypes.RequestType) qbtypes.QueryBuilderQuery[qbtypes.LogAggregation] {
keys["id"] = append([]*telemetrytypes.TelemetryFieldKey{{
Name: "id",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeString,
}}, keys["id"]...)
keys["timestamp"] = append([]*telemetrytypes.TelemetryFieldKey{{
Name: "timestamp",
Signal: telemetrytypes.SignalLogs,
FieldContext: telemetrytypes.FieldContextLog,
FieldDataType: telemetrytypes.FieldDataTypeNumber,
}}, keys["timestamp"]...)
actions := querybuilder.AdjustKeysForAliasExpressions(&query, requestType)
actions = append(actions, querybuilder.AdjustDuplicateKeys(&query)...)
for idx := range query.SelectFields {
actions = append(actions, b.adjustKey(&query.SelectFields[idx], keys)...)
}
for idx := range query.GroupBy {
actions = append(actions, b.adjustKey(&query.GroupBy[idx].TelemetryFieldKey, keys)...)
}
for idx := range query.Order {
actions = append(actions, b.adjustKey(&query.Order[idx].Key.TelemetryFieldKey, keys)...)
}
for _, action := range actions {
b.logger.InfoContext(ctx, "key adjustment action", slog.String("action", action))
}
return query
}
func (b *auditQueryStatementBuilder) adjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemetrytypes.TelemetryFieldKey) []string {
if _, ok := IntrinsicFields[key.Name]; ok {
intrinsicField := IntrinsicFields[key.Name]
return querybuilder.AdjustKey(key, keys, &intrinsicField)
}
return querybuilder.AdjustKey(key, keys, nil)
}
func (b *auditQueryStatementBuilder) buildListQuery(
ctx context.Context,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
var (
cteFragments []string
cteArgs [][]any
)
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
} else if frag != "" {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
sb.Select(TimestampColumn)
sb.SelectMore(IDColumn)
if len(query.SelectFields) == 0 {
sb.SelectMore(TraceIDColumn)
sb.SelectMore(SpanIDColumn)
sb.SelectMore(TraceFlagsColumn)
sb.SelectMore(SeverityTextColumn)
sb.SelectMore(SeverityNumberColumn)
sb.SelectMore(ScopeNameColumn)
sb.SelectMore(ScopeVersionColumn)
sb.SelectMore(BodyColumn)
sb.SelectMore(EventNameColumn)
sb.SelectMore(AttributesStringColumn)
sb.SelectMore(AttributesNumberColumn)
sb.SelectMore(AttributesBoolColumn)
sb.SelectMore(ResourceColumn)
sb.SelectMore(ScopeStringColumn)
} else {
for index := range query.SelectFields {
if query.SelectFields[index].Name == TimestampColumn || query.SelectFields[index].Name == IDColumn {
continue
}
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &query.SelectFields[index], keys)
if err != nil {
return nil, err
}
sb.SelectMore(colExpr)
}
}
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
}
for _, orderBy := range query.Order {
colExpr, err := b.fm.ColumnExpressionFor(ctx, start, end, &orderBy.Key.TelemetryFieldKey, keys)
if err != nil {
return nil, err
}
sb.OrderBy(fmt.Sprintf("%s %s", colExpr, orderBy.Direction.StringValue()))
}
if query.Limit > 0 {
sb.Limit(query.Limit)
} else {
sb.Limit(100)
}
if query.Offset > 0 {
sb.Offset(query.Offset)
}
mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
finalSQL := querybuilder.CombineCTEs(cteFragments) + mainSQL
finalArgs := querybuilder.PrependArgs(cteArgs, mainArgs)
stmt := &qbtypes.Statement{
Query: finalSQL,
Args: finalArgs,
}
if preparedWhereClause != nil {
stmt.Warnings = preparedWhereClause.Warnings
stmt.WarningsDocURL = preparedWhereClause.WarningsDocURL
}
return stmt, nil
}
func (b *auditQueryStatementBuilder) buildTimeSeriesQuery(
ctx context.Context,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
var (
cteFragments []string
cteArgs [][]any
)
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
} else if frag != "" {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
sb.SelectMore(fmt.Sprintf(
"toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL %d SECOND) AS ts",
int64(query.StepInterval.Seconds()),
))
var allGroupByArgs []any
fieldNames := make([]string, 0, len(query.GroupBy))
for _, gb := range query.GroupBy {
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
if err != nil {
return nil, err
}
colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.Name)
allGroupByArgs = append(allGroupByArgs, args...)
sb.SelectMore(colExpr)
fieldNames = append(fieldNames, fmt.Sprintf("`%s`", gb.Name))
}
allAggChArgs := make([]any, 0)
for i, agg := range query.Aggregations {
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(ctx, start, end, agg.Expression, uint64(query.StepInterval.Seconds()), keys)
if err != nil {
return nil, err
}
allAggChArgs = append(allAggChArgs, chArgs...)
sb.SelectMore(fmt.Sprintf("%s AS __result_%d", rewritten, i))
}
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
}
var finalSQL string
var finalArgs []any
if query.Limit > 0 && len(query.GroupBy) > 0 {
cteSB := sqlbuilder.NewSelectBuilder()
cteStmt, err := b.buildScalarQuery(ctx, cteSB, query, start, end, keys, true, variables)
if err != nil {
return nil, err
}
cteFragments = append(cteFragments, fmt.Sprintf("__limit_cte AS (%s)", cteStmt.Query))
cteArgs = append(cteArgs, cteStmt.Args)
tuple := fmt.Sprintf("(%s)", strings.Join(fieldNames, ", "))
sb.Where(fmt.Sprintf("%s GLOBAL IN (SELECT %s FROM __limit_cte)", tuple, strings.Join(fieldNames, ", ")))
sb.GroupBy("ts")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForLogs(query.Having.Expression, query.Aggregations)
if err != nil {
return nil, err
}
sb.Having(rewrittenExpr)
}
if len(query.Order) != 0 {
for _, orderBy := range query.Order {
_, ok := aggOrderBy(orderBy, query)
if !ok {
sb.OrderBy(fmt.Sprintf("`%s` %s", orderBy.Key.Name, orderBy.Direction.StringValue()))
}
}
sb.OrderBy("ts desc")
}
combinedArgs := append(allGroupByArgs, allAggChArgs...)
mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...)
finalSQL = querybuilder.CombineCTEs(cteFragments) + mainSQL
finalArgs = querybuilder.PrependArgs(cteArgs, mainArgs)
} else {
sb.GroupBy("ts")
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForLogs(query.Having.Expression, query.Aggregations)
if err != nil {
return nil, err
}
sb.Having(rewrittenExpr)
}
if len(query.Order) != 0 {
for _, orderBy := range query.Order {
_, ok := aggOrderBy(orderBy, query)
if !ok {
sb.OrderBy(fmt.Sprintf("`%s` %s", orderBy.Key.Name, orderBy.Direction.StringValue()))
}
}
sb.OrderBy("ts desc")
}
combinedArgs := append(allGroupByArgs, allAggChArgs...)
mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...)
finalSQL = querybuilder.CombineCTEs(cteFragments) + mainSQL
finalArgs = querybuilder.PrependArgs(cteArgs, mainArgs)
}
stmt := &qbtypes.Statement{
Query: finalSQL,
Args: finalArgs,
}
if preparedWhereClause != nil {
stmt.Warnings = preparedWhereClause.Warnings
stmt.WarningsDocURL = preparedWhereClause.WarningsDocURL
}
return stmt, nil
}
func (b *auditQueryStatementBuilder) buildScalarQuery(
ctx context.Context,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
keys map[string][]*telemetrytypes.TelemetryFieldKey,
skipResourceCTE bool,
variables map[string]qbtypes.VariableItem,
) (*qbtypes.Statement, error) {
var (
cteFragments []string
cteArgs [][]any
)
if frag, args, err := b.maybeAttachResourceFilter(ctx, sb, query, start, end, variables); err != nil {
return nil, err
} else if frag != "" && !skipResourceCTE {
cteFragments = append(cteFragments, frag)
cteArgs = append(cteArgs, args)
}
allAggChArgs := []any{}
var allGroupByArgs []any
for _, gb := range query.GroupBy {
expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, start, end, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
if err != nil {
return nil, err
}
colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.Name)
allGroupByArgs = append(allGroupByArgs, args...)
sb.SelectMore(colExpr)
}
rateInterval := (end - start) / querybuilder.NsToSeconds
if len(query.Aggregations) > 0 {
for idx := range query.Aggregations {
aggExpr := query.Aggregations[idx]
rewritten, chArgs, err := b.aggExprRewriter.Rewrite(ctx, start, end, aggExpr.Expression, rateInterval, keys)
if err != nil {
return nil, err
}
allAggChArgs = append(allAggChArgs, chArgs...)
sb.SelectMore(fmt.Sprintf("%s AS __result_%d", rewritten, idx))
}
}
sb.From(fmt.Sprintf("%s.%s", DBName, AuditLogsTableName))
preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
if err != nil {
return nil, err
}
sb.GroupBy(querybuilder.GroupByKeys(query.GroupBy)...)
if query.Having != nil && query.Having.Expression != "" {
rewriter := querybuilder.NewHavingExpressionRewriter()
rewrittenExpr, err := rewriter.RewriteForLogs(query.Having.Expression, query.Aggregations)
if err != nil {
return nil, err
}
sb.Having(rewrittenExpr)
}
for _, orderBy := range query.Order {
idx, ok := aggOrderBy(orderBy, query)
if ok {
sb.OrderBy(fmt.Sprintf("__result_%d %s", idx, orderBy.Direction.StringValue()))
} else {
sb.OrderBy(fmt.Sprintf("`%s` %s", orderBy.Key.Name, orderBy.Direction.StringValue()))
}
}
if len(query.Order) == 0 {
sb.OrderBy("__result_0 DESC")
}
if query.Limit > 0 {
sb.Limit(query.Limit)
}
combinedArgs := append(allGroupByArgs, allAggChArgs...)
mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...)
finalSQL := querybuilder.CombineCTEs(cteFragments) + mainSQL
finalArgs := querybuilder.PrependArgs(cteArgs, mainArgs)
stmt := &qbtypes.Statement{
Query: finalSQL,
Args: finalArgs,
}
if preparedWhereClause != nil {
stmt.Warnings = preparedWhereClause.Warnings
stmt.WarningsDocURL = preparedWhereClause.WarningsDocURL
}
return stmt, nil
}
func (b *auditQueryStatementBuilder) addFilterCondition(
ctx context.Context,
sb *sqlbuilder.SelectBuilder,
start, end uint64,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
keys map[string][]*telemetrytypes.TelemetryFieldKey,
variables map[string]qbtypes.VariableItem,
) (*querybuilder.PreparedWhereClause, error) {
var preparedWhereClause *querybuilder.PreparedWhereClause
var err error
if query.Filter != nil && query.Filter.Expression != "" {
preparedWhereClause, err = querybuilder.PrepareWhereClause(query.Filter.Expression, querybuilder.FilterExprVisitorOpts{
Context: ctx,
Logger: b.logger,
FieldMapper: b.fm,
ConditionBuilder: b.cb,
FieldKeys: keys,
SkipResourceFilter: true,
FullTextColumn: b.fullTextColumn,
JsonKeyToKey: b.jsonKeyToKey,
Variables: variables,
StartNs: start,
EndNs: end,
})
if err != nil {
return nil, err
}
}
if preparedWhereClause != nil {
sb.AddWhereClause(preparedWhereClause.WhereClause)
}
startBucket := start/querybuilder.NsToSeconds - querybuilder.BucketAdjustment
var endBucket uint64
if end != 0 {
endBucket = end / querybuilder.NsToSeconds
}
if start != 0 {
sb.Where(sb.GE("timestamp", fmt.Sprintf("%d", start)), sb.GE("ts_bucket_start", startBucket))
}
if end != 0 {
sb.Where(sb.L("timestamp", fmt.Sprintf("%d", end)), sb.LE("ts_bucket_start", endBucket))
}
return preparedWhereClause, nil
}
func aggOrderBy(k qbtypes.OrderBy, q qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]) (int, bool) {
for i, agg := range q.Aggregations {
if k.Key.Name == agg.Alias || k.Key.Name == agg.Expression || k.Key.Name == fmt.Sprintf("%d", i) {
return i, true
}
}
return 0, false
}
func (b *auditQueryStatementBuilder) maybeAttachResourceFilter(
ctx context.Context,
sb *sqlbuilder.SelectBuilder,
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation],
start, end uint64,
variables map[string]qbtypes.VariableItem,
) (cteSQL string, cteArgs []any, err error) {
stmt, err := b.resourceFilterStmtBuilder.Build(ctx, start, end, qbtypes.RequestTypeRaw, query, variables)
if err != nil {
return "", nil, err
}
sb.Where("resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter)")
return fmt.Sprintf("__resource_filter AS (%s)", stmt.Query), stmt.Args, nil
}

View File

@@ -1,223 +0,0 @@
package telemetryaudit
import (
"context"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/querybuilder"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
"github.com/stretchr/testify/require"
)
func auditFieldKeyMap() map[string][]*telemetrytypes.TelemetryFieldKey {
key := func(name string, ctx telemetrytypes.FieldContext, dt telemetrytypes.FieldDataType, materialized bool) *telemetrytypes.TelemetryFieldKey {
return &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: ctx,
FieldDataType: dt,
Materialized: materialized,
}
}
attr := telemetrytypes.FieldContextAttribute
res := telemetrytypes.FieldContextResource
str := telemetrytypes.FieldDataTypeString
i64 := telemetrytypes.FieldDataTypeInt64
return map[string][]*telemetrytypes.TelemetryFieldKey{
"service.name": {key("service.name", res, str, false)},
"signoz.audit.action": {key("signoz.audit.action", attr, str, true)},
"signoz.audit.outcome": {key("signoz.audit.outcome", attr, str, true)},
"signoz.audit.principal.email": {key("signoz.audit.principal.email", attr, str, true)},
"signoz.audit.principal.id": {key("signoz.audit.principal.id", attr, str, true)},
"signoz.audit.principal.type": {key("signoz.audit.principal.type", attr, str, true)},
"signoz.audit.resource.kind": {key("signoz.audit.resource.kind", res, str, false)},
"signoz.audit.resource.id": {key("signoz.audit.resource.id", res, str, false)},
"signoz.audit.action_category": {key("signoz.audit.action_category", attr, str, false)},
"signoz.audit.error.type": {key("signoz.audit.error.type", attr, str, false)},
"signoz.audit.error.code": {key("signoz.audit.error.code", attr, str, false)},
"http.request.method": {key("http.request.method", attr, str, false)},
"http.response.status_code": {key("http.response.status_code", attr, i64, false)},
}
}
func newTestAuditStatementBuilder() *auditQueryStatementBuilder {
mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
mockMetadataStore.KeysMap = auditFieldKeyMap()
fm := NewFieldMapper()
cb := NewConditionBuilder(fm)
aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
return NewAuditQueryStatementBuilder(
instrumentationtest.New().ToProviderSettings(),
mockMetadataStore,
fm,
cb,
aggExprRewriter,
DefaultFullTextColumn,
nil,
)
}
func TestStatementBuilder(t *testing.T) {
statementBuilder := newTestAuditStatementBuilder()
ctx := context.Background()
testCases := []struct {
name string
requestType qbtypes.RequestType
query qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]
expected qbtypes.Statement
expectedErr error
}{
// List: all actions by a specific user (materialized principal.id filter)
{
name: "ListByPrincipalID",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
Filter: &qbtypes.Filter{
Expression: "signoz.audit.principal.id = '019a-1234-abcd-5678'",
},
Limit: 100,
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$principal$$id` = ? AND `attribute_string_signoz$$audit$$principal$$id_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "019a-1234-abcd-5678", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100},
},
},
// List: all failed actions (materialized outcome filter)
{
name: "ListByOutcomeFailure",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
Filter: &qbtypes.Filter{
Expression: "signoz.audit.outcome = 'failure'",
},
Limit: 100,
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "failure", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100},
},
},
// List: change history of a specific dashboard (two materialized column AND)
{
name: "ListByResourceKindAndID",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
Filter: &qbtypes.Filter{
Expression: "signoz.audit.resource.kind = 'dashboard' AND signoz.audit.resource.id = '019b-5678-efgh-9012'",
},
Limit: 100,
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE ((simpleJSONExtractString(labels, 'signoz.audit.resource.kind') = ? AND labels LIKE ? AND labels LIKE ?) AND (simpleJSONExtractString(labels, 'signoz.audit.resource.id') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"dashboard", "%signoz.audit.resource.kind%", "%signoz.audit.resource.kind\":\"dashboard%", "019b-5678-efgh-9012", "%signoz.audit.resource.id%", "%signoz.audit.resource.id\":\"019b-5678-efgh-9012%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100},
},
},
// List: all dashboard deletions (compliance — resource.kind + action AND)
{
name: "ListByResourceKindAndAction",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
Filter: &qbtypes.Filter{
Expression: "signoz.audit.resource.kind = 'dashboard' AND signoz.audit.action = 'delete'",
},
Limit: 100,
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE (simpleJSONExtractString(labels, 'signoz.audit.resource.kind') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$action` = ? AND `attribute_string_signoz$$audit$$action_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{"dashboard", "%signoz.audit.resource.kind%", "%signoz.audit.resource.kind\":\"dashboard%", uint64(1747945619), uint64(1747983448), "delete", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100},
},
},
// List: all actions by service accounts (materialized principal.type)
{
name: "ListByPrincipalType",
requestType: qbtypes.RequestTypeRaw,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
Filter: &qbtypes.Filter{
Expression: "signoz.audit.principal.type = 'service_account'",
},
Limit: 100,
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, event_name, attributes_string, attributes_number, attributes_bool, resource, scope_string FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$principal$$type` = ? AND `attribute_string_signoz$$audit$$principal$$type_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
Args: []any{uint64(1747945619), uint64(1747983448), "service_account", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 100},
},
},
// Scalar: alert — count forbidden errors (outcome + action AND)
{
name: "ScalarCountByOutcomeAndAction",
requestType: qbtypes.RequestTypeScalar,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Filter: &qbtypes.Filter{
Expression: "signoz.audit.outcome = 'failure' AND signoz.audit.action = 'update'",
},
Aggregations: []qbtypes.LogAggregation{
{Expression: "count()"},
},
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT count() AS __result_0 FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND (`attribute_string_signoz$$audit$$action` = ? AND `attribute_string_signoz$$audit$$action_exists` = ?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY __result_0 DESC",
Args: []any{uint64(1747945619), uint64(1747983448), "failure", true, "update", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
},
},
// TimeSeries: failures grouped by principal email with top-N limit
{
name: "TimeSeriesFailuresGroupedByPrincipal",
requestType: qbtypes.RequestTypeTimeSeries,
query: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
Signal: telemetrytypes.SignalLogs,
Source: telemetrytypes.SourceAudit,
StepInterval: qbtypes.Step{Duration: 60 * time.Second},
Aggregations: []qbtypes.LogAggregation{
{Expression: "count()"},
},
Filter: &qbtypes.Filter{
Expression: "signoz.audit.outcome = 'failure'",
},
GroupBy: []qbtypes.GroupByKey{
{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{Name: "signoz.audit.principal.email"}},
},
Limit: 5,
},
expected: qbtypes.Statement{
Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_audit.distributed_logs_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?), __limit_cte AS (SELECT toString(multiIf(`attribute_string_signoz$$audit$$principal$$email_exists` = ?, `attribute_string_signoz$$audit$$principal$$email`, NULL)) AS `signoz.audit.principal.email`, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? GROUP BY `signoz.audit.principal.email` ORDER BY __result_0 DESC LIMIT ?) SELECT toStartOfInterval(fromUnixTimestamp64Nano(timestamp), INTERVAL 60 SECOND) AS ts, toString(multiIf(`attribute_string_signoz$$audit$$principal$$email_exists` = ?, `attribute_string_signoz$$audit$$principal$$email`, NULL)) AS `signoz.audit.principal.email`, count() AS __result_0 FROM signoz_audit.distributed_logs WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (`attribute_string_signoz$$audit$$outcome` = ? AND `attribute_string_signoz$$audit$$outcome_exists` = ?) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? AND (`signoz.audit.principal.email`) GLOBAL IN (SELECT `signoz.audit.principal.email` FROM __limit_cte) GROUP BY ts, `signoz.audit.principal.email`",
Args: []any{uint64(1747945619), uint64(1747983448), true, "failure", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 5, true, "failure", true, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448)},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
q, err := statementBuilder.Build(ctx, 1747947419000, 1747983448000, testCase.requestType, testCase.query, nil)
if testCase.expectedErr != nil {
require.Error(t, err)
require.Contains(t, err.Error(), testCase.expectedErr.Error())
} else {
require.NoError(t, err)
require.Equal(t, testCase.expected.Query, q.Query)
require.Equal(t, testCase.expected.Args, q.Args)
}
})
}
}

View File

@@ -1,12 +0,0 @@
package telemetryaudit
const (
DBName = "signoz_audit"
AuditLogsTableName = "distributed_logs"
AuditLogsLocalTableName = "logs"
TagAttributesTableName = "distributed_tag_attributes"
TagAttributesLocalTableName = "tag_attributes"
LogAttributeKeysTblName = "distributed_logs_attribute_keys"
LogResourceKeysTblName = "distributed_logs_resource_keys"
LogsResourceTableName = "distributed_logs_resource"
)

View File

@@ -45,7 +45,6 @@ func NewLogQueryStatementBuilder(
DBName,
LogsResourceV2TableName,
telemetrytypes.SignalLogs,
telemetrytypes.SourceUnspecified,
metadataStore,
fullTextColumn,
jsonKeyToKey,

View File

@@ -13,7 +13,6 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/telemetrystore"
@@ -28,7 +27,6 @@ import (
var (
ErrFailedToGetTracesKeys = errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to get traces keys")
ErrFailedToGetLogsKeys = errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to get logs keys")
ErrFailedToGetAuditKeys = errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to get audit keys")
ErrFailedToGetTblStatement = errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to get tbl statement")
ErrFailedToGetMetricsKeys = errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to get metrics keys")
ErrFailedToGetMeterKeys = errors.Newf(errors.TypeInternal, errors.CodeInternal, "failed to get meter keys")
@@ -52,11 +50,6 @@ type telemetryMetaStore struct {
logAttributeKeysTblName string
logResourceKeysTblName string
logsV2TblName string
auditDBName string
auditLogsTblName string
auditFieldsTblName string
auditAttributeKeysTblName string
auditResourceKeysTblName string
relatedMetadataDBName string
relatedMetadataTblName string
columnEvolutionMetadataTblName string
@@ -86,11 +79,6 @@ func NewTelemetryMetaStore(
logsFieldsTblName string,
logAttributeKeysTblName string,
logResourceKeysTblName string,
auditDBName string,
auditLogsTblName string,
auditFieldsTblName string,
auditAttributeKeysTblName string,
auditResourceKeysTblName string,
relatedMetadataDBName string,
relatedMetadataTblName string,
columnEvolutionMetadataTblName string,
@@ -113,11 +101,6 @@ func NewTelemetryMetaStore(
logsFieldsTblName: logsFieldsTblName,
logAttributeKeysTblName: logAttributeKeysTblName,
logResourceKeysTblName: logResourceKeysTblName,
auditDBName: auditDBName,
auditLogsTblName: auditLogsTblName,
auditFieldsTblName: auditFieldsTblName,
auditAttributeKeysTblName: auditAttributeKeysTblName,
auditResourceKeysTblName: auditResourceKeysTblName,
relatedMetadataDBName: relatedMetadataDBName,
relatedMetadataTblName: relatedMetadataTblName,
columnEvolutionMetadataTblName: columnEvolutionMetadataTblName,
@@ -609,227 +592,6 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
return keys, complete, nil
}
func (t *telemetryMetaStore) auditTblStatementToFieldKeys(ctx context.Context) ([]*telemetrytypes.TelemetryFieldKey, error) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "auditTblStatementToFieldKeys",
})
query := fmt.Sprintf("SHOW CREATE TABLE %s.%s", t.auditDBName, t.auditLogsTblName)
statements := []telemetrytypes.ShowCreateTableStatement{}
err := t.telemetrystore.ClickhouseDB().Select(ctx, &statements, query)
if err != nil {
return nil, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetTblStatement.Error())
}
materialisedKeys, err := ExtractFieldKeysFromTblStatement(statements[0].Statement)
if err != nil {
return nil, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
for idx := range materialisedKeys {
materialisedKeys[idx].Signal = telemetrytypes.SignalLogs
}
return materialisedKeys, nil
}
func (t *telemetryMetaStore) getAuditKeys(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "getAuditKeys",
})
if len(fieldKeySelectors) == 0 {
return nil, true, nil
}
matKeys, err := t.auditTblStatementToFieldKeys(ctx)
if err != nil {
return nil, false, err
}
mapOfKeys := make(map[string]*telemetrytypes.TelemetryFieldKey)
for _, key := range matKeys {
mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key
}
var queries []string
var allArgs []any
queryAttributeTable := false
queryResourceTable := false
for _, selector := range fieldKeySelectors {
if selector.FieldContext == telemetrytypes.FieldContextUnspecified {
queryAttributeTable = true
queryResourceTable = true
break
} else if selector.FieldContext == telemetrytypes.FieldContextAttribute {
queryAttributeTable = true
} else if selector.FieldContext == telemetrytypes.FieldContextResource {
queryResourceTable = true
}
}
tablesToQuery := []struct {
fieldContext telemetrytypes.FieldContext
shouldQuery bool
tblName string
}{
{telemetrytypes.FieldContextAttribute, queryAttributeTable, t.auditDBName + "." + t.auditAttributeKeysTblName},
{telemetrytypes.FieldContextResource, queryResourceTable, t.auditDBName + "." + t.auditResourceKeysTblName},
}
for _, table := range tablesToQuery {
if !table.shouldQuery {
continue
}
fieldContext := table.fieldContext
tblName := table.tblName
sb := sqlbuilder.Select(
"name AS tag_key",
fmt.Sprintf("'%s' AS tag_type", fieldContext.TagType()),
"lower(datatype) AS tag_data_type",
fmt.Sprintf("%d AS priority", getPriorityForContext(fieldContext)),
).From(tblName)
var limit int
conds := []string{}
for _, fieldKeySelector := range fieldKeySelectors {
if fieldKeySelector.FieldContext != telemetrytypes.FieldContextUnspecified && fieldKeySelector.FieldContext != fieldContext {
continue
}
fieldKeyConds := []string{}
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
fieldKeyConds = append(fieldKeyConds, sb.E("name", fieldKeySelector.Name))
} else {
fieldKeyConds = append(fieldKeyConds, sb.ILike("name", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
}
if fieldKeySelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
fieldKeyConds = append(fieldKeyConds, sb.E("datatype", fieldKeySelector.FieldDataType.TagDataType()))
}
if len(fieldKeyConds) > 0 {
conds = append(conds, sb.And(fieldKeyConds...))
}
limit += fieldKeySelector.Limit
}
if len(conds) > 0 {
sb.Where(sb.Or(conds...))
}
sb.GroupBy("name", "datatype")
if limit == 0 {
limit = 1000
}
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
queries = append(queries, query)
allArgs = append(allArgs, args...)
}
if len(queries) == 0 {
return []*telemetrytypes.TelemetryFieldKey{}, true, nil
}
var limit int
for _, fieldKeySelector := range fieldKeySelectors {
limit += fieldKeySelector.Limit
}
if limit == 0 {
limit = 1000
}
mainQuery := fmt.Sprintf(`
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
FROM (
%s
) AS combined_results
GROUP BY tag_key, tag_type, tag_data_type
ORDER BY priority
LIMIT %d
`, strings.Join(queries, " UNION ALL "), limit+1)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
defer rows.Close()
keys := []*telemetrytypes.TelemetryFieldKey{}
rowCount := 0
searchTexts := []string{}
for _, fieldKeySelector := range fieldKeySelectors {
searchTexts = append(searchTexts, fieldKeySelector.Name)
}
for rows.Next() {
rowCount++
if rowCount > limit {
break
}
var name string
var fieldContext telemetrytypes.FieldContext
var fieldDataType telemetrytypes.FieldDataType
var priority uint8
err = rows.Scan(&name, &fieldContext, &fieldDataType, &priority)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
key, ok := mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()]
if !ok {
key = &telemetrytypes.TelemetryFieldKey{
Name: name,
Signal: telemetrytypes.SignalLogs,
FieldContext: fieldContext,
FieldDataType: fieldDataType,
}
}
keys = append(keys, key)
mapOfKeys[name+";"+fieldContext.StringValue()+";"+fieldDataType.StringValue()] = key
}
if rows.Err() != nil {
return nil, false, errors.Wrap(rows.Err(), errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
complete := rowCount <= limit
// Add intrinsic audit fields (same as logs intrinsics: body, severity_text, etc.)
staticKeys := maps.Keys(telemetryaudit.IntrinsicFields)
for _, key := range staticKeys {
found := false
for _, v := range searchTexts {
if v == "" || strings.Contains(key, v) {
found = true
break
}
}
if found {
if field, exists := telemetryaudit.IntrinsicFields[key]; exists {
if _, added := mapOfKeys[field.Name+";"+field.FieldContext.StringValue()+";"+field.FieldDataType.StringValue()]; !added {
keys = append(keys, &field)
}
}
}
}
return keys, complete, nil
}
func getPriorityForContext(ctx telemetrytypes.FieldContext) int {
switch ctx {
case telemetrytypes.FieldContextLog:
@@ -1127,11 +889,7 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele
case telemetrytypes.SignalTraces:
keys, complete, err = t.getTracesKeys(ctx, selectors)
case telemetrytypes.SignalLogs:
if fieldKeySelector.Source == telemetrytypes.SourceAudit {
keys, complete, err = t.getAuditKeys(ctx, selectors)
} else {
keys, complete, err = t.getLogsKeys(ctx, selectors)
}
keys, complete, err = t.getLogsKeys(ctx, selectors)
case telemetrytypes.SignalMetrics:
if fieldKeySelector.Source == telemetrytypes.SourceMeter {
keys, complete, err = t.getMeterSourceMetricKeys(ctx, selectors)
@@ -1180,7 +938,6 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele
func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors []*telemetrytypes.FieldKeySelector) (map[string][]*telemetrytypes.TelemetryFieldKey, bool, error) {
logsSelectors := []*telemetrytypes.FieldKeySelector{}
auditSelectors := []*telemetrytypes.FieldKeySelector{}
tracesSelectors := []*telemetrytypes.FieldKeySelector{}
metricsSelectors := []*telemetrytypes.FieldKeySelector{}
meterSourceMetricsSelectors := []*telemetrytypes.FieldKeySelector{}
@@ -1188,11 +945,7 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors
for _, fieldKeySelector := range fieldKeySelectors {
switch fieldKeySelector.Signal {
case telemetrytypes.SignalLogs:
if fieldKeySelector.Source == telemetrytypes.SourceAudit {
auditSelectors = append(auditSelectors, fieldKeySelector)
} else {
logsSelectors = append(logsSelectors, fieldKeySelector)
}
logsSelectors = append(logsSelectors, fieldKeySelector)
case telemetrytypes.SignalTraces:
tracesSelectors = append(tracesSelectors, fieldKeySelector)
case telemetrytypes.SignalMetrics:
@@ -1212,10 +965,6 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors
if err != nil {
return nil, false, err
}
auditKeys, auditComplete, err := t.getAuditKeys(ctx, auditSelectors)
if err != nil {
return nil, false, err
}
tracesKeys, tracesComplete, err := t.getTracesKeys(ctx, tracesSelectors)
if err != nil {
return nil, false, err
@@ -1230,15 +979,12 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors
return nil, false, err
}
// Complete only if all queries are complete
complete := logsComplete && auditComplete && tracesComplete && metricsComplete
complete := logsComplete && tracesComplete && metricsComplete
mapOfKeys := make(map[string][]*telemetrytypes.TelemetryFieldKey)
for _, key := range logsKeys {
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
for _, key := range auditKeys {
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
for _, key := range tracesKeys {
mapOfKeys[key.Name] = append(mapOfKeys[key.Name], key)
}
@@ -1592,97 +1338,6 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
return values, complete, nil
}
func (t *telemetryMetaStore) getAuditFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
instrumentationtypes.CodeNamespace: "metadata",
instrumentationtypes.CodeFunctionName: "getAuditFieldValues",
})
limit := fieldValueSelector.Limit
if limit == 0 {
limit = 50
}
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.auditDBName + "." + t.auditFieldsTblName)
if fieldValueSelector.Name != "" {
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
}
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
sb.Where(sb.E("tag_type", fieldValueSelector.FieldContext.TagType()))
}
if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
sb.Where(sb.E("tag_data_type", fieldValueSelector.FieldDataType.TagDataType()))
}
if fieldValueSelector.Value != "" {
switch fieldValueSelector.FieldDataType {
case telemetrytypes.FieldDataTypeString:
sb.Where(sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
case telemetrytypes.FieldDataTypeNumber:
sb.Where(sb.IsNotNull("number_value"))
sb.Where(sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"))
case telemetrytypes.FieldDataTypeUnspecified:
sb.Where(sb.Or(
sb.ILike("string_value", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
sb.ILike("toString(number_value)", "%"+escapeForLike(fieldValueSelector.Value)+"%"),
))
}
}
// fetch one extra row to detect whether the result set is complete
sb.Limit(limit + 1)
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
defer rows.Close()
values := &telemetrytypes.TelemetryFieldValues{}
seen := make(map[string]bool)
rowCount := 0
totalCount := 0
for rows.Next() {
rowCount++
var stringValue string
var numberValue float64
err = rows.Scan(&stringValue, &numberValue)
if err != nil {
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetAuditKeys.Error())
}
if stringValue != "" && !seen[stringValue] {
if totalCount >= limit {
break
}
values.StringValues = append(values.StringValues, stringValue)
seen[stringValue] = true
totalCount++
}
if numberValue != 0 {
if totalCount >= limit {
break
}
if !seen[fmt.Sprintf("%f", numberValue)] {
values.NumberValues = append(values.NumberValues, numberValue)
seen[fmt.Sprintf("%f", numberValue)] = true
totalCount++
}
}
}
complete := rowCount <= limit
return values, complete, nil
}
// getMetricFieldValues returns field values and whether the result is complete.
func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) {
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
@@ -1973,11 +1628,7 @@ func (t *telemetryMetaStore) GetAllValues(ctx context.Context, fieldValueSelecto
case telemetrytypes.SignalTraces:
values, complete, err = t.getSpanFieldValues(ctx, fieldValueSelector)
case telemetrytypes.SignalLogs:
if fieldValueSelector.Source == telemetrytypes.SourceAudit {
values, complete, err = t.getAuditFieldValues(ctx, fieldValueSelector)
} else {
values, complete, err = t.getLogFieldValues(ctx, fieldValueSelector)
}
values, complete, err = t.getLogFieldValues(ctx, fieldValueSelector)
case telemetrytypes.SignalMetrics:
if fieldValueSelector.Source == telemetrytypes.SourceMeter {
values, complete, err = t.getMeterSourceMetricFieldValues(ctx, fieldValueSelector)

View File

@@ -5,7 +5,6 @@ import (
"testing"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
@@ -38,11 +37,6 @@ func TestGetFirstSeenFromMetricMetadata(t *testing.T) {
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
DBName,
AttributesMetadataLocalTableName,
ColumnEvolutionMetadataTableName,

View File

@@ -7,7 +7,6 @@ import (
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/telemetryaudit"
"github.com/SigNoz/signoz/pkg/telemetrylogs"
"github.com/SigNoz/signoz/pkg/telemetrymeter"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
@@ -37,11 +36,6 @@ func newTestTelemetryMetaStoreTestHelper(store telemetrystore.TelemetryStore) te
telemetrylogs.TagAttributesV2TableName,
telemetrylogs.LogAttributeKeysTblName,
telemetrylogs.LogResourceKeysTblName,
telemetryaudit.DBName,
telemetryaudit.AuditLogsTableName,
telemetryaudit.TagAttributesTableName,
telemetryaudit.LogAttributeKeysTblName,
telemetryaudit.LogResourceKeysTblName,
DBName,
AttributesMetadataLocalTableName,
ColumnEvolutionMetadataTableName,

View File

@@ -9,6 +9,6 @@ const (
ColumnEvolutionMetadataTableName = "distributed_column_evolution_metadata"
PathTypesTableName = otelcollectorconst.DistributedPathTypesTable
// Column Evolution table stores promoted paths as (signal, column_name, field_context, field_name); see signoz-otel-collector metadata_migrations.
PromotedPathsTableName = "distributed_column_evolution_metadata"
SkipIndexTableName = "system.data_skipping_indices"
PromotedPathsTableName = "distributed_column_evolution_metadata"
SkipIndexTableName = "system.data_skipping_indices"
)

View File

@@ -21,7 +21,6 @@ type resourceFilterStatementBuilder[T any] struct {
conditionBuilder qbtypes.ConditionBuilder
metadataStore telemetrytypes.MetadataStore
signal telemetrytypes.Signal
source telemetrytypes.Source
fullTextColumn *telemetrytypes.TelemetryFieldKey
jsonKeyToKey qbtypes.JsonKeyToFieldFunc
@@ -38,7 +37,6 @@ func New[T any](
dbName string,
tableName string,
signal telemetrytypes.Signal,
source telemetrytypes.Source,
metadataStore telemetrytypes.MetadataStore,
fullTextColumn *telemetrytypes.TelemetryFieldKey,
jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
@@ -54,7 +52,6 @@ func New[T any](
conditionBuilder: cb,
metadataStore: metadataStore,
signal: signal,
source: source,
fullTextColumn: fullTextColumn,
jsonKeyToKey: jsonKeyToKey,
}
@@ -75,7 +72,6 @@ func (b *resourceFilterStatementBuilder[T]) getKeySelectors(query qbtypes.QueryB
continue
}
keySelectors[idx].Signal = b.signal
keySelectors[idx].Source = b.source
keySelectors[idx].SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact
filteredKeySelectors = append(filteredKeySelectors, keySelectors[idx])
}

View File

@@ -375,7 +375,6 @@ func TestResourceFilterStatementBuilder_Traces(t *testing.T) {
"signoz_traces",
"distributed_traces_v3_resource",
telemetrytypes.SignalTraces,
telemetrytypes.SourceUnspecified,
mockMetadataStore,
nil,
nil,
@@ -593,7 +592,6 @@ func TestResourceFilterStatementBuilder_Logs(t *testing.T) {
"signoz_logs",
"distributed_logs_v2_resource",
telemetrytypes.SignalLogs,
telemetrytypes.SourceUnspecified,
mockMetadataStore,
nil,
nil,
@@ -655,7 +653,6 @@ func TestResourceFilterStatementBuilder_Variables(t *testing.T) {
"signoz_traces",
"distributed_traces_v3_resource",
telemetrytypes.SignalTraces,
telemetrytypes.SourceUnspecified,
mockMetadataStore,
nil,
nil,

View File

@@ -49,7 +49,6 @@ func NewTraceQueryStatementBuilder(
DBName,
TracesResourceV3TableName,
telemetrytypes.SignalTraces,
telemetrytypes.SourceUnspecified,
metadataStore,
nil,
nil,

View File

@@ -39,7 +39,6 @@ func NewTraceOperatorStatementBuilder(
DBName,
TracesResourceV3TableName,
telemetrytypes.SignalTraces,
telemetrytypes.SourceUnspecified,
metadataStore,
nil,
nil,

View File

@@ -175,6 +175,26 @@ func NewAccountConfigFromPostable(provider CloudProviderType, config *PostableAc
return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
}
// func NewAccountFromPostableAccount(provider CloudProviderType, account *PostableAccount) (*Account, error) {
// req := &Account{
// Credentials: account.Credentials,
// }
// switch provider {
// case CloudProviderTypeAWS:
// req.Config = &ConnectionArtifactRequestConfig{
// Aws: &AWSConnectionArtifactRequest{
// DeploymentRegion: artifact.Config.Aws.DeploymentRegion,
// Regions: artifact.Config.Aws.Regions,
// },
// }
// return req, nil
// default:
// return nil, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider.StringValue())
// }
// }
func NewAgentReport(data map[string]any) *AgentReport {
return &AgentReport{
TimestampMillis: time.Now().UnixMilli(),

View File

@@ -7,13 +7,11 @@ type Source struct {
}
var (
SourceAudit = Source{valuer.NewString("audit")}
SourceMeter = Source{valuer.NewString("meter")}
SourceUnspecified = Source{valuer.NewString("")}
)
// Enum returns the acceptable values for Source.
// TODO: Add SourceAudit once the frontend is ready for consumption.
func (Source) Enum() []any {
return []any{
SourceMeter,

View File

@@ -1,10 +1,8 @@
package zeustypes
import (
"fmt"
"net/url"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/tidwall/gjson"
)
@@ -58,24 +56,3 @@ func NewGettableHost(data []byte) *GettableHost {
Hosts: hosts,
}
}
// GettableDeployment represents the parsed deployment info from zeus.GetDeployment.
type GettableDeployment struct {
Name string
SignozAPIUrl string
}
// NewGettableDeployment parses raw GetDeployment bytes into a GettableDeployment.
func NewGettableDeployment(data []byte) (*GettableDeployment, error) {
parsed := gjson.ParseBytes(data)
name := parsed.Get("name").String()
dns := parsed.Get("cluster.region.dns").String()
if name == "" || dns == "" {
return nil, errors.NewInternalf(errors.CodeInternal,
"deployment info response missing name or cluster region dns")
}
return &GettableDeployment{
Name: name,
SignozAPIUrl: fmt.Sprintf("https://%s.%s", name, dns),
}, nil
}

View File

@@ -12,7 +12,6 @@ pytest_plugins = [
"fixtures.sqlite",
"fixtures.zookeeper",
"fixtures.signoz",
"fixtures.audit",
"fixtures.logs",
"fixtures.traces",
"fixtures.metrics",

View File

@@ -1,404 +0,0 @@
import datetime
import json
from abc import ABC
from typing import Any, Callable, Generator, List, Optional
import numpy as np
import pytest
from ksuid import KsuidMs
from fixtures import types
from fixtures.fingerprint import LogsOrTracesFingerprint
class AuditResource(ABC):
labels: str
fingerprint: str
seen_at_ts_bucket_start: np.int64
def __init__(
self,
labels: dict[str, str],
fingerprint: str,
seen_at_ts_bucket_start: np.int64,
) -> None:
self.labels = json.dumps(labels, separators=(",", ":"))
self.fingerprint = fingerprint
self.seen_at_ts_bucket_start = seen_at_ts_bucket_start
def np_arr(self) -> np.array:
return np.array(
[
self.labels,
self.fingerprint,
self.seen_at_ts_bucket_start,
]
)
class AuditResourceOrAttributeKeys(ABC):
name: str
datatype: str
def __init__(self, name: str, datatype: str) -> None:
self.name = name
self.datatype = datatype
def np_arr(self) -> np.array:
return np.array([self.name, self.datatype])
class AuditTagAttributes(ABC):
unix_milli: np.int64
tag_key: str
tag_type: str
tag_data_type: str
string_value: str
int64_value: Optional[np.int64]
float64_value: Optional[np.float64]
def __init__(
self,
timestamp: datetime.datetime,
tag_key: str,
tag_type: str,
tag_data_type: str,
string_value: Optional[str],
int64_value: Optional[np.int64],
float64_value: Optional[np.float64],
) -> None:
self.unix_milli = np.int64(int(timestamp.timestamp() * 1e3))
self.tag_key = tag_key
self.tag_type = tag_type
self.tag_data_type = tag_data_type
self.string_value = string_value or ""
self.int64_value = int64_value
self.float64_value = float64_value
def np_arr(self) -> np.array:
return np.array(
[
self.unix_milli,
self.tag_key,
self.tag_type,
self.tag_data_type,
self.string_value,
self.int64_value,
self.float64_value,
]
)
class AuditLog(ABC):
"""Represents a single audit log event in signoz_audit.
Matches the ClickHouse DDL from the schema migration (ticket #1936):
- Database: signoz_audit
- Local table: logs
- Distributed table: distributed_logs
- No resources_string column (resource JSON only)
- Has event_name column
- 7 materialized columns auto-populated from attributes_string at INSERT time
"""
ts_bucket_start: np.uint64
resource_fingerprint: str
timestamp: np.uint64
observed_timestamp: np.uint64
id: str
trace_id: str
span_id: str
trace_flags: np.uint32
severity_text: str
severity_number: np.uint8
body: str
scope_name: str
scope_version: str
scope_string: dict[str, str]
attributes_string: dict[str, str]
attributes_number: dict[str, np.float64]
attributes_bool: dict[str, bool]
resource_json: dict[str, str]
event_name: str
resource: List[AuditResource]
tag_attributes: List[AuditTagAttributes]
resource_keys: List[AuditResourceOrAttributeKeys]
attribute_keys: List[AuditResourceOrAttributeKeys]
def __init__(
self,
timestamp: Optional[datetime.datetime] = None,
resources: dict[str, Any] = {},
attributes: dict[str, Any] = {},
body: str = "",
event_name: str = "",
severity_text: str = "INFO",
trace_id: str = "",
span_id: str = "",
trace_flags: np.uint32 = 0,
scope_name: str = "signoz.audit",
scope_version: str = "",
) -> None:
if timestamp is None:
timestamp = datetime.datetime.now()
self.tag_attributes = []
self.attribute_keys = []
self.resource_keys = []
self.timestamp = np.uint64(int(timestamp.timestamp() * 1e9))
self.observed_timestamp = self.timestamp
minute = timestamp.minute
bucket_minute = 0 if minute < 30 else 30
bucket_start = timestamp.replace(minute=bucket_minute, second=0, microsecond=0)
self.ts_bucket_start = np.uint64(int(bucket_start.timestamp()))
self.id = str(KsuidMs(datetime=timestamp))
self.trace_id = trace_id
self.span_id = span_id
self.trace_flags = trace_flags
self.severity_text = severity_text
self.severity_number = np.uint8(9 if severity_text == "INFO" else 17)
self.body = body
self.event_name = event_name
# Resources — JSON column only (no resources_string in audit DDL)
self.resource_json = {k: str(v) for k, v in resources.items()}
for k, v in self.resource_json.items():
self.tag_attributes.append(
AuditTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="resource",
tag_data_type="string",
string_value=str(v),
int64_value=None,
float64_value=None,
)
)
self.resource_keys.append(
AuditResourceOrAttributeKeys(name=k, datatype="string")
)
self.resource_fingerprint = LogsOrTracesFingerprint(
self.resource_json
).calculate()
# Process attributes by type
self.attributes_string = {}
self.attributes_number = {}
self.attributes_bool = {}
for k, v in attributes.items():
if isinstance(v, bool):
self.attributes_bool[k] = v
self.tag_attributes.append(
AuditTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="tag",
tag_data_type="bool",
string_value=None,
int64_value=None,
float64_value=None,
)
)
self.attribute_keys.append(
AuditResourceOrAttributeKeys(name=k, datatype="bool")
)
elif isinstance(v, int):
self.attributes_number[k] = np.float64(v)
self.tag_attributes.append(
AuditTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="tag",
tag_data_type="int64",
string_value=None,
int64_value=np.int64(v),
float64_value=None,
)
)
self.attribute_keys.append(
AuditResourceOrAttributeKeys(name=k, datatype="int64")
)
elif isinstance(v, float):
self.attributes_number[k] = np.float64(v)
self.tag_attributes.append(
AuditTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="tag",
tag_data_type="float64",
string_value=None,
int64_value=None,
float64_value=np.float64(v),
)
)
self.attribute_keys.append(
AuditResourceOrAttributeKeys(name=k, datatype="float64")
)
else:
self.attributes_string[k] = str(v)
self.tag_attributes.append(
AuditTagAttributes(
timestamp=timestamp,
tag_key=k,
tag_type="tag",
tag_data_type="string",
string_value=str(v),
int64_value=None,
float64_value=None,
)
)
self.attribute_keys.append(
AuditResourceOrAttributeKeys(name=k, datatype="string")
)
self.scope_name = scope_name
self.scope_version = scope_version
self.scope_string = {}
self.resource = [
AuditResource(
labels=self.resource_json,
fingerprint=self.resource_fingerprint,
seen_at_ts_bucket_start=self.ts_bucket_start,
)
]
def np_arr(self) -> np.array:
return np.array(
[
self.ts_bucket_start,
self.resource_fingerprint,
self.timestamp,
self.observed_timestamp,
self.id,
self.trace_id,
self.span_id,
self.trace_flags,
self.severity_text,
self.severity_number,
self.body,
self.scope_name,
self.scope_version,
self.scope_string,
self.attributes_string,
self.attributes_number,
self.attributes_bool,
self.resource_json,
self.event_name,
]
)
@pytest.fixture(name="insert_audit_logs", scope="function")
def insert_audit_logs(
clickhouse: types.TestContainerClickhouse,
) -> Generator[Callable[[List[AuditLog]], None], Any, None]:
def _insert_audit_logs(logs: List[AuditLog]) -> None:
resources: List[AuditResource] = []
for log in logs:
resources.extend(log.resource)
if len(resources) > 0:
clickhouse.conn.insert(
database="signoz_audit",
table="distributed_logs_resource",
data=[resource.np_arr() for resource in resources],
column_names=[
"labels",
"fingerprint",
"seen_at_ts_bucket_start",
],
)
tag_attributes: List[AuditTagAttributes] = []
for log in logs:
tag_attributes.extend(log.tag_attributes)
if len(tag_attributes) > 0:
clickhouse.conn.insert(
database="signoz_audit",
table="distributed_tag_attributes",
data=[ta.np_arr() for ta in tag_attributes],
column_names=[
"unix_milli",
"tag_key",
"tag_type",
"tag_data_type",
"string_value",
"int64_value",
"float64_value",
],
)
attribute_keys: List[AuditResourceOrAttributeKeys] = []
for log in logs:
attribute_keys.extend(log.attribute_keys)
if len(attribute_keys) > 0:
clickhouse.conn.insert(
database="signoz_audit",
table="distributed_logs_attribute_keys",
data=[ak.np_arr() for ak in attribute_keys],
column_names=["name", "datatype"],
)
resource_keys: List[AuditResourceOrAttributeKeys] = []
for log in logs:
resource_keys.extend(log.resource_keys)
if len(resource_keys) > 0:
clickhouse.conn.insert(
database="signoz_audit",
table="distributed_logs_resource_keys",
data=[rk.np_arr() for rk in resource_keys],
column_names=["name", "datatype"],
)
clickhouse.conn.insert(
database="signoz_audit",
table="distributed_logs",
data=[log.np_arr() for log in logs],
column_names=[
"ts_bucket_start",
"resource_fingerprint",
"timestamp",
"observed_timestamp",
"id",
"trace_id",
"span_id",
"trace_flags",
"severity_text",
"severity_number",
"body",
"scope_name",
"scope_version",
"scope_string",
"attributes_string",
"attributes_number",
"attributes_bool",
"resource",
"event_name",
],
)
yield _insert_audit_logs
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
for table in [
"logs",
"logs_resource",
"tag_attributes",
"logs_attribute_keys",
"logs_resource_keys",
]:
clickhouse.conn.query(
f"TRUNCATE TABLE signoz_audit.{table} ON CLUSTER '{cluster}' SYNC"
)

View File

@@ -14,7 +14,7 @@ logger = setup_logger(__name__)
@pytest.fixture(scope="function")
def deprecated_create_cloud_integration_account(
def create_cloud_integration_account(
request: pytest.FixtureRequest,
signoz: types.SigNoz,
) -> Callable[[str, str], dict]:
@@ -78,78 +78,3 @@ def deprecated_create_cloud_integration_account(
logger.info("Cleaned up test account: %s", account_id)
except Exception as exc: # pylint: disable=broad-except
logger.info("Post-test disconnect cleanup failed: %s", exc)
@pytest.fixture(scope="function")
def create_cloud_integration_account(
request: pytest.FixtureRequest,
signoz: types.SigNoz,
) -> Callable[[str, str], dict]:
created_accounts: list[tuple[str, str]] = []
def _create(
admin_token: str,
cloud_provider: str = "aws",
deployment_region: str = "us-east-1",
regions: list[str] | None = None,
) -> dict:
if regions is None:
regions = ["us-east-1"]
endpoint = f"/api/v1/cloud_integrations/{cloud_provider}/accounts"
request_payload = {
"config": {
cloud_provider: {
"deploymentRegion": deployment_region,
"regions": regions,
}
},
"credentials": {
"sigNozApiURL": "https://test-deployment.test.signoz.cloud",
"sigNozApiKey": "test-api-key-789",
"ingestionUrl": "https://ingest.test.signoz.cloud",
"ingestionKey": "test-ingestion-key-123456",
},
}
response = requests.post(
signoz.self.host_configs["8080"].get(endpoint),
headers={"Authorization": f"Bearer {admin_token}"},
json=request_payload,
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Failed to create test account: {response.status_code}: {response.text}"
data = response.json()["data"]
created_accounts.append((data["id"], cloud_provider))
return data
yield _create
if created_accounts:
get_token = request.getfixturevalue("get_token")
try:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
for account_id, cloud_provider in created_accounts:
delete_endpoint = (
f"/api/v1/cloud_integrations/{cloud_provider}/accounts/{account_id}"
)
r = requests.delete(
signoz.self.host_configs["8080"].get(delete_endpoint),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
if r.status_code != HTTPStatus.NO_CONTENT:
logger.info(
"Delete cleanup returned %s for account %s",
r.status_code,
account_id,
)
logger.info("Cleaned up test account: %s", account_id)
except Exception as exc: # pylint: disable=broad-except
logger.info("Post-test delete cleanup failed: %s", exc)

View File

@@ -1,15 +1,6 @@
"""Fixtures for cloud integration tests."""
from typing import Callable
import requests
from wiremock.client import (
HttpMethods,
Mapping,
MappingRequest,
MappingResponse,
WireMockMatchers,
)
from fixtures import types
from fixtures.logger import setup_logger
@@ -17,7 +8,7 @@ from fixtures.logger import setup_logger
logger = setup_logger(__name__)
def deprecated_simulate_agent_checkin(
def simulate_agent_checkin(
signoz: types.SigNoz,
admin_token: str,
cloud_provider: str,
@@ -47,108 +38,3 @@ def deprecated_simulate_agent_checkin(
)
return response
def setup_create_account_mocks(
signoz: types.SigNoz,
make_http_mocks: Callable,
) -> None:
"""Set up Zeus and Gateway mocks required by the CreateAccount endpoint."""
make_http_mocks(
signoz.zeus,
[
Mapping(
request=MappingRequest(
method=HttpMethods.GET,
url="/v2/deployments/me",
headers={
"X-Signoz-Cloud-Api-Key": {
WireMockMatchers.EQUAL_TO: "secret-key"
}
},
),
response=MappingResponse(
status=200,
json_body={
"status": "success",
"data": {
"name": "test-deployment",
"cluster": {"region": {"dns": "test.signoz.cloud"}},
},
},
),
persistent=False,
)
],
)
make_http_mocks(
signoz.gateway,
[
Mapping(
request=MappingRequest(
method=HttpMethods.GET,
url="/v1/workspaces/me/keys/search?name=aws-integration&page=1&per_page=10",
),
response=MappingResponse(
status=200,
json_body={
"status": "success",
"data": [],
"_pagination": {"page": 1, "per_page": 10, "total": 0},
},
),
persistent=False,
),
Mapping(
request=MappingRequest(
method=HttpMethods.POST,
url="/v1/workspaces/me/keys",
),
response=MappingResponse(
status=200,
json_body={
"status": "success",
"data": {
"name": "aws-integration",
"value": "test-ingestion-key-123456",
},
"error": "",
},
),
persistent=False,
),
],
)
def simulate_agent_checkin(
signoz: types.SigNoz,
admin_token: str,
cloud_provider: str,
account_id: str,
cloud_account_id: str,
data: dict | None = None,
) -> requests.Response:
endpoint = f"/api/v1/cloud_integrations/{cloud_provider}/accounts/check_in"
checkin_payload = {
"cloudIntegrationId": account_id,
"providerAccountId": cloud_account_id,
"data": data or {},
}
response = requests.post(
signoz.self.host_configs["8080"].get(endpoint),
headers={"Authorization": f"Bearer {admin_token}"},
json=checkin_payload,
timeout=10,
)
if not response.ok:
logger.error(
"Agent check-in failed: %s, response: %s",
response.status_code,
response.text,
)
return response

View File

@@ -38,7 +38,6 @@ class OrderBy:
class BuilderQuery:
signal: str
name: str = "A"
source: Optional[str] = None
limit: Optional[int] = None
filter_expression: Optional[str] = None
select_fields: Optional[List[TelemetryFieldKey]] = None
@@ -49,8 +48,6 @@ class BuilderQuery:
"signal": self.signal,
"name": self.name,
}
if self.source:
spec["source"] = self.source
if self.limit is not None:
spec["limit"] = self.limit
if self.filter_expression:
@@ -58,9 +55,7 @@ class BuilderQuery:
if self.select_fields:
spec["selectFields"] = [f.to_dict() for f in self.select_fields]
if self.order:
spec["order"] = [
o.to_dict() if hasattr(o, "to_dict") else o for o in self.order
]
spec["order"] = [o.to_dict() for o in self.order]
return {"type": "builder_query", "spec": spec}
@@ -81,9 +76,7 @@ class TraceOperatorQuery:
if self.limit is not None:
spec["limit"] = self.limit
if self.order:
spec["order"] = [
o.to_dict() if hasattr(o, "to_dict") else o for o in self.order
]
spec["order"] = [o.to_dict() for o in self.order]
return {"type": "builder_trace_operator", "spec": spec}
@@ -449,7 +442,6 @@ def build_scalar_query(
signal: str,
aggregations: List[Dict],
*,
source: Optional[str] = None,
group_by: Optional[List[Dict]] = None,
order: Optional[List[Dict]] = None,
limit: Optional[int] = None,
@@ -466,9 +458,6 @@ def build_scalar_query(
"aggregations": aggregations,
}
if source:
spec["source"] = source
if group_by:
spec["groupBy"] = group_by

View File

@@ -76,7 +76,6 @@ def create_signoz(
"SIGNOZ_ALERTMANAGER_SIGNOZ_POLL__INTERVAL": "5s",
"SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__WAIT": "1s",
"SIGNOZ_ALERTMANAGER_SIGNOZ_ROUTE_GROUP__INTERVAL": "5s",
"SIGNOZ_CLOUDINTEGRATION_AGENT_VERSION": "v0.0.8",
}
| sqlstore.env
| clickhouse.env

View File

@@ -1,441 +0,0 @@
from datetime import datetime, timedelta, timezone
from http import HTTPStatus
from typing import Callable, List
import pytest
from fixtures import types
from fixtures.audit import AuditLog
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.querier import (
BuilderQuery,
build_logs_aggregation,
build_order_by,
build_scalar_query,
make_query_request,
)
def test_audit_list_all(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_audit_logs: Callable[[List[AuditLog]], None],
) -> None:
"""List audit events across multiple resource types — verify count, ordering, and fields."""
now = datetime.now(tz=timezone.utc)
insert_audit_logs(
[
AuditLog(
timestamp=now - timedelta(seconds=3),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "alert-rule",
"signoz.audit.resource.id": "alert-001",
},
attributes={
"signoz.audit.principal.id": "user-010",
"signoz.audit.principal.email": "ops@acme.com",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "create",
"signoz.audit.outcome": "success",
},
body="ops@acme.com (user-010) created alert-rule (alert-001)",
event_name="alert-rule.created",
severity_text="INFO",
),
AuditLog(
timestamp=now - timedelta(seconds=2),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "saved-view",
"signoz.audit.resource.id": "view-001",
},
attributes={
"signoz.audit.principal.id": "user-010",
"signoz.audit.principal.email": "ops@acme.com",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "update",
"signoz.audit.outcome": "success",
},
body="ops@acme.com (user-010) updated saved-view (view-001)",
event_name="saved-view.updated",
severity_text="INFO",
),
AuditLog(
timestamp=now - timedelta(seconds=1),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "user",
"signoz.audit.resource.id": "user-020",
},
attributes={
"signoz.audit.principal.id": "user-010",
"signoz.audit.principal.email": "ops@acme.com",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "update",
"signoz.audit.action_category": "access_control",
"signoz.audit.outcome": "success",
},
body="ops@acme.com (user-010) updated user (user-020)",
event_name="user.role.changed",
severity_text="INFO",
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=timezone.utc)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
queries=[
BuilderQuery(
signal="logs",
source="audit",
limit=100,
order=[build_order_by("timestamp"), build_order_by("id")],
).to_dict()
],
request_type="raw",
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
rows = response.json()["data"]["data"]["results"][0]["rows"]
assert len(rows) == 3
# Most recent first
assert rows[0]["data"]["event_name"] == "user.role.changed"
assert rows[1]["data"]["event_name"] == "saved-view.updated"
assert rows[2]["data"]["event_name"] == "alert-rule.created"
# Verify event_name and body are present
assert rows[0]["data"]["body"] == "ops@acme.com (user-010) updated user (user-020)"
assert rows[0]["data"]["severity_text"] == "INFO"
@pytest.mark.parametrize(
"filter_expression,expected_count,expected_event_names",
[
pytest.param(
"signoz.audit.principal.id = 'user-001'",
3,
{"session.login", "dashboard.updated", "dashboard.created"},
id="filter_by_principal_id",
),
pytest.param(
"signoz.audit.outcome = 'failure'",
1,
{"dashboard.deleted"},
id="filter_by_outcome_failure",
),
pytest.param(
"signoz.audit.resource.kind = 'dashboard'"
" AND signoz.audit.resource.id = 'dash-001'",
3,
{"dashboard.deleted", "dashboard.updated", "dashboard.created"},
id="filter_by_resource_kind_and_id",
),
pytest.param(
"signoz.audit.principal.type = 'service_account'",
1,
{"serviceaccount.apikey.created"},
id="filter_by_principal_type",
),
pytest.param(
"signoz.audit.resource.kind = 'dashboard'"
" AND signoz.audit.action = 'delete'",
1,
{"dashboard.deleted"},
id="filter_by_resource_kind_and_action",
),
],
)
def test_audit_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_audit_logs: Callable[[List[AuditLog]], None],
filter_expression: str,
expected_count: int,
expected_event_names: set,
) -> None:
"""Parametrized audit filter tests covering the documented query patterns."""
now = datetime.now(tz=timezone.utc)
insert_audit_logs(
[
AuditLog(
timestamp=now - timedelta(seconds=5),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "dashboard",
"signoz.audit.resource.id": "dash-001",
},
attributes={
"signoz.audit.principal.id": "user-001",
"signoz.audit.principal.email": "alice@acme.com",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "create",
"signoz.audit.action_category": "configuration_change",
"signoz.audit.outcome": "success",
},
body="alice@acme.com created dashboard",
event_name="dashboard.created",
),
AuditLog(
timestamp=now - timedelta(seconds=4),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "dashboard",
"signoz.audit.resource.id": "dash-001",
},
attributes={
"signoz.audit.principal.id": "user-001",
"signoz.audit.principal.email": "alice@acme.com",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "update",
"signoz.audit.action_category": "configuration_change",
"signoz.audit.outcome": "success",
},
body="alice@acme.com updated dashboard",
event_name="dashboard.updated",
),
AuditLog(
timestamp=now - timedelta(seconds=3),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "dashboard",
"signoz.audit.resource.id": "dash-001",
},
attributes={
"signoz.audit.principal.id": "user-002",
"signoz.audit.principal.email": "viewer@acme.com",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "delete",
"signoz.audit.action_category": "configuration_change",
"signoz.audit.outcome": "failure",
"signoz.audit.error.type": "forbidden",
"signoz.audit.error.code": "authz_forbidden",
},
body="viewer@acme.com failed to delete dashboard",
event_name="dashboard.deleted",
severity_text="ERROR",
),
AuditLog(
timestamp=now - timedelta(seconds=2),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "serviceaccount",
"signoz.audit.resource.id": "sa-001",
},
attributes={
"signoz.audit.principal.id": "sa-001",
"signoz.audit.principal.email": "",
"signoz.audit.principal.type": "service_account",
"signoz.audit.action": "create",
"signoz.audit.action_category": "access_control",
"signoz.audit.outcome": "success",
},
body="sa-001 created serviceaccount",
event_name="serviceaccount.apikey.created",
),
AuditLog(
timestamp=now - timedelta(seconds=1),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "session",
"signoz.audit.resource.id": "*",
},
attributes={
"signoz.audit.principal.id": "user-001",
"signoz.audit.principal.email": "alice@acme.com",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "login",
"signoz.audit.action_category": "access_control",
"signoz.audit.outcome": "success",
},
body="alice@acme.com login session",
event_name="session.login",
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=timezone.utc)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
queries=[
BuilderQuery(
signal="logs",
source="audit",
limit=100,
filter_expression=filter_expression,
order=[build_order_by("timestamp"), build_order_by("id")],
).to_dict()
],
request_type="raw",
)
assert response.status_code == HTTPStatus.OK
rows = response.json()["data"]["data"]["results"][0]["rows"]
assert len(rows) == expected_count
actual_event_names = {row["data"]["event_name"] for row in rows}
assert actual_event_names == expected_event_names
def test_audit_scalar_count_failures(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_audit_logs: Callable[[List[AuditLog]], None],
) -> None:
"""Alert query — count multiple failures from different principals."""
now = datetime.now(tz=timezone.utc)
insert_audit_logs(
[
AuditLog(
timestamp=now - timedelta(seconds=3),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "dashboard",
"signoz.audit.resource.id": "dash-100",
},
attributes={
"signoz.audit.principal.id": "user-050",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "delete",
"signoz.audit.outcome": "failure",
},
body="user-050 failed to delete dashboard",
event_name="dashboard.deleted",
severity_text="ERROR",
),
AuditLog(
timestamp=now - timedelta(seconds=2),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "alert-rule",
"signoz.audit.resource.id": "alert-200",
},
attributes={
"signoz.audit.principal.id": "user-060",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "update",
"signoz.audit.outcome": "failure",
},
body="user-060 failed to update alert-rule",
event_name="alert-rule.updated",
severity_text="ERROR",
),
AuditLog(
timestamp=now - timedelta(seconds=1),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "dashboard",
"signoz.audit.resource.id": "dash-100",
},
attributes={
"signoz.audit.principal.id": "user-050",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "update",
"signoz.audit.outcome": "success",
},
body="user-050 updated dashboard",
event_name="dashboard.updated",
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=timezone.utc)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
queries=[
build_scalar_query(
name="A",
signal="logs",
source="audit",
aggregations=[build_logs_aggregation("count()")],
filter_expression="signoz.audit.outcome = 'failure'",
)
],
request_type="scalar",
)
assert response.status_code == HTTPStatus.OK
assert response.json()["status"] == "success"
scalar_data = response.json()["data"]["data"]["results"][0].get("data", [])
assert len(scalar_data) == 1
assert scalar_data[0][0] == 2
def test_audit_does_not_leak_into_logs(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
insert_audit_logs: Callable[[List[AuditLog]], None],
) -> None:
"""A single audit event in signoz_audit must not appear in regular log queries."""
now = datetime.now(tz=timezone.utc)
insert_audit_logs(
[
AuditLog(
timestamp=now - timedelta(seconds=1),
resources={
"service.name": "signoz",
"signoz.audit.resource.kind": "organization",
"signoz.audit.resource.id": "org-999",
},
attributes={
"signoz.audit.principal.id": "user-admin",
"signoz.audit.principal.type": "user",
"signoz.audit.action": "update",
"signoz.audit.outcome": "success",
},
body="user-admin updated organization (org-999)",
event_name="organization.updated",
),
]
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
now = datetime.now(tz=timezone.utc)
response = make_query_request(
signoz,
token,
start_ms=int((now - timedelta(seconds=30)).timestamp() * 1000),
end_ms=int(now.timestamp() * 1000),
queries=[
BuilderQuery(
signal="logs",
limit=100,
order=[build_order_by("timestamp"), build_order_by("id")],
).to_dict()
],
request_type="raw",
)
assert response.status_code == HTTPStatus.OK
rows = response.json()["data"]["data"]["results"][0].get("rows") or []
audit_bodies = [
row["data"]["body"]
for row in rows
if "signoz.audit"
in row["data"].get("attributes_string", {}).get("signoz.audit.action", "")
]
assert len(audit_bodies) == 0

View File

@@ -6,7 +6,7 @@ import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.cloudintegrationsutils import deprecated_simulate_agent_checkin
from fixtures.cloudintegrationsutils import simulate_agent_checkin
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
@@ -150,7 +150,7 @@ def test_duplicate_cloud_account_checkins(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test that two accounts cannot check in with the same cloud_account_id."""
@@ -159,16 +159,16 @@ def test_duplicate_cloud_account_checkins(
same_cloud_account_id = str(uuid.uuid4())
# Create two separate cloud integration accounts via generate-connection-url
account1 = deprecated_create_cloud_integration_account(admin_token, cloud_provider)
account1 = create_cloud_integration_account(admin_token, cloud_provider)
account1_id = account1["account_id"]
account2 = deprecated_create_cloud_integration_account(admin_token, cloud_provider)
account2 = create_cloud_integration_account(admin_token, cloud_provider)
account2_id = account2["account_id"]
assert account1_id != account2_id, "Two accounts should have different internal IDs"
# First check-in succeeds: account1 claims cloud_account_id
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account1_id, same_cloud_account_id
)
assert (
@@ -176,7 +176,7 @@ def test_duplicate_cloud_account_checkins(
), f"Expected 200 for first check-in, got {response.status_code}: {response.text}"
#
# Second check-in should fail: account2 tries to use the same cloud_account_id
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account2_id, same_cloud_account_id
)

View File

@@ -6,7 +6,7 @@ import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.cloudintegrationsutils import deprecated_simulate_agent_checkin
from fixtures.cloudintegrationsutils import simulate_agent_checkin
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
@@ -45,21 +45,19 @@ def test_list_connected_accounts_with_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test listing connected accounts after creating one."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
# Simulate agent check-in to mark as connected
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (
@@ -95,15 +93,13 @@ def test_get_account_status(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test getting the status of a specific account."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account (no check-in needed for status check)
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
# Get account status
@@ -156,21 +152,19 @@ def test_update_account_config(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test updating account configuration."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
# Simulate agent check-in to mark as connected
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (
@@ -226,21 +220,19 @@ def test_disconnect_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test disconnecting an account."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
# Simulate agent check-in to mark as connected
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (

View File

@@ -6,7 +6,7 @@ import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.cloudintegrationsutils import deprecated_simulate_agent_checkin
from fixtures.cloudintegrationsutils import simulate_agent_checkin
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
@@ -50,20 +50,18 @@ def test_list_services_with_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test listing services for a specific connected account."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account and do check-in
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (
@@ -146,20 +144,18 @@ def test_get_service_details_with_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test getting service details for a specific connected account."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account and do check-in
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (
@@ -252,20 +248,18 @@ def test_update_service_config(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test updating service configuration for a connected account."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account and do check-in
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (
@@ -369,20 +363,18 @@ def test_update_service_config_invalid_service(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test updating config for a non-existent service should fail."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account and do check-in
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (
@@ -418,20 +410,18 @@ def test_update_service_config_disable_service(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
deprecated_create_cloud_integration_account: Callable,
create_cloud_integration_account: Callable,
) -> None:
"""Test disabling a service by updating config with enabled=false."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Create a test account and do check-in
cloud_provider = "aws"
account_data = deprecated_create_cloud_integration_account(
admin_token, cloud_provider
)
account_data = create_cloud_integration_account(admin_token, cloud_provider)
account_id = account_data["account_id"]
cloud_account_id = str(uuid.uuid4())
response = deprecated_simulate_agent_checkin(
response = simulate_agent_checkin(
signoz, admin_token, cloud_provider, account_id, cloud_account_id
)
assert (

View File

@@ -1,164 +0,0 @@
from http import HTTPStatus
from typing import Callable
import requests
from wiremock.client import (
HttpMethods,
Mapping,
MappingRequest,
MappingResponse,
)
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.cloudintegrationsutils import setup_create_account_mocks
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
CLOUD_PROVIDER = "aws"
CREDENTIALS_ENDPOINT = f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/credentials"
def test_apply_license(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
make_http_mocks: Callable[[types.TestContainerDocker, list], None],
get_token: Callable[[str, str], str],
) -> None:
"""Apply a license so that subsequent cloud integration calls succeed."""
add_license(signoz, make_http_mocks, get_token)
def test_get_credentials_success(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
make_http_mocks: Callable[[types.TestContainerDocker, list], None],
get_token: Callable[[str, str], str],
) -> None:
"""Happy path: all four credential fields are returned when Zeus and Gateway respond."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
setup_create_account_mocks(signoz, make_http_mocks)
response = requests.get(
signoz.self.host_configs["8080"].get(CREDENTIALS_ENDPOINT),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}: {response.text}"
data = response.json()["data"]
for field in ("sigNozApiUrl", "sigNozApiKey", "ingestionUrl", "ingestionKey"):
assert field in data, f"Response should contain '{field}'"
assert isinstance(data[field], str), f"'{field}' should be a string"
assert (
len(data[field]) > 0
), f"'{field}' should be non-empty when mocks are set up"
def test_get_credentials_partial_when_zeus_unavailable(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
make_http_mocks: Callable[[types.TestContainerDocker, list], None],
get_token: Callable[[str, str], str],
) -> None:
"""When Zeus is unavailable, server still returns 200 with partial credentials.
The server silently ignores errors from individual credential lookups and returns
whatever it could resolve. The frontend is responsible for prompting the user to
fill in any empty fields.
"""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# Reset Zeus mappings so no prior mock bleeds into this test,
# ensuring sigNozApiUrl cannot be resolved and will be returned as empty.
requests.post(
signoz.zeus.host_configs["8080"].get("/__admin/reset"),
timeout=10,
)
# Only set up Gateway mocks — Zeus has no mapping, so sigNozApiUrl will be empty
make_http_mocks(
signoz.gateway,
[
Mapping(
request=MappingRequest(
method=HttpMethods.GET,
url="/v1/workspaces/me/keys/search?name=aws-integration&page=1&per_page=10",
),
response=MappingResponse(
status=200,
json_body={
"status": "success",
"data": [],
"_pagination": {"page": 1, "per_page": 10, "total": 0},
},
),
persistent=False,
),
Mapping(
request=MappingRequest(
method=HttpMethods.POST,
url="/v1/workspaces/me/keys",
),
response=MappingResponse(
status=200,
json_body={
"status": "success",
"data": {
"name": "aws-integration",
"value": "test-ingestion-key-123456",
},
"error": "",
},
),
persistent=False,
),
],
)
response = requests.get(
signoz.self.host_configs["8080"].get(CREDENTIALS_ENDPOINT),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200 even without Zeus, got {response.status_code}: {response.text}"
data = response.json()["data"]
for field in ("sigNozApiUrl", "sigNozApiKey", "ingestionUrl", "ingestionKey"):
assert field in data, f"Response should always contain '{field}' key"
assert isinstance(data[field], str), f"'{field}' should be a string"
# sigNozApiUrl comes from Zeus, which is unavailable, so it should be empty
assert (
data["sigNozApiUrl"] == ""
), "sigNozApiUrl should be empty when Zeus is unavailable"
def test_get_credentials_unsupported_provider(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""Unsupported cloud provider returns 400."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(
"/api/v1/cloud_integrations/gcp/credentials"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.BAD_REQUEST
), f"Expected 400 for unsupported provider, got {response.status_code}"
assert "error" in response.json(), "Response should contain 'error' field"

View File

@@ -1,92 +0,0 @@
from http import HTTPStatus
from typing import Callable
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
def test_apply_license(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
make_http_mocks: Callable[[types.TestContainerDocker, list], None],
get_token: Callable[[str, str], str],
) -> None:
"""Apply a license so that subsequent cloud integration calls succeed."""
add_license(signoz, make_http_mocks, get_token)
def test_create_account(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Test creating a new cloud integration account for AWS."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
cloud_provider = "aws"
data = create_cloud_integration_account(
admin_token,
cloud_provider,
deployment_region="us-east-1",
regions=["us-east-1", "us-west-2"],
)
assert "id" in data, "Response data should contain 'id' field"
assert len(data["id"]) > 0, "id should be a non-empty UUID string"
assert (
"connectionArtifact" in data
), "Response data should contain 'connectionArtifact' field"
artifact = data["connectionArtifact"]
assert "aws" in artifact, "connectionArtifact should contain 'aws' field"
assert (
"connectionUrl" in artifact["aws"]
), "connectionArtifact.aws should contain 'connectionUrl'"
connection_url = artifact["aws"]["connectionUrl"]
assert (
"console.aws.amazon.com/cloudformation" in connection_url
), "connectionUrl should be an AWS CloudFormation URL"
assert (
"region=us-east-1" in connection_url
), "connectionUrl should contain the deployment region"
def test_create_account_unsupported_provider(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""Test that creating an account with an unsupported cloud provider returns 400."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
cloud_provider = "gcp"
endpoint = f"/api/v1/cloud_integrations/{cloud_provider}/accounts"
response = requests.post(
signoz.self.host_configs["8080"].get(endpoint),
headers={"Authorization": f"Bearer {admin_token}"},
json={
"config": {
"gcp": {"deploymentRegion": "us-central1", "regions": ["us-central1"]}
},
"credentials": {
"sigNozApiURL": "https://test.signoz.cloud",
"sigNozApiKey": "test-key",
"ingestionUrl": "https://ingest.test.signoz.cloud",
"ingestionKey": "test-ingestion-key",
},
},
timeout=10,
)
assert (
response.status_code == HTTPStatus.BAD_REQUEST
), f"Expected 400 for unsupported provider, got {response.status_code}"
response_data = response.json()
assert "error" in response_data, "Response should contain 'error' field"

View File

@@ -1,129 +0,0 @@
import uuid
from http import HTTPStatus
from typing import Callable
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.cloudintegrationsutils import simulate_agent_checkin
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
CLOUD_PROVIDER = "aws"
def test_apply_license(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
make_http_mocks: Callable[[types.TestContainerDocker, list], None],
get_token: Callable[[str, str], str],
) -> None:
"""Apply a license so that subsequent cloud integration calls succeed."""
add_license(signoz, make_http_mocks, get_token)
def test_agent_check_in(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Test agent check-in with new camelCase fields returns 200 with expected response shape."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(
admin_token, CLOUD_PROVIDER, regions=["us-east-1"]
)
account_id = account["id"]
provider_account_id = str(uuid.uuid4())
response = simulate_agent_checkin(
signoz,
admin_token,
CLOUD_PROVIDER,
account_id,
provider_account_id,
data={"version": "v0.0.8"},
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}: {response.text}"
data = response.json()["data"]
# New camelCase fields
assert data["cloudIntegrationId"] == account_id, "cloudIntegrationId should match"
assert (
data["providerAccountId"] == provider_account_id
), "providerAccountId should match"
assert "integrationConfig" in data, "Response should contain 'integrationConfig'"
assert data["removedAt"] is None, "removedAt should be null for a live account"
# Backward-compat snake_case fields
assert data["account_id"] == account_id, "account_id (compat) should match"
assert (
data["cloud_account_id"] == provider_account_id
), "cloud_account_id (compat) should match"
assert (
"integration_config" in data
), "Response should contain 'integration_config' (compat)"
assert "removed_at" in data, "Response should contain 'removed_at' (compat)"
# integrationConfig should reflect the configured regions
integration_config = data["integrationConfig"]
assert "aws" in integration_config, "integrationConfig should contain 'aws' block"
assert integration_config["aws"]["enabledRegions"] == [
"us-east-1"
], "enabledRegions should match account config"
def test_agent_check_in_account_not_found(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""Test that check-in with an unknown cloudIntegrationId returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
fake_id = str(uuid.uuid4())
response = simulate_agent_checkin(
signoz, admin_token, CLOUD_PROVIDER, fake_id, str(uuid.uuid4())
)
assert (
response.status_code == HTTPStatus.NOT_FOUND
), f"Expected 404, got {response.status_code}: {response.text}"
def test_duplicate_cloud_account_checkins(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Test that two different accounts cannot check in with the same providerAccountId."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account1 = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account2 = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
assert account1["id"] != account2["id"], "Two accounts should have different IDs"
same_provider_account_id = str(uuid.uuid4())
# First check-in: account1 claims the provider account ID
response = simulate_agent_checkin(
signoz, admin_token, CLOUD_PROVIDER, account1["id"], same_provider_account_id
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200 for first check-in, got {response.status_code}: {response.text}"
# Second check-in: account2 tries to claim the same provider account ID → 409
response = simulate_agent_checkin(
signoz, admin_token, CLOUD_PROVIDER, account2["id"], same_provider_account_id
)
assert (
response.status_code == HTTPStatus.CONFLICT
), f"Expected 409 for duplicate providerAccountId, got {response.status_code}: {response.text}"

View File

@@ -1,341 +0,0 @@
import uuid
from http import HTTPStatus
from typing import Callable
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.cloudintegrationsutils import simulate_agent_checkin
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
CLOUD_PROVIDER = "aws"
def test_apply_license(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
make_http_mocks: Callable[[types.TestContainerDocker, list], None],
get_token: Callable[[str, str], str],
) -> None:
"""Apply a license so that subsequent cloud integration calls succeed."""
add_license(signoz, make_http_mocks, get_token)
def test_list_accounts_empty(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""List accounts returns an empty list when no accounts have checked in."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert "accounts" in data, "Response should contain 'accounts' field"
assert isinstance(data["accounts"], list), "accounts should be a list"
assert (
len(data["accounts"]) == 0
), "accounts list should be empty when no accounts have checked in"
def test_list_accounts_after_checkin(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""List accounts returns an account after it has checked in."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(
admin_token, CLOUD_PROVIDER, regions=["us-east-1"]
)
account_id = account["id"]
provider_account_id = str(uuid.uuid4())
checkin = simulate_agent_checkin(
signoz, admin_token, CLOUD_PROVIDER, account_id, provider_account_id
)
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}"
data = response.json()["data"]
found = next((a for a in data["accounts"] if a["id"] == account_id), None)
assert (
found is not None
), f"Account {account_id} should appear in list after check-in"
assert (
found["providerAccountId"] == provider_account_id
), "providerAccountId should match"
assert found["config"]["aws"]["regions"] == [
"us-east-1"
], "regions should match account config"
assert (
found["agentReport"] is not None
), "agentReport should be present after check-in"
assert found["removedAt"] is None, "removedAt should be null for a live account"
def test_get_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Get a specific account by ID returns the account with correct fields."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(
admin_token, CLOUD_PROVIDER, regions=["us-east-1", "eu-west-1"]
)
account_id = account["id"]
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert data["id"] == account_id, "id should match"
assert data["config"]["aws"]["regions"] == [
"us-east-1",
"eu-west-1",
], "regions should match"
assert data["removedAt"] is None, "removedAt should be null"
def test_get_account_not_found(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""Get a non-existent account returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{uuid.uuid4()}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NOT_FOUND
), f"Expected 404, got {response.status_code}"
def test_update_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Update account config and verify the change is persisted via GET."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(
admin_token, CLOUD_PROVIDER, regions=["us-east-1"]
)
account_id = account["id"]
updated_regions = ["us-east-1", "us-west-2", "eu-west-1"]
response = requests.put(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": {"aws": {"regions": updated_regions}}},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204, got {response.status_code}"
get_response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert get_response.status_code == HTTPStatus.OK
assert (
get_response.json()["data"]["config"]["aws"]["regions"] == updated_regions
), "Regions should reflect the update"
def test_update_account_after_checkin_preserves_connected_status(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Updating config after agent check-in must not remove the account from the connected list.
Regression test: previously, updating an account would reset account_id to NULL,
causing the account to disappear from the connected accounts listing
(which filters on account_id IS NOT NULL).
"""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
# 1. Create account
account = create_cloud_integration_account(
admin_token, CLOUD_PROVIDER, regions=["us-east-1"]
)
account_id = account["id"]
provider_account_id = str(uuid.uuid4())
# 2. Agent checks in — sets account_id and last_agent_report
checkin = simulate_agent_checkin(
signoz, admin_token, CLOUD_PROVIDER, account_id, provider_account_id
)
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
# 3. Verify the account appears in the connected list
list_response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert list_response.status_code == HTTPStatus.OK
accounts_before = list_response.json()["data"]["accounts"]
found_before = next((a for a in accounts_before if a["id"] == account_id), None)
assert found_before is not None, "Account should be listed after check-in"
# 4. Update account config
updated_regions = ["us-east-1", "us-west-2"]
update_response = requests.put(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": {"aws": {"regions": updated_regions}}},
timeout=10,
)
assert (
update_response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204, got {update_response.status_code}"
# 5. Verify the account still appears in the connected list with correct fields
list_response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert list_response.status_code == HTTPStatus.OK
accounts_after = list_response.json()["data"]["accounts"]
found_after = next((a for a in accounts_after if a["id"] == account_id), None)
assert (
found_after is not None
), "Account must still be listed after config update (account_id should not be reset)"
assert (
found_after["providerAccountId"] == provider_account_id
), "providerAccountId should be preserved after update"
assert (
found_after["agentReport"] is not None
), "agentReport should be preserved after update"
assert (
found_after["config"]["aws"]["regions"] == updated_regions
), "Config should reflect the update"
assert found_after["removedAt"] is None, "removedAt should still be null"
def test_disconnect_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Disconnect an account removes it from the connected list."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
checkin = simulate_agent_checkin(
signoz, admin_token, CLOUD_PROVIDER, account_id, str(uuid.uuid4())
)
assert checkin.status_code == HTTPStatus.OK, f"Check-in failed: {checkin.text}"
response = requests.delete(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204, got {response.status_code}"
list_response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
accounts = list_response.json()["data"]["accounts"]
assert not any(
a["id"] == account_id for a in accounts
), "Disconnected account should not appear in the connected list"
def test_disconnect_account_idempotent(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""Disconnect on a non-existent account ID returns 204 (blind update, no existence check)."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.delete(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{uuid.uuid4()}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204, got {response.status_code}"

View File

@@ -1,445 +0,0 @@
import uuid
from http import HTTPStatus
from typing import Callable
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD, add_license
from fixtures.logger import setup_logger
logger = setup_logger(__name__)
CLOUD_PROVIDER = "aws"
SERVICE_ID = "rds"
def test_apply_license(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
make_http_mocks: Callable[[types.TestContainerDocker, list], None],
get_token: Callable[[str, str], str],
) -> None:
"""Apply a license so that subsequent cloud integration calls succeed."""
add_license(signoz, make_http_mocks, get_token)
def test_list_services_without_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""List available services without specifying a cloud_integration_id."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert "services" in data, "Response should contain 'services' field"
assert isinstance(data["services"], list), "services should be a list"
assert len(data["services"]) > 0, "services list should be non-empty"
service = data["services"][0]
assert "id" in service, "Service should have 'id' field"
assert "title" in service, "Service should have 'title' field"
assert "icon" in service, "Service should have 'icon' field"
assert "enabled" in service, "Service should have 'enabled' field"
def test_list_services_with_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""List services filtered to a specific account — all disabled by default."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services?cloud_integration_id={account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert "services" in data, "Response should contain 'services' field"
assert len(data["services"]) > 0, "services list should be non-empty"
for svc in data["services"]:
assert "enabled" in svc, "Each service should have 'enabled' field"
assert (
svc["enabled"] is False
), f"Service {svc['id']} should be disabled before any config is set"
def test_get_service_details_without_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""Get full service definition without specifying an account."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/{SERVICE_ID}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert data["id"] == SERVICE_ID, f"id should be '{SERVICE_ID}'"
assert "title" in data, "Service should have 'title'"
assert "overview" in data, "Service should have 'overview' (markdown)"
assert "assets" in data, "Service should have 'assets'"
assert isinstance(
data["assets"]["dashboards"], list
), "assets.dashboards should be a list"
assert (
"telemetryCollectionStrategy" in data
), "Service should have 'telemetryCollectionStrategy'"
assert (
data["cloudIntegrationService"] is None
), "cloudIntegrationService should be null without account context"
def test_get_service_details_with_account(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Get service details with account context — cloudIntegrationService is null before first UpdateService."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/{SERVICE_ID}"
f"?cloud_integration_id={account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.OK
), f"Expected 200, got {response.status_code}"
data = response.json()["data"]
assert data["id"] == SERVICE_ID
assert (
data["cloudIntegrationService"] is None
), "cloudIntegrationService should be null before any service config is set"
def test_get_service_not_found(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""Get a non-existent service ID returns 400 (invalid service ID is a bad request)."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/non-existent-service"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.BAD_REQUEST
), f"Expected 400, got {response.status_code}"
def test_update_service_config(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Enable a service and verify the config is persisted via GET."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
put_response = requests.put(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"
),
headers={"Authorization": f"Bearer {admin_token}"},
json={
"config": {"aws": {"metrics": {"enabled": True}, "logs": {"enabled": True}}}
},
timeout=10,
)
assert (
put_response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204, got {put_response.status_code}: {put_response.text}"
get_response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/{SERVICE_ID}"
f"?cloud_integration_id={account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert get_response.status_code == HTTPStatus.OK
data = get_response.json()["data"]
svc = data["cloudIntegrationService"]
assert (
svc is not None
), "cloudIntegrationService should be non-null after UpdateService"
assert (
svc["config"]["aws"]["metrics"]["enabled"] is True
), "metrics should be enabled"
assert svc["config"]["aws"]["logs"]["enabled"] is True, "logs should be enabled"
assert (
svc["cloudIntegrationId"] == account_id
), "cloudIntegrationId should match the account"
def test_update_service_config_disable(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Enable then disable a service — config change is persisted."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
endpoint = signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"
)
# Enable
r = requests.put(
endpoint,
headers={"Authorization": f"Bearer {admin_token}"},
json={
"config": {"aws": {"metrics": {"enabled": True}, "logs": {"enabled": True}}}
},
timeout=10,
)
assert (
r.status_code == HTTPStatus.NO_CONTENT
), f"Enable failed: {r.status_code}: {r.text}"
# Disable
r = requests.put(
endpoint,
headers={"Authorization": f"Bearer {admin_token}"},
json={
"config": {
"aws": {"metrics": {"enabled": False}, "logs": {"enabled": False}}
}
},
timeout=10,
)
assert (
r.status_code == HTTPStatus.NO_CONTENT
), f"Disable failed: {r.status_code}: {r.text}"
get_response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/{SERVICE_ID}"
f"?cloud_integration_id={account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert get_response.status_code == HTTPStatus.OK
svc = get_response.json()["data"]["cloudIntegrationService"]
assert (
svc is not None
), "cloudIntegrationService should still be present after disable"
assert (
svc["config"]["aws"]["metrics"]["enabled"] is False
), "metrics should be disabled"
assert svc["config"]["aws"]["logs"]["enabled"] is False, "logs should be disabled"
def test_update_service_account_not_found(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""PUT with a non-existent account UUID returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.put(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{uuid.uuid4()}/services/{SERVICE_ID}"
),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": {"aws": {"metrics": {"enabled": True}}}},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NOT_FOUND
), f"Expected 404, got {response.status_code}"
def test_list_services_unsupported_provider(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
) -> None:
"""List services for an unsupported cloud provider returns 400."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = requests.get(
signoz.self.host_configs["8080"].get("/api/v1/cloud_integrations/gcp/services"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.BAD_REQUEST
), f"Expected 400, got {response.status_code}"
def test_list_services_account_removed(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""List services with a cloud_integration_id for a deleted account returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
delete_response = requests.delete(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
delete_response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204 on delete, got {delete_response.status_code}"
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services?cloud_integration_id={account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NOT_FOUND
), f"Expected 404, got {response.status_code}"
def test_get_service_details_account_removed(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""Get service details with a cloud_integration_id for a deleted account returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
delete_response = requests.delete(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
delete_response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204 on delete, got {delete_response.status_code}"
response = requests.get(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/services/{SERVICE_ID}"
f"?cloud_integration_id={account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NOT_FOUND
), f"Expected 404, got {response.status_code}"
def test_update_service_account_removed(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""PUT service config for a deleted account returns 404."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, CLOUD_PROVIDER)
account_id = account["id"]
delete_response = requests.delete(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}"
),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert (
delete_response.status_code == HTTPStatus.NO_CONTENT
), f"Expected 204 on delete, got {delete_response.status_code}"
response = requests.put(
signoz.self.host_configs["8080"].get(
f"/api/v1/cloud_integrations/{CLOUD_PROVIDER}/accounts/{account_id}/services/{SERVICE_ID}"
),
headers={"Authorization": f"Bearer {admin_token}"},
json={"config": {"aws": {"metrics": {"enabled": True}}}},
timeout=10,
)
assert (
response.status_code == HTTPStatus.NOT_FOUND
), f"Expected 404, got {response.status_code}"