Compare commits

...

1 Commits

Author SHA1 Message Date
swapnil-signoz
3669ea6aea feat: adding gcp cloud integration agent mode changes 2026-10-13 09:48:27 +05:30
21 changed files with 911 additions and 87 deletions

View File

@@ -1953,14 +1953,74 @@ components:
items:
type: string
type: array
setupMode:
$ref: '#/components/schemas/CloudintegrationtypesGCPSetupMode'
required:
- setupMode
- deploymentProjectId
- deploymentRegion
- projectIds
type: object
CloudintegrationtypesGCPConnectionArtifact:
properties:
cliCommand:
type: string
required:
- cliCommand
type: object
CloudintegrationtypesGCPIntegrationConfig:
properties:
deploymentProjectId:
type: string
deploymentRegion:
type: string
projectIds:
items:
type: string
type: array
setupMode:
$ref: '#/components/schemas/CloudintegrationtypesGCPSetupMode'
telemetryCollectionStrategy:
$ref: '#/components/schemas/CloudintegrationtypesGCPTelemetryCollectionStrategy'
required:
- setupMode
- deploymentProjectId
- deploymentRegion
- projectIds
- telemetryCollectionStrategy
type: object
CloudintegrationtypesGCPLogsCollectionStrategy:
properties:
sinkFilter:
type: string
required:
- sinkFilter
type: object
CloudintegrationtypesGCPMetricsCollectionStrategy:
properties:
metricNames:
items:
type: string
type: array
required:
- metricNames
type: object
CloudintegrationtypesGCPPostableAccountConfig:
properties:
deploymentProjectId:
type: string
deploymentRegion:
type: string
projectIds:
items:
type: string
type: array
setupMode:
$ref: '#/components/schemas/CloudintegrationtypesGCPSetupMode'
required:
- setupMode
- deploymentProjectId
- deploymentRegion
type: object
CloudintegrationtypesGCPServiceConfig:
properties:
@@ -1983,6 +2043,18 @@ components:
required:
- enabled
type: object
CloudintegrationtypesGCPSetupMode:
enum:
- manual
- agent
type: string
CloudintegrationtypesGCPTelemetryCollectionStrategy:
properties:
logs:
$ref: '#/components/schemas/CloudintegrationtypesGCPLogsCollectionStrategy'
metrics:
$ref: '#/components/schemas/CloudintegrationtypesGCPMetricsCollectionStrategy'
type: object
CloudintegrationtypesGettableAccountWithConnectionArtifact:
properties:
connectionArtifact:
@@ -2119,7 +2191,7 @@ components:
azure:
$ref: '#/components/schemas/CloudintegrationtypesAzureAccountConfig'
gcp:
$ref: '#/components/schemas/CloudintegrationtypesGCPAccountConfig'
$ref: '#/components/schemas/CloudintegrationtypesGCPPostableAccountConfig'
type: object
CloudintegrationtypesPostableAgentCheckIn:
properties:
@@ -2354,7 +2426,6 @@ components:
projectIds:
items:
type: string
nullable: true
type: array
required:
- deploymentProjectId

View File

@@ -17,14 +17,60 @@ func NewGCPCloudProvider(defStore cloudintegrationtypes.ServiceDefinitionStore)
}
}
func (g *gcpcloudprovider) BuildIntegrationConfig(ctx context.Context, account *cloudintegrationtypes.Account, services []*cloudintegrationtypes.StorableCloudIntegrationService) (*cloudintegrationtypes.ProviderIntegrationConfig, error) {
// for manual flow we don't have any integration config to return, so returning empty config for now.
return &cloudintegrationtypes.ProviderIntegrationConfig{}, nil
func (g *gcpcloudprovider) BuildIntegrationConfig(
ctx context.Context,
account *cloudintegrationtypes.Account,
services []*cloudintegrationtypes.StorableCloudIntegrationService,
) (*cloudintegrationtypes.ProviderIntegrationConfig, error) {
if account.Config.GCP.SetupMode != cloudintegrationtypes.GCPSetupModeAgent {
return &cloudintegrationtypes.ProviderIntegrationConfig{}, nil
}
var metricNames []string
var sinkFilters []string
for _, storedSvc := range services {
svcCfg, err := cloudintegrationtypes.NewServiceConfigFromJSON(cloudintegrationtypes.CloudProviderTypeGCP, storedSvc.Config)
if err != nil {
return nil, err
}
svcDef, err := g.GetServiceDefinition(ctx, storedSvc.Type)
if err != nil {
return nil, err
}
if svcDef.TelemetryCollectionStrategy.GCP == nil {
continue
}
strategy := svcDef.TelemetryCollectionStrategy.GCP
if svcCfg.IsMetricsEnabled(cloudintegrationtypes.CloudProviderTypeGCP) && strategy.Metrics != nil {
metricNames = append(metricNames, strategy.Metrics.MetricNames...)
}
if svcCfg.IsLogsEnabled(cloudintegrationtypes.CloudProviderTypeGCP) && strategy.Logs != nil {
sinkFilters = append(sinkFilters, strategy.Logs.SinkFilter)
}
}
return &cloudintegrationtypes.ProviderIntegrationConfig{
GCP: cloudintegrationtypes.NewGCPIntegrationConfig(account.Config.GCP, metricNames, sinkFilters),
}, nil
}
func (g *gcpcloudprovider) GetConnectionArtifact(ctx context.Context, account *cloudintegrationtypes.Account, req *cloudintegrationtypes.GetConnectionArtifactRequest) (*cloudintegrationtypes.ConnectionArtifact, error) {
// for manual flow we don't have any connection artifact to return, so returning empty artifact for now.
return &cloudintegrationtypes.ConnectionArtifact{}, nil
if account.Config.GCP.SetupMode != cloudintegrationtypes.GCPSetupModeAgent {
return &cloudintegrationtypes.ConnectionArtifact{}, nil
}
connectionArtifact, err := cloudintegrationtypes.NewGCPConnectionArtifact(account.ID, req.Config.AgentVersion, req.Credentials, account.Config.GCP)
if err != nil {
return nil, err
}
return &cloudintegrationtypes.ConnectionArtifact{
GCP: connectionArtifact,
}, nil
}
func (g *gcpcloudprovider) GetServiceDefinition(ctx context.Context, serviceID cloudintegrationtypes.ServiceID) (*cloudintegrationtypes.ServiceDefinition, error) {

View File

@@ -171,18 +171,6 @@ func (module *module) AgentCheckIn(ctx context.Context, orgID valuer.UUID, provi
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
}
connectedAccount, err := module.store.GetConnectedAccountByProviderAccountID(ctx, orgID, req.ProviderAccountID, provider)
if err != nil && !errors.Ast(err, errors.TypeNotFound) {
return nil, err
}
// If a different integration is already connected to this provider account ID, reject the check-in.
// Allow re-check-in from the same integration (e.g. agent restarting).
if connectedAccount != nil && connectedAccount.ID != req.CloudIntegrationID {
errMessage := fmt.Sprintf("provider account id %s is already connected to cloud integration id %s", req.ProviderAccountID, connectedAccount.ID)
return nil, errors.New(errors.TypeAlreadyExists, cloudintegrationtypes.ErrCodeCloudIntegrationAlreadyConnected, errMessage)
}
storableAccount, err := module.store.GetAccountByID(ctx, orgID, req.CloudIntegrationID, provider)
if err != nil {
return nil, err
@@ -195,41 +183,33 @@ func (module *module) AgentCheckIn(ctx context.Context, orgID valuer.UUID, provi
syncState := account.NextSyncState(req.SyncedVersion)
// If account has been removed (disconnected), return a minimal response with empty integration config.
if account.RemovedAt != nil {
// Heartbeat stays frozen after removal, only the sync state is updated.
if account.AgentReport != nil && syncState != nil {
account.UpdateSyncState(syncState)
return module.checkInRemovedAccount(ctx, account, req.ProviderAccountID, syncState)
}
storableAccount, err = cloudintegrationtypes.NewStorableCloudIntegration(account)
if err != nil {
return nil, err
}
if err := account.ValidateProviderAccountID(req.ProviderAccountID); err != nil {
return nil, err
}
err = module.store.UpdateAgentReport(ctx, storableAccount)
if err != nil {
return nil, err
}
}
connectedAccount, err := module.store.GetConnectedAccountByProviderAccountID(ctx, orgID, req.ProviderAccountID, provider)
if err != nil && !errors.Ast(err, errors.TypeNotFound) {
return nil, err
}
return cloudintegrationtypes.NewAgentCheckInResponse(
req.ProviderAccountID,
account.ID.StringValue(),
new(cloudintegrationtypes.ProviderIntegrationConfig),
account.RemovedAt,
syncState,
), nil
// If a different integration is already connected to this provider account ID, reject the check-in.
// Allow re-check-in from the same integration (e.g. agent restarting).
if connectedAccount != nil && connectedAccount.ID != req.CloudIntegrationID {
errMessage := fmt.Sprintf("provider account id %s is already connected to cloud integration id %s", req.ProviderAccountID, connectedAccount.ID)
return nil, errors.New(errors.TypeAlreadyExists, cloudintegrationtypes.ErrCodeCloudIntegrationAlreadyConnected, errMessage)
}
account.UpdateAgentReport(&req.ProviderAccountID, cloudintegrationtypes.NewAgentReport(req.Data, syncState))
storableAccount, err = cloudintegrationtypes.NewStorableCloudIntegration(account)
if err != nil {
return nil, err
}
err = module.store.UpdateAgentReport(ctx, storableAccount)
if err != nil {
if err := module.store.UpdateAgentReport(ctx, storableAccount); err != nil {
return nil, err
}
@@ -503,6 +483,37 @@ func (module *module) Collect(ctx context.Context, orgID valuer.UUID) (map[strin
return stats, nil
}
// checkInRemovedAccount returns an empty integration config and saves cleanup acknowledgments without refreshing the heartbeat.
func (module *module) checkInRemovedAccount(ctx context.Context, account *cloudintegrationtypes.Account, providerAccountID string, syncState *cloudintegrationtypes.SyncState) (*cloudintegrationtypes.AgentCheckInResponse, error) {
response := cloudintegrationtypes.NewAgentCheckInResponse(
providerAccountID,
account.ID.StringValue(),
new(cloudintegrationtypes.ProviderIntegrationConfig),
account.RemovedAt,
syncState,
)
if account.AgentReport == nil {
return response, nil
}
if syncState == nil {
return response, nil
}
account.UpdateSyncState(syncState)
storableAccount, err := cloudintegrationtypes.NewStorableCloudIntegration(account)
if err != nil {
return nil, err
}
if err := module.store.UpdateAgentReport(ctx, storableAccount); err != nil {
return nil, err
}
return response, nil
}
func (module *module) getCloudProvider(provider cloudintegrationtypes.CloudProviderType) (cloudintegration.CloudProviderModule, error) {
if cloudProviderModule, ok := module.cloudProvidersMap[provider]; ok {
return cloudProviderModule, nil

View File

@@ -3464,6 +3464,10 @@ export interface CloudintegrationtypesAzureAccountConfigDTO {
resourceGroups: string[];
}
export enum CloudintegrationtypesGCPSetupModeDTO {
manual = 'manual',
agent = 'agent',
}
export interface CloudintegrationtypesGCPAccountConfigDTO {
/**
* @type string
@@ -3477,6 +3481,7 @@ export interface CloudintegrationtypesGCPAccountConfigDTO {
* @type array
*/
projectIds: string[];
setupMode: CloudintegrationtypesGCPSetupModeDTO;
}
export interface CloudintegrationtypesAccountConfigDTO {
@@ -3711,7 +3716,10 @@ export interface CloudintegrationtypesCollectedMetricDTO {
}
export interface CloudintegrationtypesGCPConnectionArtifactDTO {
[key: string]: unknown;
/**
* @type string
*/
cliCommand: string;
}
export interface CloudintegrationtypesConnectionArtifactDTO {
@@ -3750,8 +3758,56 @@ export interface CloudintegrationtypesDataCollectedDTO {
metrics?: CloudintegrationtypesCollectedMetricDTO[] | null;
}
export interface CloudintegrationtypesGCPLogsCollectionStrategyDTO {
/**
* @type string
*/
sinkFilter: string;
}
export interface CloudintegrationtypesGCPMetricsCollectionStrategyDTO {
/**
* @type array
*/
metricNames: string[];
}
export interface CloudintegrationtypesGCPTelemetryCollectionStrategyDTO {
logs?: CloudintegrationtypesGCPLogsCollectionStrategyDTO;
metrics?: CloudintegrationtypesGCPMetricsCollectionStrategyDTO;
}
export interface CloudintegrationtypesGCPIntegrationConfigDTO {
[key: string]: unknown;
/**
* @type string
*/
deploymentProjectId: string;
/**
* @type string
*/
deploymentRegion: string;
/**
* @type array
*/
projectIds: string[];
setupMode: CloudintegrationtypesGCPSetupModeDTO;
telemetryCollectionStrategy: CloudintegrationtypesGCPTelemetryCollectionStrategyDTO;
}
export interface CloudintegrationtypesGCPPostableAccountConfigDTO {
/**
* @type string
*/
deploymentProjectId: string;
/**
* @type string
*/
deploymentRegion: string;
/**
* @type array
*/
projectIds?: string[];
setupMode: CloudintegrationtypesGCPSetupModeDTO;
}
export interface CloudintegrationtypesGettableAccountWithConnectionArtifactDTO {
@@ -3906,7 +3962,7 @@ export interface CloudintegrationtypesGettableServicesMetadataDTO {
export interface CloudintegrationtypesPostableAccountConfigDTO {
aws?: CloudintegrationtypesAWSPostableAccountConfigDTO;
azure?: CloudintegrationtypesAzureAccountConfigDTO;
gcp?: CloudintegrationtypesGCPAccountConfigDTO;
gcp?: CloudintegrationtypesGCPPostableAccountConfigDTO;
}
export interface CloudintegrationtypesPostableAccountDTO {
@@ -4050,9 +4106,9 @@ export interface CloudintegrationtypesUpdatableGCPAccountConfigDTO {
*/
deploymentRegion: string;
/**
* @type array,null
* @type array
*/
projectIds: string[] | null;
projectIds: string[];
}
export interface CloudintegrationtypesUpdatableAccountConfigDTO {

View File

@@ -142,6 +142,7 @@ describe('GCP CloudAccountSetupDrawer', () => {
expect(createAccountPayload).toStrictEqual({
config: {
gcp: {
setupMode: 'manual',
deploymentRegion: 'asia-south1',
deploymentProjectId: 'my-deployment-project-123',
projectIds: ['project-a', 'project-b'],

View File

@@ -11,6 +11,7 @@ import {
} from 'api/generated/services/cloudintegration';
import {
CloudintegrationtypesCredentialsDTO,
CloudintegrationtypesGCPSetupModeDTO,
CloudintegrationtypesPostableAccountDTO,
RenderErrorResponseDTO,
} from 'api/generated/services/sigNoz.schemas';
@@ -96,6 +97,7 @@ export function useCloudAccountSetupDrawer({
const payload: CloudintegrationtypesPostableAccountDTO = {
config: {
gcp: {
setupMode: CloudintegrationtypesGCPSetupModeDTO.manual,
deploymentRegion: values.deploymentRegion,
deploymentProjectId: values.deploymentProjectId,
projectIds: values.projectIds || [],

View File

@@ -3,6 +3,7 @@
* Do not hand-edit: regenerate instead.
*/
import { CloudintegrationtypesGCPSetupModeDTO } from 'api/generated/services/sigNoz.schemas';
import type {
CloudintegrationtypesAccountConfigDTO,
CloudintegrationtypesAccountDTO,
@@ -279,6 +280,7 @@ const accountConfig = (
return {
gcp: {
setupMode: CloudintegrationtypesGCPSetupModeDTO.manual,
deploymentRegion: 'us-central1',
deploymentProjectId: 'signoz-prod-4471',
projectIds: ['signoz-prod-4471', 'signoz-analytics-8820'],

View File

@@ -108,6 +108,12 @@ func validateServiceDefinition(def *citypes.ServiceDefinition) error {
return errors.NewInternalf(errors.CodeInternal, "telemetryCollectionStrategy is required")
}
if gcpStrategy := def.TelemetryCollectionStrategy.GCP; gcpStrategy != nil && gcpStrategy.Logs != nil {
if strings.TrimSpace(gcpStrategy.Logs.SinkFilter) == "" {
return errors.NewInternalf(errors.CodeInternal, "sinkFilter is required for GCP logs collection")
}
}
seenDashboardIDs := map[string]struct{}{}
for _, d := range def.Assets.Dashboards {
if _, seen := seenDashboardIDs[d.ID]; seen {

View File

@@ -121,7 +121,33 @@
"logs": []
},
"telemetryCollectionStrategy": {
"gcp": {}
"gcp": {
"metrics": {
"metricNames": [
"cloudsql.googleapis.com/database/up",
"cloudsql.googleapis.com/database/instance_state",
"cloudsql.googleapis.com/database/replication/replica_lag",
"cloudsql.googleapis.com/database/cpu/utilization",
"cloudsql.googleapis.com/database/memory/utilization",
"cloudsql.googleapis.com/database/disk/utilization",
"cloudsql.googleapis.com/database/network/connections",
"cloudsql.googleapis.com/database/mysql/queries",
"cloudsql.googleapis.com/database/mysql/dml_operations_count",
"cloudsql.googleapis.com/database/mysql/threads",
"cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_reads_count",
"cloudsql.googleapis.com/database/mysql/innodb/buffer_pool_read_requests_count",
"cloudsql.googleapis.com/database/mysql/slow_queries_count",
"cloudsql.googleapis.com/database/mysql/aborted_connects_count",
"cloudsql.googleapis.com/database/mysql/innodb/deadlocks_count",
"cloudsql.googleapis.com/database/mysql/innodb/row_lock_waits_count",
"cloudsql.googleapis.com/database/disk/read_ops_count",
"cloudsql.googleapis.com/database/disk/write_ops_count"
]
},
"logs": {
"sinkFilter": "resource.type=\"cloudsql_database\" AND (logName:\"/logs/cloudsql.googleapis.com%2Fmysql\" OR (log_id(\"cloudaudit.googleapis.com/data_access\") AND protoPayload.request.@type=\"type.googleapis.com/google.cloud.sql.audit.v1.MysqlAuditEntry\"))"
}
}
},
"assets": {
"dashboards": [

View File

@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
"logs": false
"logs": true
},
"dataCollected": {
"metrics": [
@@ -91,7 +91,28 @@
"logs": []
},
"telemetryCollectionStrategy": {
"gcp": {}
"gcp": {
"metrics": {
"metricNames": [
"cloudsql.googleapis.com/database/up",
"cloudsql.googleapis.com/database/cpu/utilization",
"cloudsql.googleapis.com/database/memory/utilization",
"cloudsql.googleapis.com/database/memory/usage",
"cloudsql.googleapis.com/database/disk/bytes_used",
"cloudsql.googleapis.com/database/postgresql/num_backends",
"cloudsql.googleapis.com/database/postgresql/num_backends_by_state",
"cloudsql.googleapis.com/database/postgresql/transaction_count",
"cloudsql.googleapis.com/database/postgresql/deadlock_count",
"cloudsql.googleapis.com/database/postgresql/vacuum/oldest_transaction_age",
"cloudsql.googleapis.com/database/postgresql/insights/aggregate/execution_time",
"cloudsql.googleapis.com/database/postgresql/insights/perquery/execution_time",
"cloudsql.googleapis.com/database/postgresql/replication/replica_byte_lag"
]
},
"logs": {
"sinkFilter": "resource.type=\"cloudsql_database\" AND (logName:\"/logs/cloudsql.googleapis.com%2Fpostgres\" OR (log_id(\"cloudaudit.googleapis.com/data_access\") AND protoPayload.request.@type=\"type.googleapis.com/google.cloud.sql.audit.v1.PgAuditEntry\"))"
}
}
},
"assets": {
"dashboards": [

View File

@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
"logs": false
"logs": true
},
"dataCollected": {
"metrics": [
@@ -55,7 +55,22 @@
"logs": []
},
"telemetryCollectionStrategy": {
"gcp": {}
"gcp": {
"metrics": {
"metricNames": [
"storage.googleapis.com/storage/v2/total_bytes",
"storage.googleapis.com/storage/v2/total_count",
"storage.googleapis.com/storage/v2/deleted_bytes",
"storage.googleapis.com/api/request_count",
"storage.googleapis.com/network/received_bytes_count",
"storage.googleapis.com/network/sent_bytes_count",
"storage.googleapis.com/authn/authentication_count"
]
},
"logs": {
"sinkFilter": "resource.type=\"gcs_bucket\""
}
}
},
"assets": {
"dashboards": [

View File

@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
"logs": false
"logs": true
},
"dataCollected": {
"metrics": [
@@ -109,7 +109,31 @@
"logs": []
},
"telemetryCollectionStrategy": {
"gcp": {}
"gcp": {
"metrics": {
"metricNames": [
"compute.googleapis.com/instance/uptime_total",
"compute.googleapis.com/instance/cpu/utilization",
"compute.googleapis.com/guest/memory/bytes_used",
"compute.googleapis.com/guest/memory/percent_used",
"agent.googleapis.com/memory/percent_used",
"compute.googleapis.com/instance/disk/average_io_latency",
"compute.googleapis.com/instance/disk/read_bytes_count",
"compute.googleapis.com/instance/disk/write_bytes_count",
"compute.googleapis.com/instance/disk/read_ops_count",
"compute.googleapis.com/instance/disk/write_ops_count",
"compute.googleapis.com/instance/disk/performance_status",
"compute.googleapis.com/instance/network/received_bytes_count",
"compute.googleapis.com/instance/network/sent_bytes_count",
"compute.googleapis.com/instance/network/received_packets_count",
"compute.googleapis.com/instance/network/sent_packets_count",
"compute.googleapis.com/firewall/dropped_packets_count"
]
},
"logs": {
"sinkFilter": "resource.type=\"gce_instance\" OR protoPayload.serviceName=\"compute.googleapis.com\""
}
}
},
"assets": {
"dashboards": [

View File

@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
"logs": false
"logs": true
},
"dataCollected": {
"metrics": [
@@ -109,7 +109,31 @@
"logs": []
},
"telemetryCollectionStrategy": {
"gcp": {}
"gcp": {
"metrics": {
"metricNames": [
"kubernetes.io/container/cpu/limit_utilization",
"kubernetes.io/container/cpu/request_utilization",
"kubernetes.io/container/memory/limit_utilization",
"kubernetes.io/container/memory/request_utilization",
"kubernetes.io/container/restart_count",
"kubernetes.io/pod/latencies/pod_first_ready",
"kubernetes.io/pod/network/received_bytes_count",
"kubernetes.io/pod/network/sent_bytes_count",
"kubernetes.io/pod/volume/utilization",
"kubernetes.io/node/cpu/allocatable_utilization",
"kubernetes.io/node/memory/allocatable_utilization",
"kubernetes.io/node/ephemeral_storage/used_bytes",
"kubernetes.io/node/ephemeral_storage/allocatable_bytes",
"kubernetes.io/node/network/received_bytes_count",
"kubernetes.io/node/network/sent_bytes_count",
"kubernetes.io/node/status_condition"
]
},
"logs": {
"sinkFilter": "resource.type=(\"k8s_container\" OR \"k8s_pod\" OR \"k8s_node\" OR \"k8s_cluster\" OR \"k8s_control_plane_component\" OR \"gke_cluster\" OR \"gke_nodepool\")"
}
}
},
"assets": {
"dashboards": [

View File

@@ -5,7 +5,7 @@
"overview": "file://overview.md",
"supportedSignals": {
"metrics": true,
"logs": false
"logs": true
},
"dataCollected": {
"metrics": [
@@ -79,7 +79,26 @@
"logs": []
},
"telemetryCollectionStrategy": {
"gcp": {}
"gcp": {
"metrics": {
"metricNames": [
"redis.googleapis.com/server/uptime",
"redis.googleapis.com/stats/memory/usage_ratio",
"redis.googleapis.com/stats/memory/system_memory_usage_ratio",
"redis.googleapis.com/stats/cpu_utilization_main_thread",
"redis.googleapis.com/clients/connected",
"redis.googleapis.com/stats/cache_hit_ratio",
"redis.googleapis.com/keyspace/keys",
"redis.googleapis.com/commands/usec_per_call",
"redis.googleapis.com/commands/calls",
"redis.googleapis.com/clients/blocked",
"redis.googleapis.com/stats/reject_connections_count"
]
},
"logs": {
"sinkFilter": "resource.type=\"redis_instance\" OR (protoPayload.serviceName=\"redis.googleapis.com\" AND protoPayload.methodName:\".CloudRedis.\") OR (log_id(\"maintenance.googleapis.com/activity_log\") AND labels.activity_type_name:\"CloudRedisRollout\")"
}
}
},
"assets": {
"dashboards": [

View File

@@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"maps"
"strings"
"time"
"github.com/SigNoz/signoz/pkg/errors"
@@ -150,7 +151,7 @@ func NewAccountFromStorable(storableAccount *StorableCloudIntegration) (*Account
}
account.Config.Azure = azureConfig
case CloudProviderTypeGCP:
gcpConfig := new(GCPAccountConfig)
gcpConfig := &GCPAccountConfig{SetupMode: GCPSetupModeManual}
err := json.Unmarshal([]byte(storableAccount.Config), gcpConfig)
if err != nil {
return nil, err
@@ -240,7 +241,8 @@ func NewAccountConfigFromPostable(provider CloudProviderType, config *PostableAc
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "GCP config can not be nil for GCP provider")
}
if config.GCP.DeploymentProjectID == "" {
deploymentProjectID := strings.TrimSpace(config.GCP.DeploymentProjectID)
if deploymentProjectID == "" {
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "deployment project ID is required for GCP provider")
}
@@ -248,14 +250,26 @@ func NewAccountConfigFromPostable(provider CloudProviderType, config *PostableAc
return nil, err
}
if len(config.GCP.ProjectIDs) == 0 {
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "at least one project id is required for GCP provider")
setupMode := config.GCP.SetupMode
projectIDs := config.GCP.ProjectIDs
switch setupMode {
case GCPSetupModeManual:
var err error
projectIDs, err = normalizeGCPProjectIDs(projectIDs)
if err != nil {
return nil, err
}
case GCPSetupModeAgent:
projectIDs = []string{deploymentProjectID}
default:
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "unsupported GCP setup mode: %q", setupMode.StringValue())
}
return &AccountConfig{
GCP: &GCPAccountConfig{
DeploymentProjectID: config.GCP.DeploymentProjectID,
ProjectIDs: config.GCP.ProjectIDs,
SetupMode: setupMode,
DeploymentProjectID: deploymentProjectID,
ProjectIDs: projectIDs,
DeploymentRegion: config.GCP.DeploymentRegion,
},
}, nil
@@ -301,18 +315,20 @@ func NewAccountConfigFromUpdatable(provider CloudProviderType, config *Updatable
return nil, err
}
if len(config.Config.GCP.ProjectIDs) == 0 {
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "at least one project id is required for GCP provider")
projectIDs, err := normalizeGCPProjectIDs(config.Config.GCP.ProjectIDs)
if err != nil {
return nil, err
}
if config.Config.GCP.DeploymentProjectID == "" {
deploymentProjectID := strings.TrimSpace(config.Config.GCP.DeploymentProjectID)
if deploymentProjectID == "" {
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "deployment project ID is required for GCP provider")
}
return &AccountConfig{
GCP: &GCPAccountConfig{
DeploymentProjectID: config.Config.GCP.DeploymentProjectID,
ProjectIDs: config.Config.GCP.ProjectIDs,
DeploymentProjectID: deploymentProjectID,
ProjectIDs: projectIDs,
DeploymentRegion: config.Config.GCP.DeploymentRegion,
},
}, nil
@@ -355,6 +371,13 @@ func GetSigNozAPIURLFromDeployment(deployment *zeustypes.GettableDeployment) (st
}
func (account *Account) Update(provider CloudProviderType, config *AccountConfig) error {
if provider == CloudProviderTypeGCP {
if account.Config.GCP.SetupMode == GCPSetupModeAgent {
return errors.NewInvalidInputf(ErrCodeInvalidInput, "GCP agent account configuration cannot be updated")
}
config.GCP.SetupMode = account.Config.GCP.SetupMode
}
// deployment region can not be updated once set for Azure
if provider == CloudProviderTypeAzure {
config.Azure.DeploymentRegion = account.Config.Azure.DeploymentRegion
@@ -366,6 +389,19 @@ func (account *Account) Update(provider CloudProviderType, config *AccountConfig
return nil
}
func (account *Account) ValidateProviderAccountID(providerAccountID string) error {
if account.Provider != CloudProviderTypeGCP || account.Config.GCP.SetupMode != GCPSetupModeAgent {
return nil
}
if providerAccountID != account.Config.GCP.DeploymentProjectID {
return errors.NewInvalidInputf(ErrCodeInvalidInput, "providerAccountId must match deploymentProjectId for GCP agent setup")
}
return nil
}
// UpdateAgentReport replaces the provider account ID and heartbeat for an active account.
func (account *Account) UpdateAgentReport(providerAccountID *string, agentReport *AgentReport) {
account.ProviderAccountID = providerAccountID
account.AgentReport = agentReport

View File

@@ -1,30 +1,88 @@
package cloudintegrationtypes
import (
"bytes"
"fmt"
"slices"
"strings"
"text/template"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/valuer"
)
type GCPSetupMode struct{ valuer.String }
var (
GCPSetupModeManual = GCPSetupMode{valuer.NewString("manual")}
GCPSetupModeAgent = GCPSetupMode{valuer.NewString("agent")}
// TODO: Confirm the published bootstrap module location before release.
AgentGCPTerraformModuleStorePath = "gs://signoz-cloud-integration/gcp/%s/bootstrap/"
AgentGCPBootstrapDeploymentName = "signoz-integration-bootstrap"
gcpCLITemplate = template.Must(template.New("gcpCLI").Funcs(template.FuncMap{
"shellQuote": func(value string) string {
return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'"
},
}).Parse(gcpCLITemplateStr()))
)
func (GCPSetupMode) Enum() []any {
return []any{GCPSetupModeManual, GCPSetupModeAgent}
}
type GCPAccountConfig struct {
// Project ID where central pub/sub for logs exist
SetupMode GCPSetupMode `json:"setupMode" required:"true"`
// DeploymentProjectID is the project hosting the integration resources.
DeploymentProjectID string `json:"deploymentProjectId" required:"true"`
// Project ID where otel collector will be deployed
// DeploymentRegion is the region used for the integration resources.
DeploymentRegion string `json:"deploymentRegion" required:"true"`
// List of project IDs to monitor
// Agent accounts store exactly the deployment project; manual accounts may monitor multiple projects.
ProjectIDs []string `json:"projectIds" required:"true" nullable:"false"`
}
type GCPPostableAccountConfig = GCPAccountConfig
type UpdatableGCPAccountConfig struct {
// Project ID where central pub/sub for logs exist
type GCPPostableAccountConfig struct {
SetupMode GCPSetupMode `json:"setupMode" required:"true"`
// DeploymentProjectID is the project hosting the integration resources.
DeploymentProjectID string `json:"deploymentProjectId" required:"true"`
// Compute service region where otel collector will be deployed
// DeploymentRegion is the region used for the integration resources.
DeploymentRegion string `json:"deploymentRegion" required:"true"`
// List of project IDs to monitor
ProjectIDs []string `json:"projectIds" required:"true"`
// Required for manual setup; always replaced with DeploymentProjectID for Agent setup.
ProjectIDs []string `json:"projectIds,omitempty" required:"false" nullable:"false"`
}
type GCPConnectionArtifact struct{}
// UpdatableGCPAccountConfig contains settings for manual integrations only.
type UpdatableGCPAccountConfig struct {
// DeploymentProjectID is the project hosting the manually configured integration resources.
DeploymentProjectID string `json:"deploymentProjectId" required:"true"`
// DeploymentRegion is the region used for the manually configured integration resources.
DeploymentRegion string `json:"deploymentRegion" required:"true"`
// ProjectIDs lists the projects monitored by the manual integration.
ProjectIDs []string `json:"projectIds" required:"true" nullable:"false"`
}
type GCPIntegrationConfig struct{}
type GCPConnectionArtifact struct {
CLICommand string `json:"cliCommand" required:"true"`
}
type GCPTelemetryCollectionStrategy struct{}
type GCPIntegrationConfig struct {
GCPAccountConfig
TelemetryCollectionStrategy *GCPTelemetryCollectionStrategy `json:"telemetryCollectionStrategy" required:"true" nullable:"false"`
}
type GCPTelemetryCollectionStrategy struct {
Metrics *GCPMetricsCollectionStrategy `json:"metrics,omitempty" required:"false" nullable:"false"`
Logs *GCPLogsCollectionStrategy `json:"logs,omitempty" required:"false" nullable:"false"`
}
type GCPMetricsCollectionStrategy struct {
MetricNames []string `json:"metricNames" required:"true" nullable:"false"`
}
type GCPLogsCollectionStrategy struct {
SinkFilter string `json:"sinkFilter" required:"true"`
}
type GCPServiceConfig struct {
Logs *GCPServiceLogsConfig `json:"logs,omitempty" required:"false"`
@@ -38,3 +96,95 @@ type GCPServiceLogsConfig struct {
type GCPServiceMetricsConfig struct {
Enabled bool `json:"enabled" required:"true"`
}
func NewGCPConnectionArtifact(accountID valuer.UUID, agentVersion string, creds *Credentials, cfg *GCPAccountConfig) (*GCPConnectionArtifact, error) {
data := gcpTemplateData{
DeploymentName: fmt.Sprintf("projects/%s/locations/%s/deployments/%s", cfg.DeploymentProjectID, cfg.DeploymentRegion, AgentGCPBootstrapDeploymentName),
ProjectID: cfg.DeploymentProjectID,
ServiceAccount: fmt.Sprintf("projects/%s/serviceAccounts/signoz-integration-bootstrap@%s.iam.gserviceaccount.com", cfg.DeploymentProjectID, cfg.DeploymentProjectID),
ModuleURL: fmt.Sprintf(AgentGCPTerraformModuleStorePath, agentVersion),
InputValues: fmt.Sprintf(
"project_id=%s,deployment_region=%s,integration_account_id=%s,signoz_api_url=%s,ingestion_url=%s,"+
"api_key_secret_id=signoz-integration-credentials-api-key,"+
"ingestion_key_secret_id=signoz-integration-credentials-ingestion-key",
cfg.DeploymentProjectID, cfg.DeploymentRegion, accountID.StringValue(), creds.SigNozAPIURL, creds.IngestionURL,
),
}
cliCommand, err := newGCPConnectionCLICommand(data)
if err != nil {
return nil, err
}
return &GCPConnectionArtifact{CLICommand: cliCommand}, nil
}
// NewGCPIntegrationConfig sorts and deduplicates the input slices in place and formats sink filters for joining with OR.
func NewGCPIntegrationConfig(config *GCPAccountConfig, metricNames, sinkFilters []string) *GCPIntegrationConfig {
// Parentheses preserve each service filter's meaning when combined with OR.
for i, filter := range sinkFilters {
sinkFilters[i] = "(" + strings.TrimSpace(filter) + ")"
}
// Keep the generated config stable and avoid collecting shared metrics more than once.
slices.Sort(metricNames)
metricNames = slices.Compact(metricNames)
slices.Sort(sinkFilters)
sinkFilters = slices.Compact(sinkFilters)
strategy := new(GCPTelemetryCollectionStrategy)
if len(metricNames) > 0 {
strategy.Metrics = &GCPMetricsCollectionStrategy{MetricNames: metricNames}
}
if len(sinkFilters) > 0 {
strategy.Logs = &GCPLogsCollectionStrategy{SinkFilter: strings.Join(sinkFilters, " OR ")}
}
return &GCPIntegrationConfig{
GCPAccountConfig: *config,
TelemetryCollectionStrategy: strategy,
}
}
type gcpTemplateData struct {
DeploymentName string
ProjectID string
ServiceAccount string
ModuleURL string
InputValues string
}
func newGCPConnectionCLICommand(data gcpTemplateData) (string, error) {
var buf bytes.Buffer
if err := gcpCLITemplate.Execute(&buf, data); err != nil {
return "", err
}
return buf.String(), nil
}
func gcpCLITemplateStr() string {
return `gcloud infra-manager deployments apply \
{{.DeploymentName | shellQuote}} \
--project={{.ProjectID | shellQuote}} \
--service-account={{.ServiceAccount | shellQuote}} \
--gcs-source={{.ModuleURL | shellQuote}} \
--tf-version-constraint='=1.5.7' \
--input-values={{.InputValues | shellQuote}},api_key_secret_version="${api_key_version:-latest}",ingestion_key_secret_version="${ingestion_key_version:-latest}"`
}
func normalizeGCPProjectIDs(projectIDs []string) ([]string, error) {
if len(projectIDs) == 0 {
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "at least one project id is required for GCP provider")
}
normalized := make([]string, len(projectIDs))
for i, projectID := range projectIDs {
projectID = strings.TrimSpace(projectID)
if projectID == "" {
return nil, errors.NewInvalidInputf(ErrCodeInvalidInput, "projectIds cannot contain an empty or whitespace-only project ID")
}
normalized[i] = projectID
}
return normalized, nil
}

View File

@@ -49,8 +49,7 @@ class ProviderAccountSpec:
class ProviderServiceSpec:
provider: str
service_id: str
# GCP ships every service with supportedSignals.logs false, so a logs block
# is neither required on write nor persisted.
# Whether writes require a logs block and reads include it.
supports_logs: bool
account_config: dict
# id shown in parametrized test names; defaults to the provider slug.

View File

@@ -1,3 +1,5 @@
import os
import subprocess
from collections.abc import Callable
from http import HTTPStatus
@@ -27,12 +29,14 @@ GCP_ACCOUNT_SPEC = ProviderAccountSpec(
},
build_config=lambda p: {
"gcp": {
"setupMode": "manual",
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
}
},
expected_config=lambda p: {
"setupMode": "manual",
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
@@ -91,6 +95,56 @@ def test_create_account(
assert artifact.get("gcp") is None, f"GCP should not return a connection artifact, got: {artifact}"
@pytest.mark.parametrize(
"secret_versions,expected_api_key_version,expected_ingestion_key_version",
[
pytest.param({"api_key_version": "3", "ingestion_key_version": "7"}, "3", "7", id="uploaded_versions"),
pytest.param({}, "latest", "latest", id="latest_versions"),
],
)
def test_create_gcp_agent_account_connection_command(
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
secret_versions: dict,
expected_api_key_version: str,
expected_ingestion_key_version: str,
) -> None:
"""The generated apply command uses the bootstrap's secret references and the uploaded versions, defaulting to latest."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(
admin_token,
"gcp",
config={"gcp": {"setupMode": "agent", "deploymentProjectId": " signoz-agent-project ", "deploymentRegion": "us-central1"}},
)
command = account["connectionArtifact"]["gcp"]["cliCommand"]
# Capture the expanded arguments with a shell function instead of deploying GCP resources.
script = "gcloud() { printf '%s\\0' \"$@\"; }\n" + command
env = {key: value for key, value in os.environ.items() if key not in ("api_key_version", "ingestion_key_version")} | secret_versions
result = subprocess.run(["/bin/sh", "-c", script], env=env, capture_output=True, text=True, check=True, timeout=5)
args = result.stdout.split("\0")[:-1]
assert args[:4] == ["infra-manager", "deployments", "apply", "projects/signoz-agent-project/locations/us-central1/deployments/signoz-integration-bootstrap"]
flags = dict(arg.removeprefix("--").split("=", 1) for arg in args[4:])
assert flags["project"] == "signoz-agent-project"
assert flags["service-account"] == "projects/signoz-agent-project/serviceAccounts/signoz-integration-bootstrap@signoz-agent-project.iam.gserviceaccount.com"
assert flags["gcs-source"] == "gs://signoz-cloud-integration/gcp/v0.0.8/bootstrap/"
assert flags["tf-version-constraint"] == "=1.5.7"
assert dict(value.split("=", 1) for value in flags["input-values"].split(",")) == {
"project_id": "signoz-agent-project",
"deployment_region": "us-central1",
"integration_account_id": account["id"],
"signoz_api_url": "https://test-deployment.test.signoz.cloud",
"ingestion_url": "https://ingest.test.signoz.cloud",
"api_key_secret_id": "signoz-integration-credentials-api-key",
"api_key_secret_version": expected_api_key_version,
"ingestion_key_secret_id": "signoz-integration-credentials-ingestion-key",
"ingestion_key_secret_version": expected_ingestion_key_version,
}
assert "test-api-key-789" not in command
assert "test-ingestion-key-123456" not in command
def test_create_account_unsupported_provider(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
@@ -135,6 +189,7 @@ def test_create_gcp_account_without_project_ids(
json={
"config": {
"gcp": {
"setupMode": "manual",
"deploymentProjectId": "signoz-test-project",
"deploymentRegion": "us-central1",
"projectIds": [],

View File

@@ -31,12 +31,14 @@ GCP_ACCOUNT_SPEC = ProviderAccountSpec(
},
build_config=lambda p: {
"gcp": {
"setupMode": "manual",
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
}
},
expected_config=lambda p: {
"setupMode": "manual",
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
@@ -51,6 +53,25 @@ provider_spec = pytest.mark.parametrize(
ids=[s.id for s in PROVIDER_ACCOUNT_SPECS],
)
provider_account_reuse = pytest.mark.parametrize(
"provider,account_config,provider_account_id",
[
pytest.param("aws", {"aws": {"deploymentRegion": "us-east-1", "regions": ["us-east-1"]}}, "123456789012", id="aws"),
pytest.param(
"gcp",
{"gcp": {"setupMode": "manual", "deploymentProjectId": "signoz-reuse-project", "deploymentRegion": "us-central1", "projectIds": ["signoz-reuse-project"]}},
"reused-manual-account",
id="gcp-manual",
),
pytest.param(
"gcp",
{"gcp": {"setupMode": "agent", "deploymentProjectId": "signoz-reuse-project", "deploymentRegion": "us-central1"}},
"signoz-reuse-project",
id="gcp-agent",
),
],
)
def test_apply_license(
signoz: types.SigNoz,
@@ -155,6 +176,201 @@ def test_duplicate_cloud_account_checkins(
assert response.status_code == HTTPStatus.CONFLICT, f"Expected 409 for duplicate providerAccountId, got {response.status_code}: {response.text}"
def test_gcp_agent_check_in_returns_collection_strategy(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
headers = {"Authorization": f"Bearer {admin_token}"}
project_id = "signoz-agent-strategy-project"
account_id = create_cloud_integration_account(admin_token, "gcp", config={"gcp": {"setupMode": "agent", "deploymentProjectId": project_id, "deploymentRegion": "us-central1"}})["id"]
services_url = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/gcp/accounts/{account_id}/services")
mysql_sink_filter = '(resource.type="cloudsql_database" AND (logName:"/logs/cloudsql.googleapis.com%2Fmysql" OR (log_id("cloudaudit.googleapis.com/data_access") AND protoPayload.request.@type="type.googleapis.com/google.cloud.sql.audit.v1.MysqlAuditEntry")))'
postgres_sink_filter = '(resource.type="cloudsql_database" AND (logName:"/logs/cloudsql.googleapis.com%2Fpostgres" OR (log_id("cloudaudit.googleapis.com/data_access") AND protoPayload.request.@type="type.googleapis.com/google.cloud.sql.audit.v1.PgAuditEntry")))'
response = simulate_agent_checkin(signoz, admin_token, "gcp", account_id, project_id)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"]["integrationConfig"]["gcp"] == {
"setupMode": "agent",
"deploymentProjectId": project_id,
"deploymentRegion": "us-central1",
"projectIds": [project_id],
"telemetryCollectionStrategy": {},
}
for service_id, service_config in (
("cloudsql_mysql", {"metrics": {"enabled": True}, "logs": {"enabled": False}}),
("cloudsql_postgres", {"metrics": {"enabled": True}, "logs": {"enabled": True}}),
):
response = requests.put(f"{services_url}/{service_id}", headers=headers, json={"config": {"gcp": service_config}}, timeout=10)
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
response = simulate_agent_checkin(signoz, admin_token, "gcp", account_id, project_id)
assert response.status_code == HTTPStatus.OK, response.text
strategy = response.json()["data"]["integrationConfig"]["gcp"]["telemetryCollectionStrategy"]
metric_names = strategy["metrics"]["metricNames"]
assert metric_names == sorted(set(metric_names)), "metrics shared by MySQL and Postgres should be listed once, sorted"
assert {"cloudsql.googleapis.com/database/up", "cloudsql.googleapis.com/database/mysql/queries", "cloudsql.googleapis.com/database/postgresql/num_backends"} <= set(metric_names)
assert strategy["logs"] == {"sinkFilter": postgres_sink_filter}, "only services with logs enabled should contribute a sink filter"
response = requests.put(f"{services_url}/cloudsql_mysql", headers=headers, json={"config": {"gcp": {"metrics": {"enabled": False}, "logs": {"enabled": True}}}}, timeout=10)
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
response = simulate_agent_checkin(signoz, admin_token, "gcp", account_id, project_id)
assert response.status_code == HTTPStatus.OK, response.text
strategy = response.json()["data"]["integrationConfig"]["gcp"]["telemetryCollectionStrategy"]
assert "cloudsql.googleapis.com/database/up" in strategy["metrics"]["metricNames"], "Postgres should still collect the shared metrics"
assert not [name for name in strategy["metrics"]["metricNames"] if "/database/mysql/" in name], "MySQL metrics should stop once its metrics are disabled"
assert strategy["logs"] == {"sinkFilter": f"{mysql_sink_filter} OR {postgres_sink_filter}"}
def test_gcp_agent_check_in_rejects_other_project(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
project_id = "signoz-agent-mismatch-project"
account_id = create_cloud_integration_account(admin_token, "gcp", config={"gcp": {"setupMode": "agent", "deploymentProjectId": project_id, "deploymentRegion": "us-central1"}})["id"]
response = simulate_agent_checkin(signoz, admin_token, "gcp", account_id, "signoz-other-project")
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
response = requests.get(signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/gcp/accounts/{account_id}"), headers={"Authorization": f"Bearer {admin_token}"}, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"]["providerAccountId"] is None, "a rejected check-in must not connect the account"
assert response.json()["data"]["agentReport"] is None
response = simulate_agent_checkin(signoz, admin_token, "gcp", account_id, project_id)
assert response.status_code == HTTPStatus.OK, response.text
@provider_account_reuse
def test_removed_account_checkins_after_provider_account_id_is_reused(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
provider: str,
account_config: dict,
provider_account_id: str,
) -> None:
"""Connect old account, remove it, connect replacement, then check in from the old agent."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
headers = {"Authorization": f"Bearer {admin_token}"}
accounts_url = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{provider}/accounts")
# The original agent connects before its integration is removed.
old_account_id = create_cloud_integration_account(admin_token, provider, config=account_config)["id"]
response = simulate_agent_checkin(signoz, admin_token, provider, old_account_id, provider_account_id, data={"version": "old-agent"})
assert response.status_code == HTTPStatus.OK, response.text
response = requests.delete(f"{accounts_url}/{old_account_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
response = requests.get(f"{accounts_url}/{old_account_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
old_account_before_checkins = response.json()["data"]
# A new integration claims the same provider account ID.
replacement_id = create_cloud_integration_account(admin_token, provider, config=account_config)["id"]
response = simulate_agent_checkin(signoz, admin_token, provider, replacement_id, provider_account_id, data={"version": "replacement-agent"})
assert response.status_code == HTTPStatus.OK, response.text
response = requests.get(f"{accounts_url}/{replacement_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
replacement_before_checkins = response.json()["data"]
# The old agent keeps retrying. It must receive removal instructions, rather than a 409.
for _ in range(3):
response = simulate_agent_checkin(signoz, admin_token, provider, old_account_id, provider_account_id, data={"version": "old-agent-after-removal"})
assert response.status_code == HTTPStatus.OK, response.text
cleanup_response = response.json()["data"]
assert cleanup_response["cloudIntegrationId"] == old_account_id
assert cleanup_response["providerAccountId"] == provider_account_id
assert cleanup_response["removedAt"] is not None
assert cleanup_response["removedAt"] == old_account_before_checkins["removedAt"]
assert cleanup_response["integrationConfig"] == {}
# Cleanup may update sync state, but must preserve the original heartbeat, data and identity.
response = requests.get(f"{accounts_url}/{old_account_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
removed_account = response.json()["data"]
assert removed_account["providerAccountId"] == old_account_before_checkins["providerAccountId"]
assert removed_account["agentReport"]["timestampMillis"] == old_account_before_checkins["agentReport"]["timestampMillis"]
assert removed_account["agentReport"]["data"] == old_account_before_checkins["agentReport"]["data"]
assert removed_account["agentReport"]["syncState"] == cleanup_response["syncState"]
response = requests.get(f"{accounts_url}/{replacement_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"] == replacement_before_checkins, "old check-ins must not change the replacement account"
response = simulate_agent_checkin(signoz, admin_token, provider, replacement_id, provider_account_id)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"]["removedAt"] is None
@provider_account_reuse
def test_removed_account_first_checkin_after_provider_account_id_is_reused(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
provider: str,
account_config: dict,
provider_account_id: str,
) -> None:
"""Removal before the first heartbeat must still be reported after another integration claims the provider ID."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
headers = {"Authorization": f"Bearer {admin_token}"}
accounts_url = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/{provider}/accounts")
# Remove the original integration before its agent has checked in even once.
old_account_id = create_cloud_integration_account(admin_token, provider, config=account_config)["id"]
response = requests.delete(f"{accounts_url}/{old_account_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
response = requests.get(f"{accounts_url}/{old_account_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
old_account_before_checkins = response.json()["data"]
assert old_account_before_checkins["providerAccountId"] is None
assert old_account_before_checkins["agentReport"] is None
replacement_id = create_cloud_integration_account(admin_token, provider, config=account_config)["id"]
response = simulate_agent_checkin(signoz, admin_token, provider, replacement_id, provider_account_id)
assert response.status_code == HTTPStatus.OK, response.text
response = requests.get(f"{accounts_url}/{replacement_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
replacement_before_checkins = response.json()["data"]
# Late first check-ins must report removal without creating a heartbeat or claiming the ID.
for _ in range(3):
response = simulate_agent_checkin(signoz, admin_token, provider, old_account_id, provider_account_id)
assert response.status_code == HTTPStatus.OK, response.text
cleanup_response = response.json()["data"]
assert cleanup_response["cloudIntegrationId"] == old_account_id
assert cleanup_response["providerAccountId"] == provider_account_id
assert cleanup_response["removedAt"] == old_account_before_checkins["removedAt"]
assert cleanup_response["removedAt"] is not None
assert cleanup_response["integrationConfig"] == {}
response = requests.get(f"{accounts_url}/{old_account_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"] == old_account_before_checkins, "late first check-ins must not change a removed account"
response = requests.get(f"{accounts_url}/{replacement_id}", headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"] == replacement_before_checkins
response = simulate_agent_checkin(signoz, admin_token, provider, replacement_id, provider_account_id)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"]["removedAt"] is None
def test_sync_state_drops_removed_region_after_ack(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
@@ -346,12 +562,13 @@ def test_sync_state_region_removed_during_sync(
}
def test_sync_state_after_disconnect(
def test_sync_state_after_disconnect_when_provider_account_id_is_reused(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""The old AWS agent can finish cleanup while its provider ID belongs to a replacement integration."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account_id = create_cloud_integration_account(admin_token, "aws", regions=["us-east-1", "us-west-2"])["id"]
provider_account_id = str(uuid.uuid4())
@@ -366,6 +583,10 @@ def test_sync_state_after_disconnect(
)
assert response.status_code == HTTPStatus.NO_CONTENT, response.text
replacement_id = create_cloud_integration_account(admin_token, "aws", regions=["us-east-1", "us-west-2"])["id"]
response = simulate_agent_checkin(signoz, admin_token, "aws", replacement_id, provider_account_id)
assert response.status_code == HTTPStatus.OK, response.text
for _ in range(2):
response = simulate_agent_checkin(signoz, admin_token, "aws", account_id, provider_account_id)
assert response.status_code == HTTPStatus.OK, response.text
@@ -380,3 +601,11 @@ def test_sync_state_after_disconnect(
response = simulate_agent_checkin(signoz, admin_token, "aws", account_id, provider_account_id, synced_version=2)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"]["syncState"] == {"version": 2, "inSync": True, "regions": {}}, "acked removal should leave no regions"
response = requests.get(
signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/aws/accounts/{account_id}"),
headers={"Authorization": f"Bearer {admin_token}"},
timeout=10,
)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"]["agentReport"]["syncState"] == {"version": 2, "inSync": True, "regions": {}}, "cleanup acknowledgment must be saved"

View File

@@ -42,12 +42,14 @@ GCP_ACCOUNT_SPEC = ProviderAccountSpec(
},
build_config=lambda p: {
"gcp": {
"setupMode": "manual",
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
}
},
expected_config=lambda p: {
"setupMode": "manual",
"deploymentProjectId": p["deployment_project_id"],
"deploymentRegion": p["deployment_region"],
"projectIds": p["project_ids"],
@@ -293,6 +295,34 @@ def test_update_account_after_checkin_preserves_connected_status(
assert found_after["removedAt"] is None, "removedAt should still be null"
def test_update_gcp_agent_account_rejected(
signoz: types.SigNoz,
create_user_admin: types.Operation, # pylint: disable=unused-argument
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
headers = {"Authorization": f"Bearer {admin_token}"}
project_id = "signoz-agent-update-project"
account_id = create_cloud_integration_account(admin_token, "gcp", config={"gcp": {"setupMode": "agent", "deploymentProjectId": project_id, "deploymentRegion": "us-central1"}})["id"]
account_url = signoz.self.host_configs["8080"].get(f"/api/v1/cloud_integrations/gcp/accounts/{account_id}")
checkin = simulate_agent_checkin(signoz, admin_token, "gcp", account_id, project_id)
assert checkin.status_code == HTTPStatus.OK, checkin.text
response = requests.put(account_url, headers=headers, json={"config": {"gcp": {"deploymentProjectId": project_id, "deploymentRegion": "us-central1", "projectIds": [project_id, "signoz-agent-update-project-2"]}}}, timeout=10)
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
response = requests.get(account_url, headers=headers, timeout=10)
assert response.status_code == HTTPStatus.OK, response.text
assert response.json()["data"]["config"]["gcp"] == {
"setupMode": "agent",
"deploymentProjectId": project_id,
"deploymentRegion": "us-central1",
"projectIds": [project_id],
}, "a rejected update must not change the agent account config"
@provider_spec
def test_disconnect_account(
signoz: types.SigNoz,

View File

@@ -26,9 +26,10 @@ AWS_SERVICE_SPEC = ProviderServiceSpec(
GCP_SERVICE_SPEC = ProviderServiceSpec(
provider="gcp",
service_id="cloudsql_postgres",
supports_logs=False,
supports_logs=True,
account_config={
"gcp": {
"setupMode": "manual",
"deploymentProjectId": "signoz-test-project",
"deploymentRegion": "us-central1",
"projectIds": ["signoz-test-project"],
@@ -337,7 +338,7 @@ def test_update_gcp_service_without_metrics_config(
get_token: Callable[[str, str], str],
create_cloud_integration_account: Callable,
) -> None:
"""GCP services support metrics only, so a config omitting metrics is rejected."""
"""GCP services require a metrics config, so a logs-only config is rejected."""
admin_token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
account = create_cloud_integration_account(admin_token, "gcp", config=GCP_SERVICE_SPEC.account_config)