feat: adding cloud integration type for refactor

pkg/modules/cloudintegration/cloudintegration.go

@@ -0,0 +1,62 @@
+package cloudintegration
+
+import (
+	"context"
+	"net/http"
+
+	"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
+	"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+type Module interface {
+	GetName() cloudintegrationtypes.CloudProviderType
+
+	// AgentCheckIn is called by agent to heartbeat and get latest config in response.
+	AgentCheckIn(ctx context.Context, req *cloudintegrationtypes.PostableAgentCheckInPayload) (any, error)
+
+	GenerateConnectionParams(ctx context.Context) (*cloudintegrationtypes.GettableCloudIntegrationConnectionParams, error)
+	// GenerateConnectionArtifact generates cloud provider specific connection information, client side handles how this information is shown
+	GenerateConnectionArtifact(ctx context.Context, req *cloudintegrationtypes.PostableConnectionArtifact) (any, error)
+	// GetAccountStatus returns agent connection status for a cloud integration account
+	GetAccountStatus(ctx context.Context, orgID, accountID string) (*cloudintegrationtypes.GettableAccountStatus, error)
+	// ListConnectedAccounts lists accounts where agent is connected
+	ListConnectedAccounts(ctx context.Context, orgID string) (*cloudintegrationtypes.GettableConnectedAccountsList, error)
+
+	// LIstServices return list of services for a cloud provider attached with the accountID. This just returns a summary
+	ListServices(ctx context.Context, orgID string, accountID *string) (any, error) // returns either GettableAWSServices or GettableAzureServices
+	// GetServiceDetails returns service definition details for a serviceId. This returns config and other details required to show in service details page on client.
+	GetServiceDetails(ctx context.Context, req *cloudintegrationtypes.GetServiceDetailsReq) (any, error)
+
+	// GetDashboard returns dashboard json for a give cloud integration service dashboard.
+	// this only returns the dashboard when account is connected and service is enabled
+	GetDashboard(ctx context.Context, id string, orgID valuer.UUID) (*dashboardtypes.Dashboard, error)
+	// GetAvailableDashboards returns list of available dashboards across all connected cloud integration accounts in the org.
+	// this list gets added to dashboard list page
+	GetAvailableDashboards(ctx context.Context, orgID valuer.UUID) ([]*dashboardtypes.Dashboard, error)
+
+	// UpdateAccountConfig updates cloud integration account config
+	UpdateAccountConfig(ctx context.Context, orgId valuer.UUID, accountId string, config []byte) (any, error)
+	// UpdateServiceConfig updates cloud integration service config
+	UpdateServiceConfig(ctx context.Context, serviceId string, orgID valuer.UUID, config []byte) (any, error)
+
+	// DisconnectAccount soft deletes/removes a cloud integration account.
+	DisconnectAccount(ctx context.Context, orgID, accountID string) (*cloudintegrationtypes.CloudIntegration, error)
+}
+
+type Handler interface {
+	AgentCheckIn(http.ResponseWriter, *http.Request)
+
+	GenerateConnectionParams(http.ResponseWriter, *http.Request)
+	GenerateConnectionArtifact(http.ResponseWriter, *http.Request)
+
+	ListConnectedAccounts(http.ResponseWriter, *http.Request)
+	GetAccountStatus(http.ResponseWriter, *http.Request)
+	ListServices(http.ResponseWriter, *http.Request)
+	GetServiceDetails(http.ResponseWriter, *http.Request)
+
+	UpdateAccountConfig(http.ResponseWriter, *http.Request)
+	UpdateServiceConfig(http.ResponseWriter, *http.Request)
+
+	DisconnectAccount(http.ResponseWriter, *http.Request)
+}

pkg/types/cloudintegrationtypes/cloudintegration.go

@@ -0,0 +1,643 @@
+// NOTE:
+//   - When Account keyword is used in struct names, it refers cloud integration account. CloudIntegration refers to DB schema.
+//   - When Account Config keyword is used in struct names, it refers to configuration for cloud integration accounts
+//   - When Service keyword is used in struct names, it refers to cloud integration service. CloudIntegrationService refers to DB schema.
+//     where `service` is services provided by each cloud provider like AWS S3, Azure BlobStorage etc.
+//   - When Service Config keyword is used in struct names, it refers to configuration for cloud integration services
+package cloudintegrationtypes
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/uptrace/bun"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+// CloudProviderType type alias
+type CloudProviderType struct{ valuer.String }
+
+var (
+	CloudProviderTypeAWS   = valuer.NewString("aws")
+	CloudProviderTypeAzure = valuer.NewString("azure")
+)
+
+var ErrCodeCloudProviderInvalidInput = errors.MustNewCode("invalid_cloud_provider")
+
+// NewCloudProvider returns a new CloudProviderType from a string. It validates the input and returns an error if the input is not valid.
+func NewCloudProvider(provider string) (CloudProviderType, error) {
+	switch provider {
+	case CloudProviderTypeAWS.String():
+		return CloudProviderType{CloudProviderTypeAWS}, nil
+	case CloudProviderTypeAzure.String():
+		return CloudProviderType{CloudProviderTypeAzure}, nil
+	default:
+		return CloudProviderType{}, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid cloud provider: %s", provider)
+	}
+}
+
+var (
+	AWSIntegrationUserEmail   = valuer.MustNewEmail("aws-integration@signoz.io")
+	AzureIntegrationUserEmail = valuer.MustNewEmail("azure-integration@signoz.io")
+)
+
+// CloudIntegrationUserEmails is the list of valid emails for Cloud One Click integrations.
+// This is used for validation and restrictions in different contexts, across codebase.
+var CloudIntegrationUserEmails = []valuer.Email{
+	AWSIntegrationUserEmail,
+	AzureIntegrationUserEmail,
+}
+
+func IsCloudIntegrationDashboardUuid(dashboardUuid string) bool {
+	parts := strings.SplitN(dashboardUuid, "--", 4)
+	if len(parts) != 4 {
+		return false
+	}
+
+	return parts[0] == "cloud-integration"
+}
+
+// GetCloudIntegrationDashboardID returns the cloud provider from dashboard id, if it's a cloud integration dashboard id.
+// throws an error if invalid format or invalid cloud provider is provided in the dashboard id.
+func GetCloudProviderFromDashboardID(dashboardUuid string) (CloudProviderType, error) {
+	parts := strings.SplitN(dashboardUuid, "--", 4)
+	if len(parts) != 4 {
+		return CloudProviderType{}, errors.NewInvalidInputf(ErrCodeCloudProviderInvalidInput, "invalid dashboard uuid: %s", dashboardUuid)
+	}
+
+	providerStr := parts[1]
+
+	cloudProvider, err := NewCloudProvider(providerStr)
+	if err != nil {
+		return CloudProviderType{}, err
+	}
+
+	return cloudProvider, nil
+}
+
+// Generic utility functions for JSON serialization/deserialization
+// this is helpful to return right errors from a common place and avoid repeating the same code in multiple places.
+// UnmarshalJSON is a generic function to unmarshal JSON data into any type
+func UnmarshalJSON[T any](src []byte, target *T) error {
+	err := json.Unmarshal(src, target)
+	if err != nil {
+		return errors.WrapInternalf(
+			err, errors.CodeInternal, "couldn't deserialize JSON",
+		)
+	}
+	return nil
+}
+
+// MarshalJSON is a generic function to marshal any type to JSON
+func MarshalJSON[T any](source *T) ([]byte, error) {
+	if source == nil {
+		return nil, errors.NewInternalf(errors.CodeInternal, "source is nil")
+	}
+
+	serialized, err := json.Marshal(source)
+	if err != nil {
+		return nil, errors.WrapInternalf(
+			err, errors.CodeInternal, "couldn't serialize to JSON",
+		)
+	}
+	return serialized, nil
+}
+
+// GettableConnectedAccountsList is the response for listing connected accounts for a cloud provider.
+type GettableConnectedAccountsList struct {
+	Accounts []*Account `json:"accounts"`
+}
+
+// SigNozAgentConfig represents parameters required for agent deployment in cloud provider accounts
+// these represent parameters passed during agent deployment, how they are passed might change for each cloud provider but the purpose is same.
+type SigNozAgentConfig struct {
+	Region string `json:"region,omitempty"` // AWS-specific: The region in which SigNoz agent should be installed
+
+	IngestionUrl string `json:"ingestion_url"`
+	IngestionKey string `json:"ingestion_key"`
+	SigNozAPIUrl string `json:"signoz_api_url"`
+	SigNozAPIKey string `json:"signoz_api_key"`
+
+	Version string `json:"version,omitempty"`
+}
+
+// PostableConnectionArtifact represent request body for generating connection artifact API.
+// Data is request body raw bytes since each cloud provider will have have different request body structure and generics hardly help in such cases.
+// Artifact is a generic name for different types of connection methods like connection URL for AWS, connection command for Azure etc.
+type PostableConnectionArtifact struct {
+	OrgID string
+	Data  []byte // either PostableAWSConnectionUrl or PostableAzureConnectionCommand
+}
+
+// PostableAWSConnectionUrl is request body for AWS connection artifact API
+type PostableAWSConnectionUrl struct {
+	AgentConfig   *SigNozAgentConfig `json:"agent_config"`
+	AccountConfig *AWSAccountConfig  `json:"account_config"`
+}
+
+// PostableAzureConnectionCommand is request body for Azure connection artifact API
+type PostableAzureConnectionCommand struct {
+	AgentConfig   *SigNozAgentConfig  `json:"agent_config"`
+	AccountConfig *AzureAccountConfig `json:"account_config"`
+}
+
+// GettableAzureConnectionArtifact is Azure specific connection artifact which contains connection commands for agent deployment
+type GettableAzureConnectionArtifact struct {
+	AzureShellConnectionCommand string `json:"az_shell_connection_command"`
+	AzureCliConnectionCommand   string `json:"az_cli_connection_command"`
+}
+
+// GettableAWSConnectionUrl is AWS specific connection artifact which contains connection url for agent deployment
+type GettableAWSConnectionUrl struct {
+	AccountId     string `json:"account_id"`
+	ConnectionUrl string `json:"connection_url"`
+}
+
+// GettableAzureConnectionCommand is Azure specific connection artifact which contains connection commands for agent deployment
+type GettableAzureConnectionCommand struct {
+	AccountId                   string `json:"account_id"`
+	AzureShellConnectionCommand string `json:"az_shell_connection_command"`
+	AzureCliConnectionCommand   string `json:"az_cli_connection_command"`
+}
+
+// GettableAccountStatus is cloud integration account status response
+type GettableAccountStatus struct {
+	Id             string        `json:"id"`
+	CloudAccountId *string       `json:"cloud_account_id,omitempty"`
+	Status         AccountStatus `json:"status"`
+}
+
+// PostableAgentCheckInPayload is request body for agent check-in API.
+// This is used by agent to send heartbeat.
+type PostableAgentCheckInPayload struct {
+	ID        string `json:"account_id"`
+	AccountID string `json:"cloud_account_id"`
+	// Arbitrary cloud specific Agent data
+	Data  map[string]any `json:"data,omitempty"`
+	OrgID string         `json:"-"`
+}
+
+// AWSAgentIntegrationConfig is used by agent for deploying infra to send telemetry to SigNoz
+type AWSAgentIntegrationConfig struct {
+	EnabledRegions              []string               `json:"enabled_regions"`
+	TelemetryCollectionStrategy *AWSCollectionStrategy `json:"telemetry,omitempty"`
+}
+
+// AzureAgentIntegrationConfig is used by agent for deploying infra to send telemetry to SigNoz
+type AzureAgentIntegrationConfig struct {
+	DeploymentRegion      string   `json:"deployment_region"` // will not be changed once set
+	EnabledResourceGroups []string `json:"resource_groups"`
+	// TelemetryCollectionStrategy is map of service to telemetry config
+	TelemetryCollectionStrategy map[string]*AzureCollectionStrategy `json:"telemetry,omitempty"`
+}
+
+// GettableAgentCheckInRes is generic response from agent check-in API.
+// AWSAgentIntegrationConfig and AzureAgentIntegrationConfig these configs are used by agent to deploy the infra and send telemetry to SigNoz
+type GettableAgentCheckInRes[AgentConfigT any] struct {
+	AccountId         string       `json:"account_id"`
+	CloudAccountId    string       `json:"cloud_account_id"`
+	RemovedAt         *time.Time   `json:"removed_at"`
+	IntegrationConfig AgentConfigT `json:"integration_config"`
+}
+
+// UpdatableServiceConfig is generic
+type UpdatableServiceConfig[ServiceConfigT any] struct {
+	CloudAccountId string         `json:"cloud_account_id"`
+	Config         ServiceConfigT `json:"config"`
+}
+
+// ServiceConfigTyped is a generic interface for cloud integration service's configuration
+// this is generic interface to define helper functions for CloudIntegrationService.Config field.
+type ServiceConfigTyped[definition Definition] interface {
+	Validate(def definition) error
+	IsMetricsEnabled() bool
+	IsLogsEnabled() bool
+}
+
+type AWSServiceConfig struct {
+	Logs    *AWSServiceLogsConfig    `json:"logs,omitempty"`
+	Metrics *AWSServiceMetricsConfig `json:"metrics,omitempty"`
+}
+
+type AWSServiceLogsConfig struct {
+	Enabled   bool                `json:"enabled"`
+	S3Buckets map[string][]string `json:"s3_buckets,omitempty"`
+}
+
+type AWSServiceMetricsConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+// IsMetricsEnabled returns true if metrics collection is configured and enabled
+func (a *AWSServiceConfig) IsMetricsEnabled() bool {
+	return a.Metrics != nil && a.Metrics.Enabled
+}
+
+// IsLogsEnabled returns true if logs collection is configured and enabled
+func (a *AWSServiceConfig) IsLogsEnabled() bool {
+	return a.Logs != nil && a.Logs.Enabled
+}
+
+type AzureServiceConfig struct {
+	Logs    []*AzureServiceLogsConfig    `json:"logs,omitempty"`
+	Metrics []*AzureServiceMetricsConfig `json:"metrics,omitempty"`
+}
+
+// AzureServiceLogsConfig is Azure specific service config for logs
+type AzureServiceLogsConfig struct {
+	Enabled bool   `json:"enabled"`
+	Name    string `json:"name"`
+}
+
+// AzureServiceMetricsConfig is Azure specific service config for metrics
+type AzureServiceMetricsConfig struct {
+	Enabled bool   `json:"enabled"`
+	Name    string `json:"name"`
+}
+
+// IsMetricsEnabled returns true if any metric is configured and enabled
+func (a *AzureServiceConfig) IsMetricsEnabled() bool {
+	if a.Metrics == nil {
+		return false
+	}
+	for _, m := range a.Metrics {
+		if m.Enabled {
+			return true
+		}
+	}
+	return false
+}
+
+// IsLogsEnabled returns true if any log is configured and enabled
+func (a *AzureServiceConfig) IsLogsEnabled() bool {
+	if a.Logs == nil {
+		return false
+	}
+	for _, l := range a.Logs {
+		if l.Enabled {
+			return true
+		}
+	}
+	return false
+}
+
+func (a *AWSServiceConfig) Validate(def *AWSDefinition) error {
+	if def.Id != S3Sync.String() && a.Logs != nil && a.Logs.S3Buckets != nil {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput, "s3 buckets can only be added to service-type[%s]", S3Sync)
+	} else if def.Id == S3Sync.String() && a.Logs != nil && a.Logs.S3Buckets != nil {
+		for region := range a.Logs.S3Buckets {
+			if _, found := ValidAWSRegions[region]; !found {
+				return errors.NewInvalidInputf(CodeInvalidCloudRegion, "invalid cloud region: %s", region)
+			}
+		}
+	}
+
+	return nil
+}
+
+func (a *AzureServiceConfig) Validate(def *AzureDefinition) error {
+	logsMap := make(map[string]bool)
+	metricsMap := make(map[string]bool)
+
+	if def.Strategy != nil && def.Strategy.Logs != nil {
+		for _, log := range def.Strategy.Logs {
+			logsMap[log.Name] = true
+		}
+	}
+
+	if def.Strategy != nil && def.Strategy.Metrics != nil {
+		for _, metric := range def.Strategy.Metrics {
+			metricsMap[metric.Name] = true
+		}
+	}
+
+	for _, log := range a.Logs {
+		if _, found := logsMap[log.Name]; !found {
+			return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid log name: %s", log.Name)
+		}
+	}
+
+	for _, metric := range a.Metrics {
+		if _, found := metricsMap[metric.Name]; !found {
+			return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid metric name: %s", metric.Name)
+		}
+	}
+
+	return nil
+}
+
+// UpdatableServiceConfigRes is response for UpdateServiceConfig API
+// TODO: find a better way to name this
+type UpdatableServiceConfigRes struct {
+	ServiceId string `json:"id"`
+	Config    any    `json:"config"`
+}
+
+// UpdatableAccountConfigTyped is a generic struct for updating cloud integration account config used in UpdateAccountConfig API
+type UpdatableAccountConfigTyped[AccountConfigT any] struct {
+	Config *AccountConfigT `json:"config"`
+}
+
+type (
+	UpdatableAWSAccountConfig   = UpdatableAccountConfigTyped[AWSAccountConfig]
+	UpdatableAzureAccountConfig = UpdatableAccountConfigTyped[AzureAccountConfig]
+)
+
+// AWSAccountConfig is the configuration for AWS cloud integration account
+type AWSAccountConfig struct {
+	EnabledRegions []string `json:"regions"`
+}
+
+// AzureAccountConfig is the configuration for Azure cloud integration account
+type AzureAccountConfig struct {
+	DeploymentRegion      string   `json:"deployment_region,omitempty"`
+	EnabledResourceGroups []string `json:"resource_groups,omitempty"`
+}
+
+// GettableServices is a generic struct for listing services of a cloud integration account used in ListServices API
+type GettableServices[ServiceSummaryT any] struct {
+	Services []ServiceSummaryT `json:"services"`
+}
+
+type (
+	GettableAWSServices   = GettableServices[AWSServiceSummary]
+	GettableAzureServices = GettableServices[AzureServiceSummary]
+)
+
+// GetServiceDetailsReq is a req struct for getting service definition details
+type GetServiceDetailsReq struct {
+	OrgID          valuer.UUID
+	ServiceId      string
+	CloudAccountID *string
+}
+
+// ServiceSummary is a generic struct for service summary used in ListServices API
+type ServiceSummary[ServiceConfigT any] struct {
+	DefinitionMetadata
+	Config *ServiceConfigT `json:"config"`
+}
+
+type (
+	AWSServiceSummary   = ServiceSummary[AWSServiceConfig]
+	AzureServiceSummary = ServiceSummary[AzureServiceConfig]
+)
+
+// GettableServiceDetails is a generic struct for service details used in GetServiceDetails API
+type GettableServiceDetails[DefinitionT any, ServiceConfigT any] struct {
+	Definition       DefinitionT              `json:",inline"`
+	Config           ServiceConfigT           `json:"config"`
+	ConnectionStatus *ServiceConnectionStatus `json:"status,omitempty"`
+}
+
+type (
+	GettableAWSServiceDetails   = GettableServiceDetails[AWSDefinition, *AWSServiceConfig]
+	GettableAzureServiceDetails = GettableServiceDetails[AzureDefinition, *AzureServiceConfig]
+)
+
+// ServiceConnectionStatus represents integration connection status for a particular service
+// this struct helps to check ingested data and determines connection status by whether data was ingested or not.
+// this is composite struct for both metrics and logs
+type ServiceConnectionStatus struct {
+	Logs    []*SignalConnectionStatus `json:"logs"`
+	Metrics []*SignalConnectionStatus `json:"metrics"`
+}
+
+// SignalConnectionStatus represents connection status for a particular signal type (logs or metrics) for a service
+// this struct is used in API responses for clients to show relevant information about the connection status.
+type SignalConnectionStatus struct {
+	CategoryID           string `json:"category"`
+	CategoryDisplayName  string `json:"category_display_name"`
+	LastReceivedTsMillis int64  `json:"last_received_ts_ms"` // epoch milliseconds
+	LastReceivedFrom     string `json:"last_received_from"`  // resource identifier
+}
+
+// GettableCloudIntegrationConnectionParams is response for connection params API
+type GettableCloudIntegrationConnectionParams struct {
+	IngestionUrl string `json:"ingestion_url,omitempty"`
+	IngestionKey string `json:"ingestion_key,omitempty"`
+	SigNozAPIUrl string `json:"signoz_api_url,omitempty"`
+	SigNozAPIKey string `json:"signoz_api_key,omitempty"`
+}
+
+// GettableIngestionKey is a struct for ingestion key returned from gateway
+type GettableIngestionKey struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+	// other attributes from gateway response not included here since they are not being used.
+}
+
+// GettableIngestionKeysSearch is a struct for response of ingestion keys search API on gateway
+type GettableIngestionKeysSearch struct {
+	Status string                 `json:"status"`
+	Data   []GettableIngestionKey `json:"data"`
+	Error  string                 `json:"error"`
+}
+
+// GettableCreateIngestionKey is a struct for response of create ingestion key API on gateway
+type GettableCreateIngestionKey struct {
+	Status string               `json:"status"`
+	Data   GettableIngestionKey `json:"data"`
+	Error  string               `json:"error"`
+}
+
+// GettableDeployment is response struct for deployment details fetched from Zeus
+type GettableDeployment struct {
+	Name        string `json:"name"`
+	ClusterInfo struct {
+		Region struct {
+			DNS string `json:"dns"`
+		} `json:"region"`
+	} `json:"cluster"`
+}
+
+// --------------------------------------------------------------------------
+// Cloud integration uses the cloud_integration table
+// and cloud_integrations_service table
+// --------------------------------------------------------------------------
+
+type CloudIntegration struct {
+	bun.BaseModel `bun:"table:cloud_integration"`
+
+	types.Identifiable
+	types.TimeAuditable
+	Provider        string         `json:"provider" bun:"provider,type:text,unique:provider_id"`
+	Config          *AccountConfig `json:"config" bun:"config,type:text"`
+	AccountID       *string        `json:"account_id" bun:"account_id,type:text"`
+	LastAgentReport *AgentReport   `json:"last_agent_report" bun:"last_agent_report,type:text"`
+	RemovedAt       *time.Time     `json:"removed_at" bun:"removed_at,type:timestamp,nullzero"`
+	OrgID           string         `bun:"org_id,type:text,unique:provider_id"`
+}
+
+func (a *CloudIntegration) Status() AccountStatus {
+	status := AccountStatus{}
+	if a.LastAgentReport != nil {
+		lastHeartbeat := a.LastAgentReport.TimestampMillis
+		status.Integration.LastHeartbeatTsMillis = &lastHeartbeat
+	}
+	return status
+}
+
+func (a *CloudIntegration) Account() Account {
+	ca := Account{Id: a.ID.StringValue(), Status: a.Status()}
+
+	if a.AccountID != nil {
+		ca.CloudAccountId = *a.AccountID
+	}
+
+	if a.Config != nil {
+		ca.Config = *a.Config
+	} else {
+		ca.Config = DefaultAccountConfig()
+	}
+	return ca
+}
+
+type Account struct {
+	Id             string        `json:"id"`
+	CloudAccountId string        `json:"cloud_account_id"`
+	Config         AccountConfig `json:"config"`
+	Status         AccountStatus `json:"status"`
+}
+
+type AccountStatus struct {
+	Integration AccountIntegrationStatus `json:"integration"`
+}
+
+type AccountIntegrationStatus struct {
+	LastHeartbeatTsMillis *int64 `json:"last_heartbeat_ts_ms"`
+}
+
+func DefaultAccountConfig() AccountConfig {
+	return AccountConfig{
+		EnabledRegions: []string{},
+	}
+}
+
+type AccountConfig struct {
+	EnabledRegions []string `json:"regions"`
+}
+
+// For serializing from db
+func (c *AccountConfig) Scan(src any) error {
+	var data []byte
+	switch v := src.(type) {
+	case []byte:
+		data = v
+	case string:
+		data = []byte(v)
+	default:
+		return errors.NewInternalf(errors.CodeInternal, "tried to scan from %T instead of string or bytes", src)
+	}
+
+	return json.Unmarshal(data, c)
+}
+
+// For serializing to db
+func (c *AccountConfig) Value() (driver.Value, error) {
+	if c == nil {
+		return nil, errors.NewInternalf(errors.CodeInternal, "cloud account config is nil")
+	}
+
+	serialized, err := json.Marshal(c)
+	if err != nil {
+		return nil, errors.WrapInternalf(err, errors.CodeInternal, "couldn't serialize cloud account config to JSON")
+	}
+	// Return as string instead of []byte to ensure PostgreSQL stores as text, not bytea
+	return string(serialized), nil
+}
+
+type AgentReport struct {
+	TimestampMillis int64          `json:"timestamp_millis"`
+	Data            map[string]any `json:"data"`
+}
+
+// For serializing from db
+func (r *AgentReport) Scan(src any) error {
+	var data []byte
+	switch v := src.(type) {
+	case []byte:
+		data = v
+	case string:
+		data = []byte(v)
+	default:
+		return errors.NewInternalf(errors.CodeInternal, "tried to scan from %T instead of string or bytes", src)
+	}
+
+	return json.Unmarshal(data, r)
+}
+
+// For serializing to db
+func (r *AgentReport) Value() (driver.Value, error) {
+	if r == nil {
+		return nil, errors.NewInternalf(errors.CodeInternal, "agent report is nil")
+	}
+
+	serialized, err := json.Marshal(r)
+	if err != nil {
+		return nil, errors.WrapInternalf(
+			err, errors.CodeInternal, "couldn't serialize agent report to JSON",
+		)
+	}
+	// Return as string instead of []byte to ensure PostgreSQL stores as text, not bytea
+	return string(serialized), nil
+}
+
+type CloudIntegrationService struct {
+	bun.BaseModel `bun:"table:cloud_integration_service,alias:cis"`
+
+	types.Identifiable
+	types.TimeAuditable
+	Type               string             `bun:"type,type:text,notnull,unique:cloud_integration_id_type"`
+	Config             CloudServiceConfig `bun:"config,type:text"`
+	CloudIntegrationID string             `bun:"cloud_integration_id,type:text,notnull,unique:cloud_integration_id_type,references:cloud_integrations(id),on_delete:cascade"`
+}
+
+type CloudServiceLogsConfig struct {
+	Enabled   bool                `json:"enabled"`
+	S3Buckets map[string][]string `json:"s3_buckets,omitempty"`
+}
+
+type CloudServiceMetricsConfig struct {
+	Enabled bool `json:"enabled"`
+}
+
+type CloudServiceConfig struct {
+	Logs    *CloudServiceLogsConfig    `json:"logs,omitempty"`
+	Metrics *CloudServiceMetricsConfig `json:"metrics,omitempty"`
+}
+
+// For serializing from db
+func (c *CloudServiceConfig) Scan(src any) error {
+	var data []byte
+	switch src := src.(type) {
+	case []byte:
+		data = src
+	case string:
+		data = []byte(src)
+	default:
+		return errors.NewInternalf(errors.CodeInternal, "tried to scan from %T instead of string or bytes", src)
+	}
+
+	return json.Unmarshal(data, c)
+}
+
+// For serializing to db
+func (c *CloudServiceConfig) Value() (driver.Value, error) {
+	if c == nil {
+		return nil, errors.NewInternalf(errors.CodeInternal, "cloud service config is nil")
+	}
+
+	serialized, err := json.Marshal(c)
+	if err != nil {
+		return nil, errors.WrapInternalf(
+			err, errors.CodeInternal, "couldn't serialize cloud service config to JSON",
+		)
+	}
+	// Return as string instead of []byte to ensure PostgreSQL stores as text, not bytea
+	return string(serialized), nil
+}

pkg/types/cloudintegrationtypes/regions.go

@@ -0,0 +1,103 @@
+package cloudintegrationtypes
+
+import (
+	"github.com/SigNoz/signoz/pkg/errors"
+)
+
+var (
+	CodeInvalidCloudRegion    = errors.MustNewCode("invalid_cloud_region")
+	CodeMismatchCloudProvider = errors.MustNewCode("cloud_provider_mismatch")
+)
+
+// List of all valid cloud regions on Amazon Web Services
+var ValidAWSRegions = map[string]bool{
+	"af-south-1":     true, // Africa (Cape Town).
+	"ap-east-1":      true, // Asia Pacific (Hong Kong).
+	"ap-northeast-1": true, // Asia Pacific (Tokyo).
+	"ap-northeast-2": true, // Asia Pacific (Seoul).
+	"ap-northeast-3": true, // Asia Pacific (Osaka).
+	"ap-south-1":     true, // Asia Pacific (Mumbai).
+	"ap-south-2":     true, // Asia Pacific (Hyderabad).
+	"ap-southeast-1": true, // Asia Pacific (Singapore).
+	"ap-southeast-2": true, // Asia Pacific (Sydney).
+	"ap-southeast-3": true, // Asia Pacific (Jakarta).
+	"ap-southeast-4": true, // Asia Pacific (Melbourne).
+	"ca-central-1":   true, // Canada (Central).
+	"ca-west-1":      true, // Canada West (Calgary).
+	"eu-central-1":   true, // Europe (Frankfurt).
+	"eu-central-2":   true, // Europe (Zurich).
+	"eu-north-1":     true, // Europe (Stockholm).
+	"eu-south-1":     true, // Europe (Milan).
+	"eu-south-2":     true, // Europe (Spain).
+	"eu-west-1":      true, // Europe (Ireland).
+	"eu-west-2":      true, // Europe (London).
+	"eu-west-3":      true, // Europe (Paris).
+	"il-central-1":   true, // Israel (Tel Aviv).
+	"me-central-1":   true, // Middle East (UAE).
+	"me-south-1":     true, // Middle East (Bahrain).
+	"sa-east-1":      true, // South America (Sao Paulo).
+	"us-east-1":      true, // US East (N. Virginia).
+	"us-east-2":      true, // US East (Ohio).
+	"us-west-1":      true, // US West (N. California).
+	"us-west-2":      true, // US West (Oregon).
+}
+
+// List of all valid cloud regions for Microsoft Azure
+var ValidAzureRegions = map[string]bool{
+	"australiacentral":   true, // Australia Central
+	"australiacentral2":  true, // Australia Central 2
+	"australiaeast":      true, // Australia East
+	"australiasoutheast": true, // Australia Southeast
+	"austriaeast":        true, // Austria East
+	"belgiumcentral":     true, // Belgium Central
+	"brazilsouth":        true, // Brazil South
+	"brazilsoutheast":    true, // Brazil Southeast
+	"canadacentral":      true, // Canada Central
+	"canadaeast":         true, // Canada East
+	"centralindia":       true, // Central India
+	"centralus":          true, // Central US
+	"chilecentral":       true, // Chile Central
+	"denmarkeast":        true, // Denmark East
+	"eastasia":           true, // East Asia
+	"eastus":             true, // East US
+	"eastus2":            true, // East US 2
+	"francecentral":      true, // France Central
+	"francesouth":        true, // France South
+	"germanynorth":       true, // Germany North
+	"germanywestcentral": true, // Germany West Central
+	"indonesiacentral":   true, // Indonesia Central
+	"israelcentral":      true, // Israel Central
+	"italynorth":         true, // Italy North
+	"japaneast":          true, // Japan East
+	"japanwest":          true, // Japan West
+	"koreacentral":       true, // Korea Central
+	"koreasouth":         true, // Korea South
+	"malaysiawest":       true, // Malaysia West
+	"mexicocentral":      true, // Mexico Central
+	"newzealandnorth":    true, // New Zealand North
+	"northcentralus":     true, // North Central US
+	"northeurope":        true, // North Europe
+	"norwayeast":         true, // Norway East
+	"norwaywest":         true, // Norway West
+	"polandcentral":      true, // Poland Central
+	"qatarcentral":       true, // Qatar Central
+	"southafricanorth":   true, // South Africa North
+	"southafricawest":    true, // South Africa West
+	"southcentralus":     true, // South Central US
+	"southindia":         true, // South India
+	"southeastasia":      true, // Southeast Asia
+	"spaincentral":       true, // Spain Central
+	"swedencentral":      true, // Sweden Central
+	"switzerlandnorth":   true, // Switzerland North
+	"switzerlandwest":    true, // Switzerland West
+	"uaecentral":         true, // UAE Central
+	"uaenorth":           true, // UAE North
+	"uksouth":            true, // UK South
+	"ukwest":             true, // UK West
+	"westcentralus":      true, // West Central US
+	"westeurope":         true, // West Europe
+	"westindia":          true, // West India
+	"westus":             true, // West US
+	"westus2":            true, // West US 2
+	"westus3":            true, // West US 3
+}

pkg/types/cloudintegrationtypes/servicedefinitions.go

@@ -0,0 +1,263 @@
+package cloudintegrationtypes
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/types"
+	"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+var S3Sync = valuer.NewString("s3sync")
+
+// Generic interface for cloud service definition.
+// This is implemented by AWSDefinition and AzureDefinition, which represent service definitions for AWS and Azure respectively.
+// Generics work well so far because service definitions share a similar logic.
+// We dont want to over-do generics as well, if the service definitions functionally diverge in the future consider breaking generics.
+type Definition interface {
+	GetId() string
+	Validate() error
+	PopulateDashboardURLs(cloudProvider CloudProviderType, svcId string)
+	GetIngestionStatusCheck() *IngestionStatusCheck
+	GetAssets() Assets
+}
+
+// AWSDefinition represents AWS Service definition, which includes collection strategy, dashboards and meta info for integration
+type AWSDefinition = ServiceDefinition[AWSCollectionStrategy]
+
+// AzureDefinition represents Azure Service definition, which includes collection strategy, dashboards and meta info for integration
+type AzureDefinition = ServiceDefinition[AzureCollectionStrategy]
+
+// Making AWSDefinition and AzureDefinition satisfy Definition interface, so that they can be used in a generic way
+var (
+	_ Definition = &AWSDefinition{}
+	_ Definition = &AzureDefinition{}
+)
+
+// ServiceDefinition represents generic struct for cloud service, regardless of the cloud provider.
+// this struct must satify Definition interface.
+// StrategyT is of either AWSCollectionStrategy or AzureCollectionStrategy, depending on the cloud provider.
+type ServiceDefinition[StrategyT any] struct {
+	DefinitionMetadata
+	Overview             string                `json:"overview"` // markdown
+	Assets               Assets                `json:"assets"`
+	SupportedSignals     SupportedSignals      `json:"supported_signals"`
+	DataCollected        DataCollected         `json:"data_collected"`
+	IngestionStatusCheck *IngestionStatusCheck `json:"ingestion_status_check,omitempty"`
+	Strategy             *StrategyT            `json:"telemetry_collection_strategy"`
+}
+
+// Following methods are quite self explanatory, they are just to satisfy the Definition interface and provide some utility functions for service definitions.
+func (def *ServiceDefinition[StrategyT]) GetId() string {
+	return def.Id
+}
+
+func (def *ServiceDefinition[StrategyT]) Validate() error {
+	seenDashboardIds := map[string]interface{}{}
+
+	if def.Strategy == nil {
+		return errors.NewInternalf(errors.CodeInternal, "telemetry_collection_strategy is required")
+	}
+
+	for _, dd := range def.Assets.Dashboards {
+		if _, seen := seenDashboardIds[dd.Id]; seen {
+			return errors.NewInternalf(errors.CodeInternal, "multiple dashboards found with id %s", dd.Id)
+		}
+		seenDashboardIds[dd.Id] = nil
+	}
+
+	return nil
+}
+
+func (def *ServiceDefinition[StrategyT]) PopulateDashboardURLs(cloudProvider CloudProviderType, svcId string) {
+	for i := range def.Assets.Dashboards {
+		dashboardId := def.Assets.Dashboards[i].Id
+		url := "/dashboard/" + GetCloudIntegrationDashboardID(cloudProvider, svcId, dashboardId)
+		def.Assets.Dashboards[i].Url = url
+	}
+}
+
+func (def *ServiceDefinition[StrategyT]) GetIngestionStatusCheck() *IngestionStatusCheck {
+	return def.IngestionStatusCheck
+}
+
+func (def *ServiceDefinition[StrategyT]) GetAssets() Assets {
+	return def.Assets
+}
+
+// DefinitionMetadata represents service definition metadata. This is useful for showing service overview
+type DefinitionMetadata struct {
+	Id    string `json:"id"`
+	Title string `json:"title"`
+	Icon  string `json:"icon"`
+}
+
+// IngestionStatusCheckCategory represents a category of ingestion status check. Applies for both metrics and logs.
+// A category can be "Overview" of metrics or "Enhanced" Metrics for AWS, and "Transaction" or "Capacity" metrics for Azure.
+// Each category can have multiple checks (AND logic), if all checks pass,
+// then we can be sure that data is being ingested for that category of the signal
+type IngestionStatusCheckCategory struct {
+	Category    string                           `json:"category"`
+	DisplayName string                           `json:"display_name"`
+	Checks      []*IngestionStatusCheckAttribute `json:"checks"`
+}
+
+// IngestionStatusCheckAttribute represents a check or condition for ingestion status.
+// Key can be metric name or part of log message
+type IngestionStatusCheckAttribute struct {
+	Key        string                                 `json:"key"` // OPTIONAL search key (metric name or log message)
+	Attributes []*IngestionStatusCheckAttributeFilter `json:"attributes"`
+}
+
+// IngestionStatusCheck represents combined checks for metrics and logs for a service
+type IngestionStatusCheck struct {
+	Metrics []*IngestionStatusCheckCategory `json:"metrics"`
+	Logs    []*IngestionStatusCheckCategory `json:"logs"`
+}
+
+// IngestionStatusCheckAttributeFilter represents filter for a check, which can be used to filter specific log messages or metrics with specific attributes.
+// For example, we can use it to filter logs with specific log level or metrics with specific dimensions.
+type IngestionStatusCheckAttributeFilter struct {
+	Name     string `json:"name"`
+	Operator string `json:"operator"`
+	Value    string `json:"value"` // OPTIONAL
+}
+
+// Assets represents the collection of dashboards
+type Assets struct {
+	Dashboards []Dashboard `json:"dashboards"`
+}
+
+// SupportedSignals for cloud provider's service
+type SupportedSignals struct {
+	Logs    bool `json:"logs"`
+	Metrics bool `json:"metrics"`
+}
+
+// DataCollected is curated static list of metrics and logs, this is shown as part of service overview
+type DataCollected struct {
+	Logs    []CollectedLogAttribute `json:"logs"`
+	Metrics []CollectedMetric       `json:"metrics"`
+}
+
+// CollectedLogAttribute represents a log attribute that is present in all log entries for a service,
+// this is shown as part of service overview
+type CollectedLogAttribute struct {
+	Name string `json:"name"`
+	Path string `json:"path"`
+	Type string `json:"type"`
+}
+
+// CollectedMetric represents a metric that is collected for a service, this is shown as part of service overview
+type CollectedMetric struct {
+	Name        string `json:"name"`
+	Type        string `json:"type"`
+	Unit        string `json:"unit"`
+	Description string `json:"description"`
+}
+
+// AWSCollectionStrategy represents signal collection strategy for AWS services.
+// this is AWS specific.
+type AWSCollectionStrategy struct {
+	Metrics   *AWSMetricsStrategy `json:"aws_metrics,omitempty"`
+	Logs      *AWSLogsStrategy    `json:"aws_logs,omitempty"`
+	S3Buckets map[string][]string `json:"s3_buckets,omitempty"` // Only available in S3 Sync Service Type in AWS
+}
+
+// AzureCollectionStrategy represents signal collection strategy for Azure services.
+// this is Azure specific.
+type AzureCollectionStrategy struct {
+	Metrics []*AzureMetricsStrategy `json:"azure_metrics,omitempty"`
+	Logs    []*AzureLogsStrategy    `json:"azure_logs,omitempty"`
+}
+
+// AWSMetricsStrategy represents metrics collection strategy for AWS services.
+// this is AWS specific.
+type AWSMetricsStrategy struct {
+	// to be used as https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-cloudwatch-metricstream.html#cfn-cloudwatch-metricstream-includefilters
+	StreamFilters []struct {
+		// json tags here are in the shape expected by AWS API as detailed at
+		// https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-cloudwatch-metricstream-metricstreamfilter.html
+		Namespace   string   `json:"Namespace"`
+		MetricNames []string `json:"MetricNames,omitempty"`
+	} `json:"cloudwatch_metric_stream_filters"`
+}
+
+// AWSLogsStrategy represents logs collection strategy for AWS services.
+// this is AWS specific.
+type AWSLogsStrategy struct {
+	Subscriptions []struct {
+		// subscribe to all logs groups with specified prefix.
+		// eg: `/aws/rds/`
+		LogGroupNamePrefix string `json:"log_group_name_prefix"`
+
+		// https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html
+		// "" implies no filtering is required.
+		FilterPattern string `json:"filter_pattern"`
+	} `json:"cloudwatch_logs_subscriptions"`
+}
+
+// AzureMetricsStrategy represents metrics collection strategy for Azure services.
+// this is Azure specific.
+type AzureMetricsStrategy struct {
+	CategoryType string `json:"category_type"`
+	Name         string `json:"name"`
+}
+
+// AzureLogsStrategy represents logs collection strategy for Azure services.
+// this is Azure specific. Even though this is similar to AzureMetricsStrategy, keeping it separate for future flexibility and clarity.
+type AzureLogsStrategy struct {
+	CategoryType string `json:"category_type"`
+	Name         string `json:"name"`
+}
+
+// Dashboard represents a dashboard definition for cloud integration.
+type Dashboard struct {
+	Id          string                                `json:"id"`
+	Url         string                                `json:"url"`
+	Title       string                                `json:"title"`
+	Description string                                `json:"description"`
+	Image       string                                `json:"image"`
+	Definition  *dashboardtypes.StorableDashboardData `json:"definition,omitempty"`
+}
+
+// UTILS
+
+// GetCloudIntegrationDashboardID returns the dashboard id for a cloud integration, given the cloud provider, service id, and dashboard id.
+// This is used to generate unique dashboard ids for cloud integration, and also to parse the dashboard id to get the cloud provider and service id when needed.
+func GetCloudIntegrationDashboardID(cloudProvider CloudProviderType, svcId, dashboardId string) string {
+	return fmt.Sprintf("cloud-integration--%s--%s--%s", cloudProvider, svcId, dashboardId)
+}
+
+// GetDashboardsFromAssets returns the list of dashboards for the cloud provider service from definition
+func GetDashboardsFromAssets(
+	svcId string,
+	orgID valuer.UUID,
+	cloudProvider CloudProviderType,
+	createdAt *time.Time,
+	assets Assets,
+) []*dashboardtypes.Dashboard {
+	dashboards := make([]*dashboardtypes.Dashboard, 0)
+
+	for _, d := range assets.Dashboards {
+		author := fmt.Sprintf("%s-integration", cloudProvider)
+		dashboards = append(dashboards, &dashboardtypes.Dashboard{
+			ID:     GetCloudIntegrationDashboardID(cloudProvider, svcId, d.Id),
+			Locked: true,
+			OrgID:  orgID,
+			Data:   *d.Definition,
+			TimeAuditable: types.TimeAuditable{
+				CreatedAt: *createdAt,
+				UpdatedAt: *createdAt,
+			},
+			UserAuditable: types.UserAuditable{
+				CreatedBy: author,
+				UpdatedBy: author,
+			},
+		})
+	}
+
+	return dashboards
+}

pkg/types/cloudintegrationtypes/store.go

@@ -0,0 +1,57 @@
+package cloudintegrationtypes
+
+import (
+	"context"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/query-service/model"
+	"github.com/SigNoz/signoz/pkg/types"
+)
+
+type CloudIntegrationAccountStore interface {
+	ListConnected(ctx context.Context, orgId string, provider string) ([]types.CloudIntegration, *model.ApiError)
+
+	Get(ctx context.Context, orgId string, provider string, id string) (*types.CloudIntegration, *model.ApiError)
+
+	GetConnectedCloudAccount(ctx context.Context, orgId string, provider string, accountID string) (*types.CloudIntegration, *model.ApiError)
+
+	// Insert an account or update it by (cloudProvider, id)
+	// for specified non-empty fields
+	Upsert(
+		ctx context.Context,
+		orgId string,
+		provider string,
+		id *string,
+		config *types.AccountConfig,
+		accountId *string,
+		agentReport *types.AgentReport,
+		removedAt *time.Time,
+	) (*types.CloudIntegration, *model.ApiError)
+}
+
+type CloudIntegrationServiceStore interface {
+	Get(
+		ctx context.Context,
+		orgID string,
+		cloudAccountId string,
+		serviceType string,
+	) (*types.CloudServiceConfig, *model.ApiError)
+
+	Upsert(
+		ctx context.Context,
+		orgID string,
+		cloudProvider string,
+		cloudAccountId string,
+		serviceId string,
+		config types.CloudServiceConfig,
+	) (*types.CloudServiceConfig, *model.ApiError)
+
+	GetAllForAccount(
+		ctx context.Context,
+		orgID string,
+		cloudAccountId string,
+	) (
+		configsBySvcId map[string]*types.CloudServiceConfig,
+		apiErr *model.ApiError,
+	)
+}