put back json

* chore: add tests for trace waterfall

* chore: remove unhelpful tests

.prettierrc

@@ -0,0 +1,3 @@
+{
+  "trailingComma": "none"
+}

.vscode/settings.json

@@ -1,23 +1,25 @@
 {
-	"eslint.workingDirectories": [
-		"./frontend"
-	],
-	"editor.formatOnSave": true,
-	"editor.defaultFormatter": "esbenp.prettier-vscode",
-	"editor.codeActionsOnSave": {
-		"source.fixAll.eslint": "explicit"
-	},
-	"prettier.requireConfig": true,
-	"[go]": {
-		"editor.formatOnSave": true,
-		"editor.defaultFormatter": "golang.go"
-	},
-	"[sql]": {
-		"editor.defaultFormatter": "adpyke.vscode-sql-formatter"
-	},
-	"[html]": {
-		"editor.defaultFormatter": "vscode.html-language-features"
-	},
-	"python-envs.defaultEnvManager": "ms-python.python:system",
-	"python-envs.pythonProjects": []
+  "eslint.workingDirectories": ["./frontend"],
+  "editor.formatOnSave": true,
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "editor.codeActionsOnSave": {
+    "source.fixAll.eslint": "explicit"
+  },
+  "prettier.requireConfig": true,
+  "[go]": {
+    "editor.formatOnSave": true,
+    "editor.defaultFormatter": "golang.go"
+  },
+  "[sql]": {
+    "editor.defaultFormatter": "adpyke.vscode-sql-formatter"
+  },
+  "[html]": {
+    "editor.defaultFormatter": "vscode.html-language-features"
+  },
+  "python-envs.defaultEnvManager": "ms-python.python:system",
+  "python-envs.pythonProjects": [],
+  "editor.tokenColorCustomizations": {
+    "comments": "",
+    "textMateRules": []
+  }
 }

conf/example.yaml

@@ -144,6 +144,8 @@ telemetrystore:
 
 ##################### Prometheus #####################
 prometheus:
+  # The maximum time a PromQL query is allowed to run before being aborted.
+  timeout: 2m
   active_query_tracker:
     # Whether to enable the active query tracker.
     enabled: true

deploy/docker-swarm/docker-compose.ha.yaml

@@ -190,7 +190,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:v0.116.1
+    image: signoz/signoz:v0.117.0
     ports:
       - "8080:8080" # signoz port
     #   - "6060:6060"     # pprof port

deploy/docker-swarm/docker-compose.yaml

@@ -117,7 +117,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:v0.116.1
+    image: signoz/signoz:v0.117.0
     ports:
       - "8080:8080" # signoz port
     volumes:

deploy/docker/docker-compose.ha.yaml

@@ -181,7 +181,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:${VERSION:-v0.116.1}
+    image: signoz/signoz:${VERSION:-v0.117.0}
     container_name: signoz
     ports:
       - "8080:8080" # signoz port

deploy/docker/docker-compose.yaml

@@ -109,7 +109,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:${VERSION:-v0.116.1}
+    image: signoz/signoz:${VERSION:-v0.117.0}
     container_name: signoz
     ports:
       - "8080:8080" # signoz port

docs/api/openapi.yml

@@ -342,6 +342,409 @@ components:
         config:
           $ref: '#/components/schemas/AuthtypesAuthDomainConfig'
       type: object
+    CloudintegrationtypesAWSAccountConfig:
+      properties:
+        regions:
+          items:
+            type: string
+          type: array
+      required:
+      - regions
+      type: object
+    CloudintegrationtypesAWSCollectionStrategy:
+      properties:
+        aws_logs:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSLogsStrategy'
+        aws_metrics:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSMetricsStrategy'
+        s3_buckets:
+          additionalProperties:
+            items:
+              type: string
+            type: array
+          type: object
+      type: object
+    CloudintegrationtypesAWSConnectionArtifact:
+      properties:
+        connectionURL:
+          type: string
+      required:
+      - connectionURL
+      type: object
+    CloudintegrationtypesAWSConnectionArtifactRequest:
+      properties:
+        deploymentRegion:
+          type: string
+        regions:
+          items:
+            type: string
+          type: array
+      required:
+      - deploymentRegion
+      - regions
+      type: object
+    CloudintegrationtypesAWSIntegrationConfig:
+      properties:
+        enabledRegions:
+          items:
+            type: string
+          type: array
+        telemetry:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSCollectionStrategy'
+      required:
+      - enabledRegions
+      - telemetry
+      type: object
+    CloudintegrationtypesAWSLogsStrategy:
+      properties:
+        cloudwatch_logs_subscriptions:
+          items:
+            properties:
+              filter_pattern:
+                type: string
+              log_group_name_prefix:
+                type: string
+            type: object
+          nullable: true
+          type: array
+      type: object
+    CloudintegrationtypesAWSMetricsStrategy:
+      properties:
+        cloudwatch_metric_stream_filters:
+          items:
+            properties:
+              MetricNames:
+                items:
+                  type: string
+                type: array
+              Namespace:
+                type: string
+            type: object
+          nullable: true
+          type: array
+      type: object
+    CloudintegrationtypesAWSServiceConfig:
+      properties:
+        logs:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSServiceLogsConfig'
+        metrics:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSServiceMetricsConfig'
+      type: object
+    CloudintegrationtypesAWSServiceLogsConfig:
+      properties:
+        enabled:
+          type: boolean
+        s3_buckets:
+          additionalProperties:
+            items:
+              type: string
+            type: array
+          type: object
+      type: object
+    CloudintegrationtypesAWSServiceMetricsConfig:
+      properties:
+        enabled:
+          type: boolean
+      type: object
+    CloudintegrationtypesAccount:
+      properties:
+        agentReport:
+          $ref: '#/components/schemas/CloudintegrationtypesAgentReport'
+        config:
+          $ref: '#/components/schemas/CloudintegrationtypesAccountConfig'
+        createdAt:
+          format: date-time
+          type: string
+        id:
+          type: string
+        orgId:
+          type: string
+        provider:
+          type: string
+        providerAccountId:
+          nullable: true
+          type: string
+        removedAt:
+          format: date-time
+          nullable: true
+          type: string
+        updatedAt:
+          format: date-time
+          type: string
+      required:
+      - id
+      - providerAccountId
+      - provider
+      - removedAt
+      - agentReport
+      - orgId
+      - config
+      type: object
+    CloudintegrationtypesAccountConfig:
+      properties:
+        aws:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSAccountConfig'
+      required:
+      - aws
+      type: object
+    CloudintegrationtypesAgentReport:
+      nullable: true
+      properties:
+        data:
+          additionalProperties: {}
+          nullable: true
+          type: object
+        timestampMillis:
+          format: int64
+          type: integer
+      required:
+      - timestampMillis
+      - data
+      type: object
+    CloudintegrationtypesAssets:
+      properties:
+        dashboards:
+          items:
+            $ref: '#/components/schemas/CloudintegrationtypesDashboard'
+          nullable: true
+          type: array
+      type: object
+    CloudintegrationtypesCollectedLogAttribute:
+      properties:
+        name:
+          type: string
+        path:
+          type: string
+        type:
+          type: string
+      type: object
+    CloudintegrationtypesCollectedMetric:
+      properties:
+        description:
+          type: string
+        name:
+          type: string
+        type:
+          type: string
+        unit:
+          type: string
+      type: object
+    CloudintegrationtypesCollectionStrategy:
+      properties:
+        aws:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSCollectionStrategy'
+      required:
+      - aws
+      type: object
+    CloudintegrationtypesConnectionArtifact:
+      properties:
+        aws:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSConnectionArtifact'
+      required:
+      - aws
+      type: object
+    CloudintegrationtypesConnectionArtifactRequest:
+      properties:
+        aws:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSConnectionArtifactRequest'
+      required:
+      - aws
+      type: object
+    CloudintegrationtypesDashboard:
+      properties:
+        definition:
+          $ref: '#/components/schemas/DashboardtypesStorableDashboardData'
+        description:
+          type: string
+        id:
+          type: string
+        title:
+          type: string
+      type: object
+    CloudintegrationtypesDataCollected:
+      properties:
+        logs:
+          items:
+            $ref: '#/components/schemas/CloudintegrationtypesCollectedLogAttribute'
+          nullable: true
+          type: array
+        metrics:
+          items:
+            $ref: '#/components/schemas/CloudintegrationtypesCollectedMetric'
+          nullable: true
+          type: array
+      type: object
+    CloudintegrationtypesGettableAccountWithArtifact:
+      properties:
+        connectionArtifact:
+          $ref: '#/components/schemas/CloudintegrationtypesConnectionArtifact'
+        id:
+          type: string
+      required:
+      - id
+      - connectionArtifact
+      type: object
+    CloudintegrationtypesGettableAccounts:
+      properties:
+        accounts:
+          items:
+            $ref: '#/components/schemas/CloudintegrationtypesAccount'
+          type: array
+      required:
+      - accounts
+      type: object
+    CloudintegrationtypesGettableAgentCheckInResponse:
+      properties:
+        account_id:
+          type: string
+        cloud_account_id:
+          type: string
+        cloudIntegrationId:
+          type: string
+        integration_config:
+          $ref: '#/components/schemas/CloudintegrationtypesIntegrationConfig'
+        integrationConfig:
+          $ref: '#/components/schemas/CloudintegrationtypesProviderIntegrationConfig'
+        providerAccountId:
+          type: string
+        removed_at:
+          format: date-time
+          nullable: true
+          type: string
+        removedAt:
+          format: date-time
+          nullable: true
+          type: string
+      required:
+      - account_id
+      - cloud_account_id
+      - integration_config
+      - removed_at
+      - cloudIntegrationId
+      - providerAccountId
+      - integrationConfig
+      - removedAt
+      type: object
+    CloudintegrationtypesGettableServicesMetadata:
+      properties:
+        services:
+          items:
+            $ref: '#/components/schemas/CloudintegrationtypesServiceMetadata'
+          type: array
+      required:
+      - services
+      type: object
+    CloudintegrationtypesIntegrationConfig:
+      nullable: true
+      properties:
+        enabled_regions:
+          items:
+            type: string
+          type: array
+        telemetry:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSCollectionStrategy'
+      required:
+      - enabled_regions
+      - telemetry
+      type: object
+    CloudintegrationtypesPostableAgentCheckInRequest:
+      properties:
+        account_id:
+          type: string
+        cloud_account_id:
+          type: string
+        cloudIntegrationId:
+          type: string
+        data:
+          additionalProperties: {}
+          nullable: true
+          type: object
+        providerAccountId:
+          type: string
+      required:
+      - data
+      type: object
+    CloudintegrationtypesProviderIntegrationConfig:
+      properties:
+        aws:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSIntegrationConfig'
+      required:
+      - aws
+      type: object
+    CloudintegrationtypesService:
+      properties:
+        assets:
+          $ref: '#/components/schemas/CloudintegrationtypesAssets'
+        dataCollected:
+          $ref: '#/components/schemas/CloudintegrationtypesDataCollected'
+        icon:
+          type: string
+        id:
+          type: string
+        overview:
+          type: string
+        serviceConfig:
+          $ref: '#/components/schemas/CloudintegrationtypesServiceConfig'
+        supported_signals:
+          $ref: '#/components/schemas/CloudintegrationtypesSupportedSignals'
+        telemetryCollectionStrategy:
+          $ref: '#/components/schemas/CloudintegrationtypesCollectionStrategy'
+        title:
+          type: string
+      required:
+      - id
+      - title
+      - icon
+      - overview
+      - assets
+      - supported_signals
+      - dataCollected
+      - telemetryCollectionStrategy
+      type: object
+    CloudintegrationtypesServiceConfig:
+      properties:
+        aws:
+          $ref: '#/components/schemas/CloudintegrationtypesAWSServiceConfig'
+      required:
+      - aws
+      type: object
+    CloudintegrationtypesServiceMetadata:
+      properties:
+        enabled:
+          type: boolean
+        icon:
+          type: string
+        id:
+          type: string
+        title:
+          type: string
+      required:
+      - id
+      - title
+      - icon
+      - enabled
+      type: object
+    CloudintegrationtypesSupportedSignals:
+      properties:
+        logs:
+          type: boolean
+        metrics:
+          type: boolean
+      type: object
+    CloudintegrationtypesUpdatableAccount:
+      properties:
+        config:
+          $ref: '#/components/schemas/CloudintegrationtypesAccountConfig'
+      required:
+      - config
+      type: object
+    CloudintegrationtypesUpdatableService:
+      properties:
+        config:
+          $ref: '#/components/schemas/CloudintegrationtypesServiceConfig'
+      required:
+      - config
+      type: object
     DashboardtypesDashboard:
       properties:
         createdAt:
@@ -2484,6 +2887,551 @@ paths:
       summary: Change password
       tags:
       - users
+  /api/v1/cloud-integrations/{cloud_provider}/agent-check-in:
+    post:
+      deprecated: true
+      description: '[Deprecated] This endpoint is called by the deployed agent to
+        check in'
+      operationId: AgentCheckInDeprecated
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CloudintegrationtypesPostableAgentCheckInRequest'
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/CloudintegrationtypesGettableAgentCheckInResponse'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - VIEWER
+      - tokenizer:
+        - VIEWER
+      summary: Agent check-in
+      tags:
+      - cloudintegration
+  /api/v1/cloud_integrations/{cloud_provider}/accounts:
+    get:
+      deprecated: false
+      description: This endpoint lists the accounts for the specified cloud provider
+      operationId: ListAccounts
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/CloudintegrationtypesGettableAccounts'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: List accounts
+      tags:
+      - cloudintegration
+    post:
+      deprecated: false
+      description: This endpoint creates a new cloud integration account for the specified
+        cloud provider
+      operationId: CreateAccount
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CloudintegrationtypesConnectionArtifactRequest'
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/CloudintegrationtypesGettableAccountWithArtifact'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: Create account
+      tags:
+      - cloudintegration
+  /api/v1/cloud_integrations/{cloud_provider}/accounts/{id}:
+    delete:
+      deprecated: false
+      description: This endpoint disconnects an account for the specified cloud provider
+      operationId: DisconnectAccount
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: id
+        required: true
+        schema:
+          type: string
+      responses:
+        "204":
+          description: No Content
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: Disconnect account
+      tags:
+      - cloudintegration
+    get:
+      deprecated: false
+      description: This endpoint gets an account for the specified cloud provider
+      operationId: GetAccount
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: id
+        required: true
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/CloudintegrationtypesAccount'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "400":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Bad Request
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "404":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Not Found
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: Get account
+      tags:
+      - cloudintegration
+    put:
+      deprecated: false
+      description: This endpoint updates an account for the specified cloud provider
+      operationId: UpdateAccount
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: id
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CloudintegrationtypesUpdatableAccount'
+      responses:
+        "204":
+          description: No Content
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: Update account
+      tags:
+      - cloudintegration
+  /api/v1/cloud_integrations/{cloud_provider}/accounts/check_in:
+    post:
+      deprecated: false
+      description: This endpoint is called by the deployed agent to check in
+      operationId: AgentCheckIn
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CloudintegrationtypesPostableAgentCheckInRequest'
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/CloudintegrationtypesGettableAgentCheckInResponse'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - VIEWER
+      - tokenizer:
+        - VIEWER
+      summary: Agent check-in
+      tags:
+      - cloudintegration
+  /api/v1/cloud_integrations/{cloud_provider}/services:
+    get:
+      deprecated: false
+      description: This endpoint lists the services metadata for the specified cloud
+        provider
+      operationId: ListServicesMetadata
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/CloudintegrationtypesGettableServicesMetadata'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: List services metadata
+      tags:
+      - cloudintegration
+  /api/v1/cloud_integrations/{cloud_provider}/services/{service_id}:
+    get:
+      deprecated: false
+      description: This endpoint gets a service for the specified cloud provider
+      operationId: GetService
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: service_id
+        required: true
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/CloudintegrationtypesService'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: Get service
+      tags:
+      - cloudintegration
+    put:
+      deprecated: false
+      description: This endpoint updates a service for the specified cloud provider
+      operationId: UpdateService
+      parameters:
+      - in: path
+        name: cloud_provider
+        required: true
+        schema:
+          type: string
+      - in: path
+        name: service_id
+        required: true
+        schema:
+          type: string
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/CloudintegrationtypesUpdatableService'
+      responses:
+        "204":
+          description: No Content
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - ADMIN
+      - tokenizer:
+        - ADMIN
+      summary: Update service
+      tags:
+      - cloudintegration
   /api/v1/complete/google:
     get:
       deprecated: false

ee/query-service/app/server.go

@@ -136,6 +136,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 	logParsingPipelineController, err := logparsingpipeline.NewLogParsingPipelinesController(
 		signoz.SQLStore,
 		integrationsController.GetPipelinesForInstalledIntegrations,
+		reader,
 	)
 	if err != nil {
 		return nil, err

ee/query-service/rules/manager_test.go

@@ -257,7 +257,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 						WillReturnRows(samplesRows)
 
 					// Create Prometheus provider for this test
-					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, store)
+					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, store)
 				},
 				ManagerOptionsHook: func(opts *rules.ManagerOptions) {
 					// Set Prometheus provider for PromQL queries

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -437,6 +437,436 @@ export interface AuthtypesUpdateableAuthDomainDTO {
 	config?: AuthtypesAuthDomainConfigDTO;
 }
 
+export interface CloudintegrationtypesAWSAccountConfigDTO {
+	/**
+	 * @type array
+	 */
+	regions: string[];
+}
+
+export type CloudintegrationtypesAWSCollectionStrategyDTOS3Buckets = {
+	[key: string]: string[];
+};
+
+export interface CloudintegrationtypesAWSCollectionStrategyDTO {
+	aws_logs?: CloudintegrationtypesAWSLogsStrategyDTO;
+	aws_metrics?: CloudintegrationtypesAWSMetricsStrategyDTO;
+	/**
+	 * @type object
+	 */
+	s3_buckets?: CloudintegrationtypesAWSCollectionStrategyDTOS3Buckets;
+}
+
+export interface CloudintegrationtypesAWSConnectionArtifactDTO {
+	/**
+	 * @type string
+	 */
+	connectionURL: string;
+}
+
+export interface CloudintegrationtypesAWSConnectionArtifactRequestDTO {
+	/**
+	 * @type string
+	 */
+	deploymentRegion: string;
+	/**
+	 * @type array
+	 */
+	regions: string[];
+}
+
+export interface CloudintegrationtypesAWSIntegrationConfigDTO {
+	/**
+	 * @type array
+	 */
+	enabledRegions: string[];
+	telemetry: CloudintegrationtypesAWSCollectionStrategyDTO;
+}
+
+export type CloudintegrationtypesAWSLogsStrategyDTOCloudwatchLogsSubscriptionsItem = {
+	/**
+	 * @type string
+	 */
+	filter_pattern?: string;
+	/**
+	 * @type string
+	 */
+	log_group_name_prefix?: string;
+};
+
+export interface CloudintegrationtypesAWSLogsStrategyDTO {
+	/**
+	 * @type array
+	 * @nullable true
+	 */
+	cloudwatch_logs_subscriptions?:
+		| CloudintegrationtypesAWSLogsStrategyDTOCloudwatchLogsSubscriptionsItem[]
+		| null;
+}
+
+export type CloudintegrationtypesAWSMetricsStrategyDTOCloudwatchMetricStreamFiltersItem = {
+	/**
+	 * @type array
+	 */
+	MetricNames?: string[];
+	/**
+	 * @type string
+	 */
+	Namespace?: string;
+};
+
+export interface CloudintegrationtypesAWSMetricsStrategyDTO {
+	/**
+	 * @type array
+	 * @nullable true
+	 */
+	cloudwatch_metric_stream_filters?:
+		| CloudintegrationtypesAWSMetricsStrategyDTOCloudwatchMetricStreamFiltersItem[]
+		| null;
+}
+
+export interface CloudintegrationtypesAWSServiceConfigDTO {
+	logs?: CloudintegrationtypesAWSServiceLogsConfigDTO;
+	metrics?: CloudintegrationtypesAWSServiceMetricsConfigDTO;
+}
+
+export type CloudintegrationtypesAWSServiceLogsConfigDTOS3Buckets = {
+	[key: string]: string[];
+};
+
+export interface CloudintegrationtypesAWSServiceLogsConfigDTO {
+	/**
+	 * @type boolean
+	 */
+	enabled?: boolean;
+	/**
+	 * @type object
+	 */
+	s3_buckets?: CloudintegrationtypesAWSServiceLogsConfigDTOS3Buckets;
+}
+
+export interface CloudintegrationtypesAWSServiceMetricsConfigDTO {
+	/**
+	 * @type boolean
+	 */
+	enabled?: boolean;
+}
+
+export interface CloudintegrationtypesAccountDTO {
+	agentReport: CloudintegrationtypesAgentReportDTO;
+	config: CloudintegrationtypesAccountConfigDTO;
+	/**
+	 * @type string
+	 * @format date-time
+	 */
+	createdAt?: Date;
+	/**
+	 * @type string
+	 */
+	id: string;
+	/**
+	 * @type string
+	 */
+	orgId: string;
+	/**
+	 * @type string
+	 */
+	provider: string;
+	/**
+	 * @type string
+	 * @nullable true
+	 */
+	providerAccountId: string | null;
+	/**
+	 * @type string
+	 * @format date-time
+	 * @nullable true
+	 */
+	removedAt: Date | null;
+	/**
+	 * @type string
+	 * @format date-time
+	 */
+	updatedAt?: Date;
+}
+
+export interface CloudintegrationtypesAccountConfigDTO {
+	aws: CloudintegrationtypesAWSAccountConfigDTO;
+}
+
+/**
+ * @nullable
+ */
+export type CloudintegrationtypesAgentReportDTOData = {
+	[key: string]: unknown;
+} | null;
+
+/**
+ * @nullable
+ */
+export type CloudintegrationtypesAgentReportDTO = {
+	/**
+	 * @type object
+	 * @nullable true
+	 */
+	data: CloudintegrationtypesAgentReportDTOData;
+	/**
+	 * @type integer
+	 * @format int64
+	 */
+	timestampMillis: number;
+} | null;
+
+export interface CloudintegrationtypesAssetsDTO {
+	/**
+	 * @type array
+	 * @nullable true
+	 */
+	dashboards?: CloudintegrationtypesDashboardDTO[] | null;
+}
+
+export interface CloudintegrationtypesCollectedLogAttributeDTO {
+	/**
+	 * @type string
+	 */
+	name?: string;
+	/**
+	 * @type string
+	 */
+	path?: string;
+	/**
+	 * @type string
+	 */
+	type?: string;
+}
+
+export interface CloudintegrationtypesCollectedMetricDTO {
+	/**
+	 * @type string
+	 */
+	description?: string;
+	/**
+	 * @type string
+	 */
+	name?: string;
+	/**
+	 * @type string
+	 */
+	type?: string;
+	/**
+	 * @type string
+	 */
+	unit?: string;
+}
+
+export interface CloudintegrationtypesCollectionStrategyDTO {
+	aws: CloudintegrationtypesAWSCollectionStrategyDTO;
+}
+
+export interface CloudintegrationtypesConnectionArtifactDTO {
+	aws: CloudintegrationtypesAWSConnectionArtifactDTO;
+}
+
+export interface CloudintegrationtypesConnectionArtifactRequestDTO {
+	aws: CloudintegrationtypesAWSConnectionArtifactRequestDTO;
+}
+
+export interface CloudintegrationtypesDashboardDTO {
+	definition?: DashboardtypesStorableDashboardDataDTO;
+	/**
+	 * @type string
+	 */
+	description?: string;
+	/**
+	 * @type string
+	 */
+	id?: string;
+	/**
+	 * @type string
+	 */
+	title?: string;
+}
+
+export interface CloudintegrationtypesDataCollectedDTO {
+	/**
+	 * @type array
+	 * @nullable true
+	 */
+	logs?: CloudintegrationtypesCollectedLogAttributeDTO[] | null;
+	/**
+	 * @type array
+	 * @nullable true
+	 */
+	metrics?: CloudintegrationtypesCollectedMetricDTO[] | null;
+}
+
+export interface CloudintegrationtypesGettableAccountWithArtifactDTO {
+	connectionArtifact: CloudintegrationtypesConnectionArtifactDTO;
+	/**
+	 * @type string
+	 */
+	id: string;
+}
+
+export interface CloudintegrationtypesGettableAccountsDTO {
+	/**
+	 * @type array
+	 */
+	accounts: CloudintegrationtypesAccountDTO[];
+}
+
+export interface CloudintegrationtypesGettableAgentCheckInResponseDTO {
+	/**
+	 * @type string
+	 */
+	account_id: string;
+	/**
+	 * @type string
+	 */
+	cloud_account_id: string;
+	/**
+	 * @type string
+	 */
+	cloudIntegrationId: string;
+	integration_config: CloudintegrationtypesIntegrationConfigDTO;
+	integrationConfig: CloudintegrationtypesProviderIntegrationConfigDTO;
+	/**
+	 * @type string
+	 */
+	providerAccountId: string;
+	/**
+	 * @type string
+	 * @format date-time
+	 * @nullable true
+	 */
+	removed_at: Date | null;
+	/**
+	 * @type string
+	 * @format date-time
+	 * @nullable true
+	 */
+	removedAt: Date | null;
+}
+
+export interface CloudintegrationtypesGettableServicesMetadataDTO {
+	/**
+	 * @type array
+	 */
+	services: CloudintegrationtypesServiceMetadataDTO[];
+}
+
+/**
+ * @nullable
+ */
+export type CloudintegrationtypesIntegrationConfigDTO = {
+	/**
+	 * @type array
+	 */
+	enabled_regions: string[];
+	telemetry: CloudintegrationtypesAWSCollectionStrategyDTO;
+} | null;
+
+/**
+ * @nullable
+ */
+export type CloudintegrationtypesPostableAgentCheckInRequestDTOData = {
+	[key: string]: unknown;
+} | null;
+
+export interface CloudintegrationtypesPostableAgentCheckInRequestDTO {
+	/**
+	 * @type string
+	 */
+	account_id?: string;
+	/**
+	 * @type string
+	 */
+	cloud_account_id?: string;
+	/**
+	 * @type string
+	 */
+	cloudIntegrationId?: string;
+	/**
+	 * @type object
+	 * @nullable true
+	 */
+	data: CloudintegrationtypesPostableAgentCheckInRequestDTOData;
+	/**
+	 * @type string
+	 */
+	providerAccountId?: string;
+}
+
+export interface CloudintegrationtypesProviderIntegrationConfigDTO {
+	aws: CloudintegrationtypesAWSIntegrationConfigDTO;
+}
+
+export interface CloudintegrationtypesServiceDTO {
+	assets: CloudintegrationtypesAssetsDTO;
+	dataCollected: CloudintegrationtypesDataCollectedDTO;
+	/**
+	 * @type string
+	 */
+	icon: string;
+	/**
+	 * @type string
+	 */
+	id: string;
+	/**
+	 * @type string
+	 */
+	overview: string;
+	serviceConfig?: CloudintegrationtypesServiceConfigDTO;
+	supported_signals: CloudintegrationtypesSupportedSignalsDTO;
+	telemetryCollectionStrategy: CloudintegrationtypesCollectionStrategyDTO;
+	/**
+	 * @type string
+	 */
+	title: string;
+}
+
+export interface CloudintegrationtypesServiceConfigDTO {
+	aws: CloudintegrationtypesAWSServiceConfigDTO;
+}
+
+export interface CloudintegrationtypesServiceMetadataDTO {
+	/**
+	 * @type boolean
+	 */
+	enabled: boolean;
+	/**
+	 * @type string
+	 */
+	icon: string;
+	/**
+	 * @type string
+	 */
+	id: string;
+	/**
+	 * @type string
+	 */
+	title: string;
+}
+
+export interface CloudintegrationtypesSupportedSignalsDTO {
+	/**
+	 * @type boolean
+	 */
+	logs?: boolean;
+	/**
+	 * @type boolean
+	 */
+	metrics?: boolean;
+}
+
+export interface CloudintegrationtypesUpdatableAccountDTO {
+	config: CloudintegrationtypesAccountConfigDTO;
+}
+
+export interface CloudintegrationtypesUpdatableServiceDTO {
+	config: CloudintegrationtypesServiceConfigDTO;
+}
+
 export interface DashboardtypesDashboardDTO {
 	/**
 	 * @type string
@@ -2858,6 +3288,97 @@ export type AuthzResources200 = {
 export type ChangePasswordPathParameters = {
 	id: string;
 };
+export type AgentCheckInDeprecatedPathParameters = {
+	cloudProvider: string;
+};
+export type AgentCheckInDeprecated200 = {
+	data: CloudintegrationtypesGettableAgentCheckInResponseDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type ListAccountsPathParameters = {
+	cloudProvider: string;
+};
+export type ListAccounts200 = {
+	data: CloudintegrationtypesGettableAccountsDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type CreateAccountPathParameters = {
+	cloudProvider: string;
+};
+export type CreateAccount200 = {
+	data: CloudintegrationtypesGettableAccountWithArtifactDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type DisconnectAccountPathParameters = {
+	cloudProvider: string;
+	id: string;
+};
+export type GetAccountPathParameters = {
+	cloudProvider: string;
+	id: string;
+};
+export type GetAccount200 = {
+	data: CloudintegrationtypesAccountDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type UpdateAccountPathParameters = {
+	cloudProvider: string;
+	id: string;
+};
+export type AgentCheckInPathParameters = {
+	cloudProvider: string;
+};
+export type AgentCheckIn200 = {
+	data: CloudintegrationtypesGettableAgentCheckInResponseDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type ListServicesMetadataPathParameters = {
+	cloudProvider: string;
+};
+export type ListServicesMetadata200 = {
+	data: CloudintegrationtypesGettableServicesMetadataDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type GetServicePathParameters = {
+	cloudProvider: string;
+	serviceId: string;
+};
+export type GetService200 = {
+	data: CloudintegrationtypesServiceDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type UpdateServicePathParameters = {
+	cloudProvider: string;
+	serviceId: string;
+};
 export type CreateSessionByGoogleCallback303 = {
 	data: AuthtypesGettableTokenDTO;
 	/**

frontend/src/components/CeleryTask/useNavigateToExplorer.ts

@@ -109,6 +109,7 @@ export function useNavigateToExplorer(): (
 					widgetConfig: {
 						query: preparedQuery,
 						panelTypes: PANEL_TYPES.TIME_SERIES,
+						// does this change at any time? How does it change if so?
 						timePreferance: 'GLOBAL_TIME',
 					},
 					selectedDashboard,

frontend/src/container/DashboardContainer/DashboardDescription/index.tsx

@@ -577,6 +577,7 @@ function DashboardDescription(props: DashboardDescriptionProps): JSX.Element {
 				open={isPanelNameModalOpen}
 				title="New Section"
 				rootClassName="section-naming"
+				// This is not required on the specification of a custom footer
 				onOk={(): void => handleAddRow()}
 				onCancel={(): void => {
 					setIsPanelNameModalOpen(false);

frontend/src/container/InfraMonitoringK8s/Volumes/VolumeDetails/constants.ts

@@ -62,9 +62,6 @@ export const getVolumeQueryPayload = (
 	const k8sPVCNameKey = dotMetricsEnabled
 		? 'k8s.persistentvolumeclaim.name'
 		: 'k8s_persistentvolumeclaim_name';
-	const legendTemplate = dotMetricsEnabled
-		? '{{k8s.namespace.name}}-{{k8s.pod.name}}'
-		: '{{k8s_namespace_name}}-{{k8s_pod_name}}';
 
 	return [
 		{
@@ -136,7 +133,7 @@ export const getVolumeQueryPayload = (
 							functions: [],
 							groupBy: [],
 							having: [],
-							legend: legendTemplate,
+							legend: 'Available',
 							limit: null,
 							orderBy: [],
 							queryName: 'A',
@@ -228,7 +225,7 @@ export const getVolumeQueryPayload = (
 							functions: [],
 							groupBy: [],
 							having: [],
-							legend: legendTemplate,
+							legend: 'Capacity',
 							limit: null,
 							orderBy: [],
 							queryName: 'A',
@@ -319,7 +316,7 @@ export const getVolumeQueryPayload = (
 							},
 							groupBy: [],
 							having: [],
-							legend: legendTemplate,
+							legend: 'Inodes Used',
 							limit: null,
 							orderBy: [],
 							queryName: 'A',
@@ -411,7 +408,7 @@ export const getVolumeQueryPayload = (
 							},
 							groupBy: [],
 							having: [],
-							legend: legendTemplate,
+							legend: 'Total Inodes',
 							limit: null,
 							orderBy: [],
 							queryName: 'A',
@@ -503,7 +500,7 @@ export const getVolumeQueryPayload = (
 							},
 							groupBy: [],
 							having: [],
-							legend: legendTemplate,
+							legend: 'Inodes Free',
 							limit: null,
 							orderBy: [],
 							queryName: 'A',

frontend/src/container/NewWidget/types.ts

@@ -16,8 +16,10 @@ export interface NewWidgetProps {
 }
 
 export interface WidgetGraphProps {
+	// this should be inside the querier plugin definition
 	selectedLogFields: Widgets['selectedLogFields'];
 	setSelectedLogFields?: Dispatch<SetStateAction<Widgets['selectedLogFields']>>;
+	// this should be inside the querier plugin definition
 	selectedTracesFields: Widgets['selectedTracesFields'];
 	setSelectedTracesFields?: Dispatch<
 		SetStateAction<Widgets['selectedTracesFields']>

frontend/src/types/api/dashboard/getAll.ts

@@ -118,7 +118,6 @@ export interface IBaseWidget {
 	opacity: string;
 	nullZeroValues: string;
 	timePreferance: timePreferenceType;
-	stepSize?: number;
 	yAxisUnit?: string;
 	decimalPrecision?: PrecisionOption; // number of decimals or 'full precision'
 	stackedBarChart?: boolean;

perses/common/common.cue

@@ -0,0 +1,131 @@
+package common
+
+// ──────────────────────────────────────────────
+// Shared types
+// ──────────────────────────────────────────────
+
+// ! What is used for - Got answer
+// Which one of these will be there if this exists
+#TelemetryFieldKey: {
+    name:           string
+    key?:           string
+    description?:   string
+    unit?:          string
+    signal?:        string
+    fieldContext?:   string
+    fieldDataType?: string
+    materialized?:  bool
+    isIndexed?:     bool
+}
+
+// ──────────────────────────────────────────────
+// Panel types
+// ──────────────────────────────────────────────
+
+#ContextLinkProps: {
+    url:   string
+    label: string
+}
+
+// is this used on the frontend or just for query building?
+#TimePreference: *"globalTime" | "last5Min" | "last15Min" | "last30Min" | "last1Hr" | "last6Hr" | "last1Day" | "last3Days" | "last1Week" | "last1Month"
+
+// is the default option the one in the front
+#PrecisionOption: *2 | 0 | 1 | 3 | 4 | "full"
+
+// how is this undefined and null? Would like to avoid undefined if possible
+// also bools can required always?
+#Axes: {
+    softMin?:    number | *null
+    softMax?:    number | *null
+    isLogScale?: bool | *false
+}
+
+#LegendPosition: *"bottom" | "right"
+
+#ThresholdWithLabel: {
+    value:  number
+    unit?:  string
+    // is this required? I think it is auto on the frontend and it can be overridden when set
+    color:  string
+    // What is this?
+    format: "Text" | "Background"
+    // What would be label if undefined here? There would be some default noe
+    label?: string
+}
+
+// where is this used
+#ComparisonThreshold: {
+    value:    number
+    operator: ">" | "<" | ">=" | "<=" | "="
+    unit?:    string
+    color:    string
+    format:   "Text" | "Background"
+}
+
+// ──────────────────────────────────────────────
+// Query types
+// ──────────────────────────────────────────────
+
+#QueryName: =~"^[A-Za-z][A-Za-z0-9_]*$"
+
+// where used?
+#Limit: int & >=0 & <=10000
+
+#Offset: int & >=0
+
+#ReduceTo: "sum" | "count" | "avg" | "min" | "max" | "last" | "median"
+
+// what is close?
+// where is this used?
+#MetricAggregation: close({
+    metricName:       string & !=""
+    timeAggregation:  "latest" | "sum" | "avg" | "min" | "max" | "count" | "rate" | "increase"
+    spaceAggregation: "sum" | "avg" | "min" | "max" | "count" | "p50" | "p75" | "p90" | "p95" | "p99"
+    reduceTo?:        #ReduceTo
+    // should unspecified be the default?
+    temporality?:     "delta" | "cumulative" | "unspecified"
+})
+
+#ExpressionAggregation: close({
+    expression: string & !=""
+    alias?:     string
+})
+
+#Aggregation: #MetricAggregation | #ExpressionAggregation
+
+// Can this be extended with common struct? this also has - HavingExpression
+#FilterExpression: close({
+    expression: string
+})
+
+#GroupByItem: close({
+    name:           string & !=""
+    fieldDataType?: string
+    fieldContext?:   string
+})
+
+#OrderByItem: close({
+    columnName: string & !=""
+    order:      "asc" | "desc"
+})
+
+#HavingExpression: close({
+    expression: string
+})
+
+// Is the definition used for above?
+#Function: close({
+    name: "cutOffMin" | "cutOffMax" | "clampMin" | "clampMax" |
+        "absolute" | "runningDiff" | "log2" | "log10" |
+        "cumulativeSum" | "ewma3" | "ewma5" | "ewma7" |
+        "median3" | "median5" | "median7" | "timeShift" |
+        "anomaly" | "fillZero"
+    args?: [...close({value: number | string | bool})]
+})
+
+// ──────────────────────────────────────────────
+// Variable types
+// ──────────────────────────────────────────────
+
+#VariableSortOrder: *"disabled" | "asc" | "desc"

perses/cue.mod/module.cue

@@ -0,0 +1,4 @@
+module: "github.com/signoz"
+language: {
+	version: "v0.12.0"
+}

perses/examples/perses.json

@@ -0,0 +1,926 @@
+{
+  "kind": "Dashboard",
+  "metadata": {
+    "name": "the-everything-dashboard",
+    "project": "signoz" // should this be an id? What is this project exactly?
+    // where would createdAt, updatedAt, createdBy, updatedBy be used?
+    // where would dashboardID or organizationID go
+    // where is locked going to come in
+    // can we have public path here as well? default time range and
+
+    // where would version go?
+    // where would image value go?
+    // where would tags go?
+
+    // we have uploadedGrafana. What is uploadedGrafana used for?
+
+    // ! all above looks like we need a plugin for the root dashboard as well
+
+    // how is panelMap and widgets translated to layout? Is layout the same as before?
+    // where is timePreferance set up
+    // collapsed?
+    // ? https://github.com/react-grid-layout/react-grid-layout?tab=readme-ov-file#layout-item
+  },
+  "spec": {
+    "display": {
+      "name": "The everything dashboard", // this is duplicate?
+      "description": "Trying to cover as many concepts here as possible"
+    },
+    "duration": "1h", // is this required?
+    "datasources": {
+      "SigNozDatasource": {
+        "default": true,
+        "plugin": {
+          "kind": "SigNozDatasource",
+          "spec": {}
+        }
+      }
+    },
+    "variables": [
+      {
+        // what is a ListVariable
+        "kind": "ListVariable",
+        "spec": {
+          "name": "serviceName",
+          "display": {
+            "name": "serviceName"
+          },
+          "allowAllValue": true,
+          "allowMultiple": false,
+          "plugin": {
+            "kind": "SigNozDynamicVariable",
+            "spec": {
+              "name": "service.name",
+              "source": "Metrics",
+              "sort": "disabled"
+            }
+          }
+        }
+      },
+      {
+        "kind": "ListVariable",
+        "spec": {
+          "name": "statusCodesFromQuery",
+          "display": {
+            "name": "statusCodesFromQuery"
+          },
+          "allowAllValue": true,
+          "allowMultiple": true,
+          "plugin": {
+            "kind": "SigNozQueryVariable",
+            "spec": {
+              // missing defaultValue for these list variables
+              "queryValue": "SELECT JSONExtractString(labels, 'http.status_code') AS status_code FROM signoz_metrics.distributed_time_series_v4_1day WHERE status_code != '' GROUP BY status_code",
+              "sort": "asc"
+            }
+          }
+        }
+      },
+      {
+        "kind": "ListVariable",
+        "spec": {
+          "name": "limit",
+          "display": {
+            "name": "limit"
+          },
+          "allowAllValue": false,
+          "allowMultiple": false,
+          "plugin": {
+            "kind": "SigNozCustomVariable",
+            "spec": {
+              // defaultValue missing
+              "customValue": "1,10,20,40,80,160,200",
+              "sort": "disabled"
+            }
+          }
+        }
+      },
+      {
+        // do we need uuid for id here? Do we need id at all?
+        // migration will require cleanup from localstorage as it is all saved with ids right now
+        "kind": "TextVariable",
+        "spec": {
+          "name": "textboxvar",
+          "display": {
+            "name": "textboxvar"
+          },
+          "value": "defaultvaluegoeshere",
+          "plugin": {
+            // just to confirm, this would be the type right?
+            "kind": "SigNozTextboxVariable",
+            "spec": {
+              // How should selectedValue be handled on share?
+              // we should have an option for selectedValue here
+              // should go in for all variables
+              // should be auto filled with the default or with the queried value. All for multi, first for single
+              // if selectedValue goes in, then we will need allSelected as well
+            }
+          }
+        }
+      }
+    ],
+    "panels": {
+      // need metadata inside this for section creation
+      // maybe another key just for sections. much easier
+      "24e2697b": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "total resp size",
+            "description": ""
+          },
+          "plugin": {
+            "kind": "SigNozTimeSeriesPanel",
+            "spec": {
+              "visualization": {
+                "fillSpans": true
+              },
+              "formatting": {
+                "unit": "By",
+                "decimalPrecision": 3
+              },
+              "axes": {
+                "softMax": 800,
+                "isLogScale": true
+              },
+              "legend": {
+                "position": "right",
+                "customColors": {
+                  "{service.name=\"sampleapp-gateway\"}": "#9ea5f7"
+                }
+              },
+              "contextLinks": [
+                {
+                  "label": "View service details",
+                  "url": "http://localhost:8080/{{_service.name}}?dfddf=%7B%7Blimit%7D%7D"
+                }
+              ],
+              "thresholds": [
+                {
+                  "value": 1024,
+                  "unit": "By",
+                  "color": "Red",
+                  "format": "Text",
+                  "label": "upper limit"
+                },
+                {
+                  "value": 100,
+                  "unit": "By",
+                  "color": "Orange",
+                  "format": "Text",
+                  "label": "kinda bad"
+                }
+              ]
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozBuilderQuery",
+                  "spec": {
+                    "name": "A",
+                    "signal": "metrics",
+                    "expression": "A",
+                    "aggregations": [
+                      {
+                        "metricName": "http.server.response.body.size.sum",
+                        "reduceTo": "sum",
+                        "spaceAggregation": "sum",
+                        "timeAggregation": "rate"
+                      }
+                    ],
+                    "filter": {
+                      "expression": "http.response.status_code IN $statusCodesFromQuery"
+                    },
+                    "groupBy": [
+                      {
+                        "name": "service.name",
+                        "fieldDataType": "string",
+                        "fieldContext": "tag"
+                      }
+                    ]
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "ff2f72f1": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "fraction of calls",
+            "description": ""
+          },
+          "plugin": {
+            "kind": "SigNozTimeSeriesPanel",
+            "spec": {
+              "visualization": {
+                "fillSpans": true
+              },
+              "formatting": {
+                "decimalPrecision": 1
+              },
+              "thresholds": [
+                {
+                  "value": 1,
+                  "color": "Blue",
+                  "format": "Background",
+                  "label": "max possible"
+                }
+              ]
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozCompositeQuery",
+                  "spec": {
+                    "queries": [
+                      {
+                        "type": "builder_query",
+                        "spec": {
+                          "name": "A",
+                          "signal": "metrics",
+                          "expression": "A",
+                          "disabled": true,
+                          "aggregations": [
+                            {
+                              "metricName": "signoz_calls_total",
+                              "reduceTo": "sum",
+                              "spaceAggregation": "sum",
+                              "timeAggregation": "rate"
+                            }
+                          ],
+                          "filter": {
+                            "expression": "service.name IN $serviceName  AND http.status_code IN $statusCodesFromQuery"
+                          }
+                        }
+                      },
+                      {
+                        "type": "builder_query",
+                        "spec": {
+                          "name": "B",
+                          "signal": "metrics",
+                          "expression": "B",
+                          "disabled": true,
+                          "aggregations": [
+                            {
+                              "metricName": "signoz_calls_total",
+                              "reduceTo": "sum",
+                              "spaceAggregation": "sum",
+                              "timeAggregation": "rate"
+                            }
+                          ],
+                          "filter": {
+                            "expression": "service.name in $serviceName"
+                          }
+                        }
+                      },
+                      {
+                        "type": "builder_formula",
+                        "spec": {
+                          "name": "F1",
+                          "expression": "A / B"
+                        }
+                      }
+                    ]
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "011605e7": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "total resp size"
+          },
+          "plugin": {
+            "kind": "SigNozBarChartPanel",
+            "spec": {
+              "visualization": {
+                "stackedBarChart": false
+              },
+              "formatting": {
+                "unit": "By"
+              }
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozBuilderQuery",
+                  "spec": {
+                    "name": "A",
+                    "signal": "metrics",
+                    "expression": "A",
+                    "aggregations": [
+                      {
+                        "metricName": "http.server.response.body.size.sum",
+                        "reduceTo": "sum",
+                        "spaceAggregation": "sum",
+                        "timeAggregation": "rate"
+                      }
+                    ],
+                    "filter": {
+                      "expression": "http.response.status_code IN $statusCodesFromQuery"
+                    },
+                    "groupBy": [
+                      {
+                        "name": "service.name",
+                        "fieldDataType": "string",
+                        "fieldContext": "tag"
+                      }
+                    ]
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "e23516fc": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "num traces for service"
+          },
+          "plugin": {
+            "kind": "SigNozNumberPanel",
+            "spec": {
+              "formatting": {
+                "unit": "none",
+                "decimalPrecision": 1
+              },
+              "thresholds": [
+                {
+                  "value": 1200000,
+                  "operator": ">",
+                  "unit": "none",
+                  "color": "Red",
+                  "format": "Text"
+                },
+                {
+                  "value": 1200000,
+                  "operator": "<=",
+                  "unit": "none",
+                  "color": "Green",
+                  "format": "Text"
+                }
+              ]
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozBuilderQuery",
+                  "spec": {
+                    "name": "A",
+                    "signal": "traces",
+                    "expression": "A",
+                    "aggregations": [
+                      {
+                        "expression": "count() "
+                      }
+                    ],
+                    "filter": {
+                      "expression": "service.name = $serviceName "
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "130c8d6b": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "num logs for service"
+          },
+          "plugin": {
+            "kind": "SigNozNumberPanel",
+            "spec": {
+              "formatting": {
+                "unit": "none",
+                "decimalPrecision": 1
+              }
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozBuilderQuery",
+                  "spec": {
+                    "name": "A",
+                    "signal": "logs",
+                    "expression": "A",
+                    "aggregations": [
+                      {
+                        "expression": "count() "
+                      }
+                    ],
+                    "filter": {
+                      "expression": "service.name = $serviceName "
+                    }
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "246f7c6d": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "num traces for service per resp code"
+          },
+          "plugin": {
+            "kind": "SigNozPieChartPanel",
+            "spec": {
+              "formatting": {
+                "decimalPrecision": 1
+              },
+              "legend": {
+                "customColors": {
+                  "\"201\"": "#2bc051",
+                  "\"400\"": "#cc462e",
+                  "\"500\"": "#ff0000"
+                }
+              }
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozBuilderQuery",
+                  "spec": {
+                    "name": "A",
+                    "signal": "traces",
+                    "expression": "A",
+                    "aggregations": [
+                      {
+                        "expression": "count() "
+                      }
+                    ],
+                    "filter": {
+                      "expression": "service.name = $serviceName isEntryPoint = 'true'"
+                    },
+                    "groupBy": [
+                      {
+                        "name": "http.response.status_code",
+                        "fieldDataType": "float64",
+                        "fieldContext": "tag"
+                      }
+                    ],
+                    "legend": "\"{{http.response.status_code}}\""
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "21f7d4d0": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "average latency per service"
+          },
+          "plugin": {
+            "kind": "SigNozTablePanel",
+            "spec": {
+              "formatting": {
+                "columnUnits": {
+                  "A": "s"
+                }
+              },
+              "thresholds": [
+                {
+                  "value": 1,
+                  "operator": ">",
+                  "unit": "min",
+                  "color": "Red",
+                  "format": "Text",
+                  "tableOptions": "A"
+                }
+              ]
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozClickHouseSQL",
+                  "spec": {
+                    "name": "A",
+                    "query": "WITH\n    __spatial_aggregation_cte AS\n    (\n        SELECT\n            toStartOfInterval(toDateTime(intDiv(unix_milli, 1000)), toIntervalSecond(60)) AS ts,\n            `service.name`,\n            le,\n            sum(value) / 60 AS value\n        FROM signoz_metrics.distributed_samples_v4 AS points\n        INNER JOIN\n        (\n            SELECT\n                fingerprint,\n                JSONExtractString(labels, 'service.name') AS `service.name`,\n                JSONExtractString(labels, 'le') AS le\n            FROM signoz_metrics.time_series_v4\n            WHERE (metric_name IN ('signoz_latency.bucket')) AND (LOWER(temporality) LIKE LOWER('delta')) AND (__normalized = 0)\n            GROUP BY\n                fingerprint,\n                `service.name`,\n                le\n        ) AS filtered_time_series ON points.fingerprint = filtered_time_series.fingerprint\n        WHERE metric_name IN ('signoz_latency.bucket')\n        GROUP BY\n            ts,\n            `service.name`,\n            le\n    ),\n    __histogramCTE AS\n    (\n        SELECT\n            ts,\n            `service.name`,\n            histogramQuantile(arrayMap(x -> toFloat64(x), groupArray(le)), groupArray(value), 0.9) AS value\n        FROM __spatial_aggregation_cte\n        GROUP BY\n            `service.name`,\n            ts\n        ORDER BY\n            `service.name` ASC,\n            ts ASC\n    )\nSELECT\n    `service.name` AS service,\n    avg(value) AS A\nFROM __histogramCTE\nGROUP BY `service.name`"
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "ad5fd556": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "logs from service"
+          },
+          "plugin": {
+            "kind": "SigNozListPanel",
+            "spec": {
+              "selectedLogFields": [
+                {
+                  "name": "timestamp",
+                  "type": "log",
+                  "dataType": ""
+                },
+                {
+                  "name": "body",
+                  "type": "log",
+                  "dataType": ""
+                },
+                {
+                  "name": "error",
+                  "type": "",
+                  "dataType": "string"
+                }
+              ]
+            }
+          },
+          "queries": [
+            {
+              "kind": "LogQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozBuilderQuery",
+                  "spec": {
+                    "name": "A",
+                    "signal": "logs",
+                    "expression": "A",
+                    "aggregations": [
+                      {
+                        "expression": "count() "
+                      }
+                    ],
+                    "filter": {
+                      "expression": "service.name = $serviceName"
+                    },
+                    "groupBy": [],
+                    "order": [
+                      {
+                        "columnName": "timestamp",
+                        "order": "desc"
+                      },
+                      {
+                        "columnName": "id",
+                        "order": "desc"
+                      }
+                    ]
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "f07b59ee": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "response size buckets"
+          },
+          "plugin": {
+            "kind": "SigNozHistogramPanel",
+            "spec": {
+              "histogramBuckets": {
+                "bucketCount": 60,
+                "mergeAllActiveQueries": true
+              }
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozBuilderQuery",
+                  "spec": {
+                    "name": "A",
+                    "signal": "metrics",
+                    "expression": "A",
+                    "aggregations": [
+                      {
+                        "metricName": "http.server.response.body.size.bucket",
+                        "reduceTo": "avg",
+                        "spaceAggregation": "p90",
+                        "timeAggregation": "rate"
+                      }
+                    ]
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "e1a41831": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "trace operator",
+            "description": ""
+          },
+          "plugin": {
+            "kind": "SigNozTimeSeriesPanel",
+            "spec": {
+              "legend": {
+                "position": "right"
+              }
+            }
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozCompositeQuery",
+                  "spec": {
+                    "queries": [
+                      {
+                        "type": "builder_query",
+                        "spec": {
+                          "name": "A",
+                          "signal": "traces",
+                          "expression": "A",
+                          "aggregations": [
+                            {
+                              "expression": "count() "
+                            }
+                          ],
+                          "filter": {
+                            "expression": "service.name = 'sampleapp-gateway' "
+                          },
+                          "legend": "Gateway"
+                        }
+                      },
+                      {
+                        "type": "builder_query",
+                        "spec": {
+                          "name": "B",
+                          "signal": "traces",
+                          "expression": "B",
+                          "aggregations": [
+                            {
+                              "expression": "count() "
+                            }
+                          ],
+                          "filter": {
+                            "expression": "http.response.status_code = 200"
+                          },
+                          "legend": "$serviceName"
+                        }
+                      },
+                      {
+                        "type": "builder_trace_operator",
+                        "spec": {
+                          "name": "T1",
+                          "expression": "A -> B ",
+                          "aggregations": [
+                            {
+                              "expression": "count()",
+                              "alias": "request_count"
+                            }
+                          ]
+                        }
+                      }
+                    ]
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "f0d70491": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "no results in this promql",
+            "description": ""
+          },
+          "plugin": {
+            "kind": "SigNozTimeSeriesPanel",
+            "spec": {}
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozCompositeQuery",
+                  "spec": {
+                    "queries": [
+                      {
+                        "type": "promql",
+                        "spec": {
+                          "name": "A",
+                          "query": "sum(rate(flask_exporter_info[5m]))"
+                        }
+                      },
+                      {
+                        "type": "promql",
+                        "spec": {
+                          "name": "B",
+                          "query": "sum(increase(flask_exporter_info[5m]))"
+                        }
+                      }
+                    ]
+                  }
+                }
+              }
+            }
+          ]
+        }
+      },
+      "0e6eb4ca": {
+        "kind": "Panel",
+        "spec": {
+          "display": {
+            "name": "no results in this promql",
+            "description": ""
+          },
+          "plugin": {
+            "kind": "SigNozTimeSeriesPanel",
+            "spec": {}
+          },
+          "queries": [
+            {
+              "kind": "TimeSeriesQuery",
+              "spec": {
+                "plugin": {
+                  "kind": "SigNozPromQLQuery",
+                  "spec": {
+                    "name": "A",
+                    "query": "sum(rate(flask_exporter_info[5m]))"
+                  }
+                }
+              }
+            }
+          ]
+        }
+      }
+    },
+    "layouts": [
+      {
+        "kind": "Grid",
+        "spec": {
+          // ! where is row header?
+          // w: 12,
+          // 	minW: 12,
+          // 	minH: 1,
+          // 	maxH: 1,
+          // 	x: 0,
+          // 	h: 1,
+          // 	y: 0,
+          // These above properties are required for the headers only
+          // need collapsed state for a section
+          "items": [
+            {
+              // we need id here. could be injected from the frontend
+              "x": 0,
+              "y": 0,
+              "width": 6,
+              "height": 6,
+              // how easy would it be to introduce more here? like maxW, minW, static, etc. Do we want to introduce those from the start?
+              // "moved" might need experimenting
+              "content": {
+                "$ref": "#/spec/panels/24e2697b"
+              }
+            },
+            {
+              "x": 6,
+              "y": 0,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/ff2f72f1"
+              }
+            },
+            {
+              "x": 0,
+              "y": 6,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/011605e7"
+              }
+            },
+            {
+              "x": 6,
+              "y": 6,
+              "width": 6,
+              "height": 3,
+              "content": {
+                "$ref": "#/spec/panels/e23516fc"
+              }
+            },
+            {
+              "x": 6,
+              "y": 9,
+              "width": 6,
+              "height": 3,
+              "content": {
+                "$ref": "#/spec/panels/130c8d6b"
+              }
+            },
+            {
+              "x": 0,
+              "y": 12,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/246f7c6d"
+              }
+            },
+            {
+              "x": 6,
+              "y": 12,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/21f7d4d0"
+              }
+            },
+            {
+              "x": 0,
+              "y": 18,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/ad5fd556"
+              }
+            },
+            {
+              "x": 6,
+              "y": 18,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/f07b59ee"
+              }
+            },
+            {
+              "x": 0,
+              "y": 24,
+              "width": 12,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/e1a41831"
+              }
+            },
+            {
+              "x": 0,
+              "y": 30,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/f0d70491"
+              }
+            },
+            {
+              "x": 6,
+              "y": 30,
+              "width": 6,
+              "height": 6,
+              "content": {
+                "$ref": "#/spec/panels/0e6eb4ca"
+              }
+            }
+          ]
+        }
+      }
+    ]
+  }
+}

perses/howToDefineAPanel.md

@@ -0,0 +1,140 @@
+# Adding a new panel plugin
+
+This guide is for developers adding a new panel kind to the Perses schema. It covers the file structure you need to create, the shared types available in `common.cue`, and how to compose them into your panel's spec.
+
+---
+
+## 1. File structure
+
+Create a new directory under `schemas-wrapper/schemas/` named after your panel:
+
+```
+schemas-wrapper/schemas/your-panel-name/
+├── your-panel-name.cue    # Schema definition
+└── example.json           # Example JSON that validates against the schema
+```
+
+The CUE file must use `package model` and declare a `kind` and `spec`:
+
+```cue
+package model
+
+import "github.com/signoz/common"
+
+kind: "YourPanelName"
+spec: close({
+    // your fields here
+})
+```
+
+Register the panel in `schemas-wrapper/schemas/package.json` by adding an entry to the `plugins` array:
+
+```json
+{
+    "kind": "Panel",
+    "spec": {
+        "name": "YourPanelName"
+    }
+}
+```
+
+---
+
+## 2. Available shared types from `common.cue`
+
+These types are defined in `common/common.cue` and can be used via `import "github.com/signoz/common"`. Use them instead of redefining the same structures in your panel.
+
+## 3. Defining your spec
+
+Your spec is a `close({})` block containing the fields relevant to your panel. Pick from the shared types above and add panel-specific types as needed. Every field should be optional (suffixed with `?`) since a panel should be valid with just defaults.
+
+### Typical patterns
+
+**Panel with graphs** (has axes, legend, thresholds as reference lines):
+
+```cue
+spec: close({
+    visualization?:  #Visualization
+    formatting?:     #Formatting
+    axes?:           common.#Axes
+    legend?:         #Legend
+    contextLinks?:   [...common.#ContextLinkProps]
+    thresholds?:     [...common.#ThresholdWithLabel]
+})
+```
+
+**Panel with a single value** (no axes/legend, thresholds as conditional formatting):
+
+```cue
+spec: close({
+    visualization?:  #Visualization
+    formatting?:     #Formatting
+    contextLinks?:   [...common.#ContextLinkProps]
+    thresholds?:     [...common.#ComparisonThreshold]
+})
+```
+
+**Panel with custom data structure** (panel-specific fields only):
+
+```cue
+spec: close({
+    yourCustomConfig?: #YourCustomConfig
+    contextLinks?:     [...common.#ContextLinkProps]
+})
+```
+
+### Defining panel-local types
+
+Types that are specific to your panel (not reusable) should be defined in your CUE file, not in common. For example, `#Visualization` varies per panel because each panel has different rendering options:
+
+```cue
+#Visualization: {
+    timePreference?: common.#TimePreference
+    yourCustomFlag?: bool | *false
+}
+```
+
+If your panel needs a threshold type that extends a common one, embed it:
+
+```cue
+#YourThreshold: {
+    common.#ComparisonThreshold
+    extraField: string
+}
+```
+
+---
+
+## 4. Writing the example JSON
+
+Create an `example.json` that exercises all fields in your spec. This file is used for validation — `./validate.sh` will check it against your schema.
+
+```json
+{
+    "kind": "YourPanelName",
+    "spec": {
+        ...
+    }
+}
+```
+
+Also add at least one panel using your new kind to `examples/perses.json` so it gets validated as part of a full dashboard.
+
+---
+
+## 5. Validation
+
+Run `./validate.sh` from the `perses/` directory. It lints `examples/perses.json` against all registered schemas. If your schema or example has issues, the error will point to the specific field.
+
+---
+
+## Quick reference: existing panels and what they use
+
+| Field | Time-series | Bar-chart | Number | Pie | Table | Histogram | List |
+|-------|:-----------:|:---------:|:------:|:---:|:-----:|:---------:|:----:|
+| `visualization` | yes | yes | yes | yes | yes | — | — |
+| `formatting` | yes | yes | yes | yes | yes | — | — |
+| `axes` | yes | yes | — | — | — | — | — |
+| `legend` | yes | yes | — | yes | — | yes | — |
+| `contextLinks` | yes | yes | yes | yes | yes | yes | — |
+| `thresholds` | yes | yes | yes | — | yes | — | — |

perses/schemas-wrapper/schemas/mf-manifest.json

@@ -0,0 +1,9 @@
+{
+  "id": "signoz",
+  "name": "signoz",
+  "metaData": {
+    "buildInfo": {
+      "buildVersion": "0.0.1"
+    }
+  }
+}

perses/schemas-wrapper/schemas/others/signoz-datasource/signoz-datasource.cue

@@ -0,0 +1,11 @@
+package model
+
+kind: "SigNozDatasource"
+
+// SigNoz has a single built-in backend — the frontend already knows
+// the API endpoint, so there is no connection config to validate.
+// Add fields here if SigNoz ever supports multiple backends or
+// configurable API versions.
+spec: close({})
+
+ // this is required to override the default that always requires a spec

perses/schemas-wrapper/schemas/others/signoz-datasource/signoz-datasource.json

@@ -0,0 +1,4 @@
+{
+    "kind": "SigNozDatasource",
+    "spec": {}
+}

perses/schemas-wrapper/schemas/others/signoz-everything-chart/signoz-everything-chart.cue

@@ -0,0 +1,26 @@
+package model
+
+// Fields from IBaseWidget that are NOT in any dedicated panel CUE file.
+// This file exists as a reference only and will be removed in the end.
+// Needs to be removed from package.json as well.
+
+kind: "SigNozIgnoredFieldsChart"
+spec: close({
+    opacity?:              string        // not wired to chart rendering
+    nullZeroValues?:       string        // not wired to chart rendering
+    stepSize?:             number
+    columnWidths?:         [string]: number // not a config choice — persisted user-resized column widths
+    // "Chart Appearance" section in UI — could not find this section in the app.
+    // This is behind a boolean flag right now. Can help reproduce locally
+    // These 4 fields are gated behind panelTypeVs* constants (TIME_SERIES only).
+    lineInterpolation?:    #LineInterpolation
+    showPoints?:           bool
+    lineStyle?:            #LineStyle
+    fillMode?:             #FillMode
+})
+
+#LineInterpolation: "linear" | "spline" | "stepAfter" | "stepBefore"
+
+#LineStyle: "solid" | "dashed"
+
+#FillMode: "solid" | "gradient" | "none"

perses/schemas-wrapper/schemas/others/signoz-formula/signoz-formula.cue

@@ -0,0 +1,17 @@
+package model
+
+import "github.com/signoz/common"
+
+// Source: pkg/types/querybuildertypes/querybuildertypesv5/formula.go — QueryBuilderFormula
+kind: "SigNozFormula"
+spec: close({
+	name:          common.#QueryName
+	expression:    string
+	disabled?:     bool | *false
+	legend?:       string
+	limit?:        common.#Limit
+	having?:       common.#HavingExpression
+	stepInterval?: number
+	order?:        [...common.#OrderByItem]
+	functions?:      [...common.#Function]
+})

perses/schemas-wrapper/schemas/others/signoz-formula/signoz-formula.json

@@ -0,0 +1,8 @@
+{
+    "kind": "SigNozFormula",
+    "spec": {
+        "name": "F1",
+        "expression": "A / B * 100",
+        "legend": "Hit rate %"
+    }
+}

perses/schemas-wrapper/schemas/others/signoz-trace-operator/signoz-trace-operator.cue

@@ -0,0 +1,31 @@
+package model
+
+import "github.com/signoz/common"
+
+// Source: pkg/types/querybuildertypes/querybuildertypesv5/trace_operator.go — QueryBuilderTraceOperator
+// SigNozTraceOperator composes multiple trace BuilderQueries using
+// relational operators (=>, ->, &&, ||, NOT) to query trace relationships.
+// Signal is implicitly "traces" — all referenced queries must be trace queries.
+kind: "SigNozTraceOperator"
+spec: close({
+	name:       common.#QueryName
+	// Operator expression composing trace queries, e.g. "A => B && C".
+	expression: string & !=""
+	disabled?:  bool | *false
+
+	// Which query's spans to return (must be a query referenced in expression).
+	returnSpansFrom?: common.#QueryName
+
+	aggregations?: [...common.#ExpressionAggregation]
+	filter?:       common.#FilterExpression
+	groupBy?:      [...common.#GroupByItem]
+	order?:        [...common.#OrderByItem]
+	limit?:        common.#Limit
+	offset?:       common.#Offset
+	cursor?:       string
+	functions?:    [...common.#Function]
+	stepInterval?: number
+	having?:       common.#HavingExpression
+	legend?:       string
+	selectFields?: [...common.#TelemetryFieldKey]
+})

perses/schemas-wrapper/schemas/others/signoz-trace-operator/signoz-trace-operator.json

@@ -0,0 +1,19 @@
+{
+    "kind": "SigNozTraceOperator",
+    "spec": {
+        "name": "T1",
+        "expression": "A => B",
+        "returnSpansFrom": "A",
+        "aggregations": [
+            {
+                "expression": "count()",
+                "alias": "request_count"
+            }
+        ],
+        "filter": {
+            "expression": "service.name = 'frontend'"
+        },
+        "groupBy": [],
+        "order": []
+    }
+}

perses/schemas-wrapper/schemas/package.json

@@ -0,0 +1,136 @@
+{
+  "name": "@signoz/signoz-schemas",
+  "version": "0.0.1",
+  "description": "SigNoz schema plugins",
+  "perses": {
+    "schemasPath": ".",
+    "plugins": [
+      {
+        "kind": "Datasource",
+        "spec": {
+          "name": "SigNozDatasource"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozTimeSeriesPanel"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozBarChartPanel"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozNumberPanel"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozPieChartPanel"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozTablePanel"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozHistogramPanel"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozListPanel"
+        }
+      },
+      {
+        "kind": "Panel",
+        "spec": {
+          "name": "SigNozIgnoredFieldsChart"
+        }
+      },
+      {
+        "kind": "TimeSeriesQuery",
+        "spec": {
+          "name": "SigNozBuilderQuery"
+        }
+      },
+      {
+        "kind": "TimeSeriesQuery",
+        "spec": {
+          "name": "SigNozClickHouseSQL"
+        }
+      },
+      {
+        "kind": "TimeSeriesQuery",
+        "spec": {
+          "name": "SigNozPromQLQuery"
+        }
+      },
+      {
+        "kind": "TimeSeriesQuery",
+        "spec": {
+          "name": "SigNozCompositeQuery"
+        }
+      },
+      {
+        "kind": "TimeSeriesQuery",
+        "spec": {
+          "name": "SigNozTraceOperator"
+        }
+      },
+      {
+        "kind": "TimeSeriesQuery",
+        "spec": {
+          "name": "SigNozFormula"
+        }
+      },
+      {
+        "kind": "LogQuery",
+        "spec": {
+          "name": "SigNozBuilderQuery"
+        }
+      },
+      {
+        "kind": "TraceQuery",
+        "spec": {
+          "name": "SigNozBuilderQuery"
+        }
+      },
+      {
+        "kind": "Variable",
+        "spec": {
+          "name": "SigNozCustomVariable"
+        }
+      },
+      {
+        "kind": "Variable",
+        "spec": {
+          "name": "SigNozDynamicVariable"
+        }
+      },
+      {
+        "kind": "Variable",
+        "spec": {
+          "name": "SigNozQueryVariable"
+        }
+      },
+      {
+        "kind": "Variable",
+        "spec": {
+          "name": "SigNozTextboxVariable"
+        }
+      }
+    ]
+  }
+}

perses/schemas-wrapper/schemas/panels/review.md

@@ -0,0 +1,20 @@
+which panel type only caters to specific signals?
+metrics has no option for list panel type. also, no other querying types
+
+should the panel definition be linked to the query type?
+
+| type        | QB  | P   | CH  |
+| ----------- | --- | --- | --- |
+| Time Series | ✅  | ✅  | ✅  |
+| Number      | ✅  | ✅  | ✅  |
+| Table       | ✅  | ❌  | ✅  |
+| List        | ✅  | ❌  | ❌  |
+| Bar         | ✅  | ✅  | ✅  |
+| Pie         | ✅  | ❌  | ✅  |
+| Histogram   | ✅  | ✅  | ✅  |
+
+- unsure where this is used
+  stepSize - can be removed
+  renderColumnCell - used though - only table
+  customColumTitles - used though - only table
+  hiddenColumns - used though - only table

perses/schemas-wrapper/schemas/panels/signoz-bar-chart-panel/example.json

@@ -0,0 +1,40 @@
+{
+    "kind": "SigNozBarChartPanel",
+    "spec": {
+        "visualization": {
+            "timePreference": "globalTime",
+            "fillSpans": false,
+            "stackedBarChart": false
+        },
+        "formatting": {
+            "unit": "By",
+            "decimalPrecision": 2
+        },
+        "axes": {
+            "softMin": 0,
+            "softMax": 1000,
+            "isLogScale": false
+        },
+        "legend": {
+            "position": "bottom",
+            "customColors": {
+                "series-A": "#9ea5f7"
+            }
+        },
+        "contextLinks": [
+            {
+                "label": "View details",
+                "url": "http://localhost:8080/{{_service.name}}"
+            }
+        ],
+        "thresholds": [
+            {
+                "value": 500,
+                "unit": "By",
+                "color": "Red",
+                "format": "Text",
+                "label": "upper limit"
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/panels/signoz-bar-chart-panel/signoz-bar-chart-panel.cue

@@ -0,0 +1,31 @@
+package model
+
+import "github.com/signoz/common"
+
+kind: "SigNozBarChartPanel"
+spec: close({
+    // need to put more thought into this being optional?
+    visualization?:        #Visualization
+    formatting?:           #Formatting
+    axes?:                 common.#Axes
+    legend?:               #Legend
+    contextLinks?:         [...common.#ContextLinkProps]
+    thresholds?:           [...common.#ThresholdWithLabel]
+})
+
+#Visualization: {
+    timePreference?:  common.#TimePreference
+    fillSpans?:       bool | *false
+    stackedBarChart?: bool | *true
+}
+
+#Formatting: {
+    unit?:             string | *""
+    decimalPrecision?: common.#PrecisionOption
+}
+
+// can this be more composable?
+#Legend: {
+    position?:     common.#LegendPosition
+    customColors?: [string]: string
+}

perses/schemas-wrapper/schemas/panels/signoz-histogram-panel/example.json

@@ -0,0 +1,21 @@
+{
+    "kind": "SigNozHistogramPanel",
+    "spec": {
+        "histogramBuckets": {
+            "bucketCount": 60,
+            "bucketWidth": 100,
+            "mergeAllActiveQueries": true
+        },
+        "legend": {
+            "customColors": {
+                "series-A": "#9ea5f7"
+            }
+        },
+        "contextLinks": [
+            {
+                "label": "View histogram details",
+                "url": "http://localhost:8080/histogram"
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/panels/signoz-histogram-panel/signoz-histogram-panel.cue

@@ -0,0 +1,20 @@
+package model
+
+import "github.com/signoz/common"
+
+kind: "SigNozHistogramPanel"
+spec: close({
+    histogramBuckets?:     #HistogramBuckets
+    legend?:               #Legend
+    contextLinks?:         [...common.#ContextLinkProps]
+})
+
+#HistogramBuckets: {
+    bucketCount?:          number | *30
+    bucketWidth?:          number | *0
+    mergeAllActiveQueries?: bool | *false
+}
+
+#Legend: {
+    customColors?: [string]: string
+}

perses/schemas-wrapper/schemas/panels/signoz-list-panel/example.json

@@ -0,0 +1,54 @@
+{
+    "kind": "SigNozListPanel",
+    "spec": {
+        "selectedLogFields": [
+            {
+                "name": "timestamp",
+                "type": "log",
+                "dataType": ""
+            },
+            {
+                "name": "body",
+                "type": "log",
+                "dataType": ""
+            },
+            {
+                "name": "error",
+                "type": "",
+                "dataType": "string"
+            }
+        ],
+        "selectedTracesFields": [
+            {
+                "name": "service.name",
+                "signal": "traces",
+                "fieldContext": "resource",
+                "fieldDataType": "string"
+            },
+            {
+                "name": "name",
+                "signal": "traces",
+                "fieldContext": "span",
+                "fieldDataType": "string"
+            },
+            {
+                "name": "duration_nano",
+                "signal": "traces",
+                "fieldContext": "span",
+                "fieldDataType": ""
+            },
+            {
+                "name": "http_method",
+                "signal": "traces",
+                "fieldContext": "span",
+                "fieldDataType": ""
+            },
+            {
+                "name": "response_status_code",
+                "signal": "traces",
+                "fieldContext": "span",
+                "fieldDataType": ""
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/panels/signoz-list-panel/signoz-list-panel.cue

@@ -0,0 +1,15 @@
+package model
+
+import "github.com/signoz/common"
+
+kind: "SigNozListPanel"
+spec: close({
+    selectedLogFields?:    [...#LogField]
+    selectedTracesFields?: [...common.#TelemetryFieldKey]
+})
+
+#LogField: {
+    name:     string
+    type:     string
+    dataType: string
+}

perses/schemas-wrapper/schemas/panels/signoz-number-panel/example.json

@@ -0,0 +1,34 @@
+{
+    "kind": "SigNozNumberPanel",
+    "spec": {
+        "visualization": {
+            "timePreference": "globalTime"
+        },
+        "formatting": {
+            "unit": "none",
+            "decimalPrecision": 1
+        },
+        "contextLinks": [
+            {
+                "label": "View details",
+                "url": "http://localhost:8080/details"
+            }
+        ],
+        "thresholds": [
+            {
+                "value": 1200000,
+                "operator": ">",
+                "unit": "none",
+                "color": "Red",
+                "format": "Text"
+            },
+            {
+                "value": 1200000,
+                "operator": "<=",
+                "unit": "none",
+                "color": "Green",
+                "format": "Text"
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/panels/signoz-number-panel/signoz-number-panel.cue

@@ -0,0 +1,23 @@
+package model
+
+import "github.com/signoz/common"
+
+kind: "SigNozNumberPanel"
+spec: close({
+    visualization?:        #Visualization
+    formatting?:           #Formatting
+    contextLinks?:         [...common.#ContextLinkProps]
+    thresholds?:           [...common.#ComparisonThreshold]
+})
+
+#Visualization: {
+    timePreference?: common.#TimePreference
+}
+
+// where to store if alerts can be added or not?
+
+#Formatting: {
+    // mainly need to discuss optional fields
+    unit?:             string | *""
+    decimalPrecision?: common.#PrecisionOption
+}

perses/schemas-wrapper/schemas/panels/signoz-pie-chart-panel/example.json

@@ -0,0 +1,25 @@
+{
+    "kind": "SigNozPieChartPanel",
+    "spec": {
+        "visualization": {
+            "timePreference": "globalTime"
+        },
+        "formatting": {
+            "unit": "none",
+            "decimalPrecision": 1
+        },
+        "legend": {
+            "customColors": {
+                "\"201\"": "#2bc051",
+                "\"400\"": "#cc462e",
+                "\"500\"": "#ff0000"
+            }
+        },
+        "contextLinks": [
+            {
+                "label": "View breakdown",
+                "url": "http://localhost:8080/breakdown"
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/panels/signoz-pie-chart-panel/signoz-pie-chart-panel.cue

@@ -0,0 +1,24 @@
+package model
+
+import "github.com/signoz/common"
+
+kind: "SigNozPieChartPanel"
+spec: close({
+    visualization?:        #Visualization
+    formatting?:           #Formatting
+    legend?:               #Legend
+    contextLinks?:         [...common.#ContextLinkProps]
+})
+
+#Visualization: {
+    timePreference?: common.#TimePreference
+}
+
+#Formatting: {
+    unit?:             string | *""
+    decimalPrecision?: common.#PrecisionOption
+}
+
+#Legend: {
+    customColors?: [string]: string
+}

perses/schemas-wrapper/schemas/panels/signoz-table-panel/example.json

@@ -0,0 +1,30 @@
+{
+    "kind": "SigNozTablePanel",
+    "spec": {
+        "visualization": {
+            "timePreference": "globalTime"
+        },
+        "formatting": {
+            "columnUnits": {
+                "A": "s"
+            },
+            "decimalPrecision": 2
+        },
+        "contextLinks": [
+            {
+                "label": "View trace",
+                "url": "http://localhost:8080/trace/{{traceID}}"
+            }
+        ],
+        "thresholds": [
+            {
+                "value": 1,
+                "operator": ">",
+                "unit": "min",
+                "color": "Red",
+                "format": "Text",
+                "tableOptions": "A"
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/panels/signoz-table-panel/signoz-table-panel.cue

@@ -0,0 +1,31 @@
+package model
+
+import "github.com/signoz/common"
+
+kind: "SigNozTablePanel"
+spec: close({
+    visualization?:        #Visualization
+    formatting?:           #Formatting
+    contextLinks?:         [...common.#ContextLinkProps]
+    thresholds?:           [...#TableThreshold]
+})
+
+#Visualization: {
+    timePreference?: common.#TimePreference
+}
+
+#Formatting: {
+    columnUnits?:      [string]: string
+    decimalPrecision?: common.#PrecisionOption
+}
+
+#TableThreshold: {
+    common.#ComparisonThreshold
+    tableOptions: string
+}
+
+// missing
+columnWidths
+customColumnTitles
+hiddenColumns
+renderColumnCell - can be FE

perses/schemas-wrapper/schemas/panels/signoz-time-series-panel/example.json

@@ -0,0 +1,46 @@
+{
+    "kind": "SigNozTimeSeriesPanel",
+    "spec": {
+        "visualization": {
+            "timePreference": "globalTime",
+            "fillSpans": true
+        },
+        "formatting": {
+            "unit": "By",
+            "decimalPrecision": 3
+        },
+        "axes": {
+            "softMin": 0,
+            "softMax": 800,
+            "isLogScale": true
+        },
+        "legend": {
+            "position": "right",
+            "customColors": {
+                "{service.name=\"sampleapp-gateway\"}": "#9ea5f7"
+            }
+        },
+        "contextLinks": [
+            {
+                "label": "View service details",
+                "url": "http://localhost:8080/{{_service.name}}?dfddf=%7B%7Blimit%7D%7D"
+            }
+        ],
+        "thresholds": [
+            {
+                "value": 1024,
+                "unit": "By",
+                "color": "Red",
+                "format": "Text",
+                "label": "upper limit"
+            },
+            {
+                "value": 100,
+                "unit": "By",
+                "color": "Orange",
+                "format": "Text",
+                "label": "kinda bad"
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/panels/signoz-time-series-panel/signoz-time-series-panel.cue

@@ -0,0 +1,36 @@
+package model
+
+import "github.com/signoz/common"
+
+kind: "SigNozTimeSeriesPanel"
+spec: close({
+    visualization?:        #Visualization
+    formatting?:           #Formatting
+    axes?:                 common.#Axes
+    legend?:               #Legend
+    contextLinks?:         [...common.#ContextLinkProps]
+    thresholds?:           [...common.#ThresholdWithLabel]
+})
+
+#Visualization: {
+    timePreference?: common.#TimePreference
+    fillSpans?:      bool | *false
+}
+
+#Formatting: {
+    unit?:             string | *""
+    decimalPrecision?: common.#PrecisionOption
+}
+
+#Legend: {
+    position?:     common.#LegendPosition
+    // why call it customColors?
+    customColors?: [string]: string
+}
+
+// chart appearance
+ - fill mode
+ - line style
+ - line interpolation
+ - show points
+ - span gaps

perses/schemas-wrapper/schemas/querier/signoz-builder-query/signoz-builder-query.cue

@@ -0,0 +1,35 @@
+package model
+
+import "github.com/signoz/common"
+
+// Source: pkg/types/querybuildertypes/querybuildertypesv5/builder_query.go — QueryBuilderQuery
+kind: "SigNozBuilderQuery"
+spec: close({
+	name:       common.#QueryName
+	signal:     "metrics" | "logs" | "traces"
+	expression: string
+	disabled?:  bool | *false
+
+	aggregations?:   [...common.#Aggregation]
+	filter?:         common.#FilterExpression
+	groupBy?:        [...common.#GroupByItem]
+	order?:          [...common.#OrderByItem]
+	selectFields?:   [...common.#TelemetryFieldKey]
+	limit?:          common.#Limit
+	limitBy?:        #LimitBy
+	offset?:         common.#Offset
+	cursor?:         string
+	having?:         common.#HavingExpression
+	// secondaryAggregations not added — not yet implemented.
+	functions?:      [...common.#Function]
+	legend?:         string
+	stepInterval?:   number
+	reduceTo?:       common.#ReduceTo
+	pageSize?:       int & >=1
+	source?:         string
+})
+
+#LimitBy: close({
+	keys:  [...string]
+	value: string
+})

perses/schemas-wrapper/schemas/querier/signoz-builder-query/signoz-builder-query.json

@@ -0,0 +1,24 @@
+{
+    "kind": "SigNozBuilderQuery",
+    "spec": {
+        "name": "A",
+        "signal": "metrics",
+        "expression": "A",
+        "aggregations": [
+            {
+                "metricName": "redis_keyspace_hits",
+                "timeAggregation": "rate",
+                "spaceAggregation": "sum",
+                "reduceTo": "sum"
+            }
+        ],
+        "filter": {
+            "expression": "host_name IN $host_name"
+        },
+        "groupBy": [],
+        "order": [],
+        "disabled": false,
+        "legend": "Hit/s across all hosts",
+        "stepInterval": 60
+    }
+}

perses/schemas-wrapper/schemas/querier/signoz-clickhouse-sql/signoz-clickhouse-sql.cue

@@ -0,0 +1,13 @@
+package model
+
+import "github.com/signoz/common"
+
+// Source: pkg/types/querybuildertypes/querybuildertypesv5/clickhouse_query.go — ClickHouseQuery
+kind: "SigNozClickHouseSQL"
+spec: close({
+	name:      common.#QueryName
+	query:     string & !=""
+	// required / optional
+	disabled?: bool | *false
+	legend?:   string
+})

perses/schemas-wrapper/schemas/querier/signoz-clickhouse-sql/signoz-clickhouse-sql.json

@@ -0,0 +1,9 @@
+{
+    "kind": "SigNozClickHouseSQL",
+    "spec": {
+        "name": "A",
+        "query": "SELECT toStartOfInterval(timestamp, INTERVAL 1 MINUTE) AS ts, count() AS total FROM signoz_logs.distributed_logs GROUP BY ts ORDER BY ts",
+        "disabled": false,
+        "legend": "Log count"
+    }
+}

perses/schemas-wrapper/schemas/querier/signoz-composite-query/signoz-composite-query.cue

@@ -0,0 +1,47 @@
+package model
+
+import (
+	bq "github.com/signoz/schemas-wrapper/schemas/signoz-builder-query:model"
+	f "github.com/signoz/schemas-wrapper/schemas/signoz-formula:model"
+	to "github.com/signoz/schemas-wrapper/schemas/signoz-trace-operator:model"
+	pql "github.com/signoz/schemas-wrapper/schemas/signoz-promql:model"
+	ch "github.com/signoz/schemas-wrapper/schemas/signoz-clickhouse-sql:model"
+)
+
+// Source: pkg/types/querybuildertypes/querybuildertypesv5/req.go — CompositeQuery
+// SigNozCompositeQuery groups multiple query plugins into a single
+// query request. Each entry is a typed envelope whose spec is
+// validated by the corresponding plugin schema.
+
+// this is to be used when there are multiple queries in a panel
+// in most cases, there will be only one query, and there it is a better idea to 
+// use the corresponding kind for that query instead of this composite query
+kind: "SigNozCompositeQuery"
+spec: close({
+	queries: [...#QueryEnvelope]
+})
+
+// QueryEnvelope wraps a single query plugin with a type discriminator.
+
+// need to verify the types internally are marked required where necessary, so that we can make them optional here and not have to worry about it at the top level
+#QueryEnvelope:
+	close({
+		type: "builder_query",
+		spec: bq.spec
+	}) |
+	close({
+		type: "builder_formula",
+		spec: f.spec
+	}) |
+	close({
+		type: "builder_trace_operator",
+		spec: to.spec
+	}) |
+	close({
+		type: "promql",
+		spec: pql.spec
+	}) |
+	close({
+		type: "clickhouse_sql",
+		spec: ch.spec
+	})

perses/schemas-wrapper/schemas/querier/signoz-composite-query/signoz-composite-query.json

@@ -0,0 +1,53 @@
+{
+    "kind": "SigNozCompositeQuery",
+    "spec": {
+        "queries": [
+            {
+                "type": "builder_query",
+                "spec": {
+                    "name": "A",
+                    "signal": "metrics",
+                    "expression": "A",
+                    "aggregations": [
+                        {
+                            "metricName": "redis_keyspace_hits",
+                            "timeAggregation": "rate",
+                            "spaceAggregation": "sum",
+                            "reduceTo": "sum"
+                        }
+                    ],
+                    "filter": {
+                        "expression": "host_name IN $host_name"
+                    }
+                }
+            },
+            {
+                "type": "builder_query",
+                "spec": {
+                    "name": "B",
+                    "signal": "metrics",
+                    "expression": "B",
+                    "aggregations": [
+                        {
+                            "metricName": "redis_keyspace_misses",
+                            "timeAggregation": "rate",
+                            "spaceAggregation": "sum",
+                            "reduceTo": "sum"
+                        }
+                    ],
+                    "filter": {
+                        "expression": "host_name IN $host_name"
+                    }
+                }
+            },
+            {
+                "type": "builder_formula",
+                "spec": {
+                    "name": "F1",
+                    "expression": "A / (A + B) * 100",
+                    "legend": "Hit rate %"
+                }
+            }
+        ]
+    }
+}

perses/schemas-wrapper/schemas/querier/signoz-promql/signoz-promql.cue

@@ -0,0 +1,15 @@
+package model
+
+import "github.com/signoz/common"
+
+// Source: pkg/types/querybuildertypes/querybuildertypesv5/prom_query.go — PromQuery
+kind: "SigNozPromQLQuery"
+spec: close({
+	name:      common.#QueryName
+	query:     string & !=""
+	disabled?: bool | *false
+	legend?:   string
+	// where are the below items coming from? Don't see types for it. How to arrive at this?
+	step?:     =~"^([0-9]+(\\.[0-9]+)?(ns|us|µs|ms|s|m|h))+$" | number
+	stats?:    bool
+})

perses/schemas-wrapper/schemas/querier/signoz-promql/signoz-promql.json

@@ -0,0 +1,9 @@
+{
+    "kind": "SigNozPromQLQuery",
+    "spec": {
+        "name": "A",
+        "query": "rate(http_requests_total{status=\"200\"}[5m])",
+        "disabled": false,
+        "legend": "{{method}} {{path}}"
+    }
+}

perses/schemas-wrapper/schemas/review.md

@@ -0,0 +1 @@
+creating folders based on type is helpful instead of a flat folder structure. It is how you would visualise the dashboard as components

perses/schemas-wrapper/schemas/variables/signoz-custom-variable/signoz-custom-variable.cue

@@ -0,0 +1,10 @@
+package model
+
+import "github.com/signoz/common"
+
+// defaultValue lives on the Perses ListVariable wrapper (spec level).
+kind: "SigNozCustomVariable"
+spec: close({
+	customValue: =~"^[^,]+(,[^,]+)*$"
+	sort?:       common.#VariableSortOrder
+})

perses/schemas-wrapper/schemas/variables/signoz-custom-variable/signoz-custom-variable.json

@@ -0,0 +1,7 @@
+{
+    "kind": "SigNozCustomVariable",
+    "spec": {
+        "customValue": "production,staging,development",
+        "sort": "disabled"
+    }
+}

perses/schemas-wrapper/schemas/variables/signoz-dynamic-variable/signoz-dynamic-variable.cue

@@ -0,0 +1,12 @@
+package model
+
+import "github.com/signoz/common"
+
+// defaultValue lives on the Perses ListVariable wrapper (spec level).
+kind: "SigNozDynamicVariable"
+spec: close({
+	name: string
+	source:    string
+	sort?: common.#VariableSortOrder
+	// where does attribute go here?
+})

perses/schemas-wrapper/schemas/variables/signoz-dynamic-variable/signoz-dynamic-variable.json

@@ -0,0 +1,8 @@
+{
+    "kind": "SigNozDynamicVariable",
+    "spec": {
+        "name": "host_name",
+        "source": "metrics",
+        "sort": "asc"
+    }
+}

perses/schemas-wrapper/schemas/variables/signoz-query-variable/signoz-query-variable.cue

@@ -0,0 +1,13 @@
+package model
+
+import "github.com/signoz/common"
+
+// defaultValue lives on the Perses ListVariable wrapper (spec level).
+kind: "SigNozQueryVariable"
+spec: close({
+	queryValue: string
+	sort?:      common.#VariableSortOrder
+	// This should not be optional
+	// what happens to the existing https://perses.dev/perses/docs/dac/go/variable/#sortingby inside the ListVariable spec? Do we move that to the plugin spec level?
+	// same for other list variable cues
+})

perses/schemas-wrapper/schemas/variables/signoz-query-variable/signoz-query-variable.json

@@ -0,0 +1,7 @@
+{
+    "kind": "SigNozQueryVariable",
+    "spec": {
+        "queryValue": "SELECT DISTINCT host_name FROM signoz_metrics.distributed_time_series_v4_1day WHERE metric_name = 'redis_cpu_time'",
+        "sort": "asc"
+    }
+}

perses/schemas-wrapper/schemas/variables/signoz-textbox-variable/signoz-textbox-variable.cue

@@ -0,0 +1,7 @@
+package model
+
+// Shouldn't the below line be TextVariable?
+// are any of the variables inside this required? How would that validation work?
+// defaultValue lives on the Perses ListVariable wrapper (spec level).
+kind: "SigNozTextboxVariable"
+spec: close({})

perses/schemas-wrapper/schemas/variables/signoz-textbox-variable/signoz-textbox-variable.json

@@ -0,0 +1,4 @@
+{
+    "kind": "SigNozTextboxVariable",
+    "spec": {}
+}

perses/validate.sh

@@ -0,0 +1,2 @@
+percli lint -f ./examples/perses.json --plugin.path ./schemas-wrapper
+rm ./schemas-wrapper/plugin-modules.json

pkg/apiserver/signozapiserver/cloudintegration.go

@@ -0,0 +1,216 @@
+package signozapiserver
+
+import (
+	"net/http"
+
+	"github.com/SigNoz/signoz/pkg/http/handler"
+	"github.com/SigNoz/signoz/pkg/types"
+	citypes "github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
+	"github.com/gorilla/mux"
+)
+
+func (provider *provider) addCloudIntegrationRoutes(router *mux.Router) error {
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/accounts", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.CreateAccount),
+		handler.OpenAPIDef{
+			ID:                  "CreateAccount",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Create account",
+			Description:         "This endpoint creates a new cloud integration account for the specified cloud provider",
+			Request:             new(citypes.PostableConnectionArtifact),
+			RequestContentType:  "application/json",
+			Response:            new(citypes.GettableAccountWithArtifact),
+			ResponseContentType: "application/json",
+			SuccessStatusCode:   http.StatusOK,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodPost).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/accounts", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.ListAccounts),
+		handler.OpenAPIDef{
+			ID:                  "ListAccounts",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "List accounts",
+			Description:         "This endpoint lists the accounts for the specified cloud provider",
+			Request:             nil,
+			RequestContentType:  "",
+			Response:            new(citypes.GettableAccounts),
+			ResponseContentType: "application/json",
+			SuccessStatusCode:   http.StatusOK,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodGet).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/accounts/{id}", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.GetAccount),
+		handler.OpenAPIDef{
+			ID:                  "GetAccount",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Get account",
+			Description:         "This endpoint gets an account for the specified cloud provider",
+			Request:             nil,
+			RequestContentType:  "",
+			Response:            new(citypes.GettableAccount),
+			ResponseContentType: "application/json",
+			SuccessStatusCode:   http.StatusOK,
+			ErrorStatusCodes:    []int{http.StatusBadRequest, http.StatusNotFound},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodGet).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/accounts/{id}", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.UpdateAccount),
+		handler.OpenAPIDef{
+			ID:                  "UpdateAccount",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Update account",
+			Description:         "This endpoint updates an account for the specified cloud provider",
+			Request:             new(citypes.UpdatableAccount),
+			RequestContentType:  "application/json",
+			Response:            nil,
+			ResponseContentType: "",
+			SuccessStatusCode:   http.StatusNoContent,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodPut).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/accounts/{id}", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.DisconnectAccount),
+		handler.OpenAPIDef{
+			ID:                  "DisconnectAccount",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Disconnect account",
+			Description:         "This endpoint disconnects an account for the specified cloud provider",
+			Request:             nil,
+			RequestContentType:  "",
+			Response:            nil,
+			ResponseContentType: "",
+			SuccessStatusCode:   http.StatusNoContent,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodDelete).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/services", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.ListServicesMetadata),
+		handler.OpenAPIDef{
+			ID:                  "ListServicesMetadata",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "List services metadata",
+			Description:         "This endpoint lists the services metadata for the specified cloud provider",
+			Request:             nil,
+			RequestContentType:  "",
+			Response:            new(citypes.GettableServicesMetadata),
+			ResponseContentType: "application/json",
+			SuccessStatusCode:   http.StatusOK,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodGet).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/services/{service_id}", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.GetService),
+		handler.OpenAPIDef{
+			ID:                  "GetService",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Get service",
+			Description:         "This endpoint gets a service for the specified cloud provider",
+			Request:             nil,
+			RequestContentType:  "",
+			Response:            new(citypes.GettableService),
+			ResponseContentType: "application/json",
+			SuccessStatusCode:   http.StatusOK,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodGet).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/services/{service_id}", handler.New(
+		provider.authZ.AdminAccess(provider.cloudIntegrationHandler.UpdateService),
+		handler.OpenAPIDef{
+			ID:                  "UpdateService",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Update service",
+			Description:         "This endpoint updates a service for the specified cloud provider",
+			Request:             new(citypes.UpdatableService),
+			RequestContentType:  "application/json",
+			Response:            nil,
+			ResponseContentType: "",
+			SuccessStatusCode:   http.StatusNoContent,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleAdmin),
+		},
+	)).Methods(http.MethodPut).GetError(); err != nil {
+		return err
+	}
+
+	// Agent check-in endpoint is kept same as older one to maintain backward compatibility with already deployed agents.
+	// In the future, this endpoint will be deprecated and a new endpoint will be introduced for consistency with above endpoints.
+	if err := router.Handle("/api/v1/cloud-integrations/{cloud_provider}/agent-check-in", handler.New(
+		provider.authZ.ViewAccess(provider.cloudIntegrationHandler.AgentCheckIn),
+		handler.OpenAPIDef{
+			ID:                  "AgentCheckInDeprecated",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Agent check-in",
+			Description:         "[Deprecated] This endpoint is called by the deployed agent to check in",
+			Request:             new(citypes.PostableAgentCheckInRequest),
+			RequestContentType:  "application/json",
+			Response:            new(citypes.GettableAgentCheckInResponse),
+			ResponseContentType: "application/json",
+			SuccessStatusCode:   http.StatusOK,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          true,                                 // this endpoint will be deprecated in future
+			SecuritySchemes:     newSecuritySchemes(types.RoleViewer), // agent role is viewer
+		},
+	)).Methods(http.MethodPost).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/cloud_integrations/{cloud_provider}/accounts/check_in", handler.New(
+		provider.authZ.ViewAccess(provider.cloudIntegrationHandler.AgentCheckIn),
+		handler.OpenAPIDef{
+			ID:                  "AgentCheckIn",
+			Tags:                []string{"cloudintegration"},
+			Summary:             "Agent check-in",
+			Description:         "This endpoint is called by the deployed agent to check in",
+			Request:             new(citypes.PostableAgentCheckInRequest),
+			RequestContentType:  "application/json",
+			Response:            new(citypes.GettableAgentCheckInResponse),
+			ResponseContentType: "application/json",
+			SuccessStatusCode:   http.StatusOK,
+			ErrorStatusCodes:    []int{},
+			Deprecated:          false,
+			SecuritySchemes:     newSecuritySchemes(types.RoleViewer), // agent role is viewer
+		},
+	)).Methods(http.MethodPost).GetError(); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/apiserver/signozapiserver/provider.go

@@ -12,6 +12,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/http/handler"
 	"github.com/SigNoz/signoz/pkg/http/middleware"
 	"github.com/SigNoz/signoz/pkg/modules/authdomain"
+	"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard"
 	"github.com/SigNoz/signoz/pkg/modules/fields"
 	"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
@@ -30,29 +31,30 @@ import (
 )
 
 type provider struct {
-	config                 apiserver.Config
-	settings               factory.ScopedProviderSettings
-	router                 *mux.Router
-	authZ                  *middleware.AuthZ
-	orgHandler             organization.Handler
-	userHandler            user.Handler
-	sessionHandler         session.Handler
-	authDomainHandler      authdomain.Handler
-	preferenceHandler      preference.Handler
-	globalHandler          global.Handler
-	promoteHandler         promote.Handler
-	flaggerHandler         flagger.Handler
-	dashboardModule        dashboard.Module
-	dashboardHandler       dashboard.Handler
-	metricsExplorerHandler metricsexplorer.Handler
-	gatewayHandler         gateway.Handler
-	fieldsHandler          fields.Handler
-	authzHandler           authz.Handler
-	rawDataExportHandler   rawdataexport.Handler
-	zeusHandler            zeus.Handler
-	querierHandler         querier.Handler
-	serviceAccountHandler  serviceaccount.Handler
-	factoryHandler         factory.Handler
+	config                  apiserver.Config
+	settings                factory.ScopedProviderSettings
+	router                  *mux.Router
+	authZ                   *middleware.AuthZ
+	orgHandler              organization.Handler
+	userHandler             user.Handler
+	sessionHandler          session.Handler
+	authDomainHandler       authdomain.Handler
+	preferenceHandler       preference.Handler
+	globalHandler           global.Handler
+	promoteHandler          promote.Handler
+	flaggerHandler          flagger.Handler
+	dashboardModule         dashboard.Module
+	dashboardHandler        dashboard.Handler
+	metricsExplorerHandler  metricsexplorer.Handler
+	gatewayHandler          gateway.Handler
+	fieldsHandler           fields.Handler
+	authzHandler            authz.Handler
+	rawDataExportHandler    rawdataexport.Handler
+	zeusHandler             zeus.Handler
+	querierHandler          querier.Handler
+	serviceAccountHandler   serviceaccount.Handler
+	factoryHandler          factory.Handler
+	cloudIntegrationHandler cloudintegration.Handler
 }
 
 func NewFactory(
@@ -77,6 +79,7 @@ func NewFactory(
 	querierHandler querier.Handler,
 	serviceAccountHandler serviceaccount.Handler,
 	factoryHandler factory.Handler,
+	cloudIntegrationHandler cloudintegration.Handler,
 ) factory.ProviderFactory[apiserver.APIServer, apiserver.Config] {
 	return factory.NewProviderFactory(factory.MustNewName("signoz"), func(ctx context.Context, providerSettings factory.ProviderSettings, config apiserver.Config) (apiserver.APIServer, error) {
 		return newProvider(
@@ -104,6 +107,7 @@ func NewFactory(
 			querierHandler,
 			serviceAccountHandler,
 			factoryHandler,
+			cloudIntegrationHandler,
 		)
 	})
 }
@@ -133,33 +137,35 @@ func newProvider(
 	querierHandler querier.Handler,
 	serviceAccountHandler serviceaccount.Handler,
 	factoryHandler factory.Handler,
+	cloudIntegrationHandler cloudintegration.Handler,
 ) (apiserver.APIServer, error) {
 	settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/apiserver/signozapiserver")
 	router := mux.NewRouter().UseEncodedPath()
 
 	provider := &provider{
-		config:                 config,
-		settings:               settings,
-		router:                 router,
-		orgHandler:             orgHandler,
-		userHandler:            userHandler,
-		sessionHandler:         sessionHandler,
-		authDomainHandler:      authDomainHandler,
-		preferenceHandler:      preferenceHandler,
-		globalHandler:          globalHandler,
-		promoteHandler:         promoteHandler,
-		flaggerHandler:         flaggerHandler,
-		dashboardModule:        dashboardModule,
-		dashboardHandler:       dashboardHandler,
-		metricsExplorerHandler: metricsExplorerHandler,
-		gatewayHandler:         gatewayHandler,
-		fieldsHandler:          fieldsHandler,
-		authzHandler:           authzHandler,
-		rawDataExportHandler:   rawDataExportHandler,
-		zeusHandler:            zeusHandler,
-		querierHandler:         querierHandler,
-		serviceAccountHandler:  serviceAccountHandler,
-		factoryHandler:         factoryHandler,
+		config:                  config,
+		settings:                settings,
+		router:                  router,
+		orgHandler:              orgHandler,
+		userHandler:             userHandler,
+		sessionHandler:          sessionHandler,
+		authDomainHandler:       authDomainHandler,
+		preferenceHandler:       preferenceHandler,
+		globalHandler:           globalHandler,
+		promoteHandler:          promoteHandler,
+		flaggerHandler:          flaggerHandler,
+		dashboardModule:         dashboardModule,
+		dashboardHandler:        dashboardHandler,
+		metricsExplorerHandler:  metricsExplorerHandler,
+		gatewayHandler:          gatewayHandler,
+		fieldsHandler:           fieldsHandler,
+		authzHandler:            authzHandler,
+		rawDataExportHandler:    rawDataExportHandler,
+		zeusHandler:             zeusHandler,
+		querierHandler:          querierHandler,
+		serviceAccountHandler:   serviceAccountHandler,
+		factoryHandler:          factoryHandler,
+		cloudIntegrationHandler: cloudIntegrationHandler,
 	}
 
 	provider.authZ = middleware.NewAuthZ(settings.Logger(), orgGetter, authz)
@@ -252,6 +258,10 @@ func (provider *provider) AddToRouter(router *mux.Router) error {
 		return err
 	}
 
+	if err := provider.addCloudIntegrationRoutes(router); err != nil {
+		return err
+	}
+
 	return nil
 }
 

pkg/modules/cloudintegration/cloudintegration.go

@@ -53,7 +53,7 @@ type Module interface {
 }
 
 type Handler interface {
-	GetConnectionArtifact(http.ResponseWriter, *http.Request)
+	CreateAccount(http.ResponseWriter, *http.Request)
 	ListAccounts(http.ResponseWriter, *http.Request)
 	GetAccount(http.ResponseWriter, *http.Request)
 	UpdateAccount(http.ResponseWriter, *http.Request)

pkg/modules/cloudintegration/implcloudintegration/handler.go

@@ -0,0 +1,58 @@
+package implcloudintegration
+
+import (
+	"net/http"
+
+	"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
+)
+
+type handler struct{}
+
+func NewHandler() cloudintegration.Handler {
+	return &handler{}
+}
+
+func (handler *handler) CreateAccount(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) ListAccounts(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) GetAccount(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) UpdateAccount(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) DisconnectAccount(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) ListServicesMetadata(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) GetService(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) UpdateService(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}
+
+func (handler *handler) AgentCheckIn(writer http.ResponseWriter, request *http.Request) {
+	// TODO implement me
+	panic("implement me")
+}

pkg/prometheus/config.go

@@ -3,6 +3,7 @@ package prometheus
 import (
 	"time"
 
+	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
 )
 
@@ -20,6 +21,9 @@ type Config struct {
 	//
 	// If not set, the prometheus default is used (currently 5m).
 	LookbackDelta time.Duration `mapstructure:"lookback_delta"`
+
+	// Timeout is the maximum time a query is allowed to run before being aborted.
+	Timeout time.Duration `mapstructure:"timeout"`
 }
 
 func NewConfigFactory() factory.ConfigFactory {
@@ -33,10 +37,14 @@ func newConfig() factory.Config {
 			Path:          "",
 			MaxConcurrent: 20,
 		},
+		Timeout: 2 * time.Minute,
 	}
 }
 
 func (c Config) Validate() error {
+	if c.Timeout <= 0 {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "prometheus::timeout must be greater than 0")
+	}
 	return nil
 }
 

pkg/prometheus/constants.go

@@ -1,3 +0,0 @@
-package prometheus
-
-const FingerprintAsPromLabelName = "fingerprint"

pkg/prometheus/engine.go

@@ -2,7 +2,6 @@ package prometheus
 
 import (
 	"log/slog"
-	"time"
 
 	"github.com/prometheus/prometheus/promql"
 )
@@ -21,7 +20,7 @@ func NewEngine(logger *slog.Logger, cfg Config) *Engine {
 		Logger:             logger,
 		Reg:                nil,
 		MaxSamples:         5_0000_000,
-		Timeout:            2 * time.Minute,
+		Timeout:            cfg.Timeout,
 		ActiveQueryTracker: activeQueryTracker,
 		LookbackDelta:      cfg.LookbackDelta,
 	})

pkg/prometheus/label.go

@@ -6,6 +6,8 @@ import (
 	"github.com/prometheus/prometheus/promql"
 )
 
+const FingerprintAsPromLabelName string = "fingerprint"
+
 func RemoveExtraLabels(res *promql.Result, labelsToRemove ...string) error {
 	if len(labelsToRemove) == 0 || res == nil {
 		return nil

pkg/querier/builder_query.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/base64"
 	"fmt"
+	"log/slog"
 	"strconv"
 	"strings"
 	"time"
@@ -11,6 +12,7 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/telemetrytraces"
 	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
 	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -18,6 +20,7 @@ import (
 )
 
 type builderQuery[T any] struct {
+	logger         *slog.Logger
 	telemetryStore telemetrystore.TelemetryStore
 	stmtBuilder    qbtypes.StatementBuilder[T]
 	spec           qbtypes.QueryBuilderQuery[T]
@@ -31,6 +34,7 @@ type builderQuery[T any] struct {
 var _ qbtypes.Query = (*builderQuery[any])(nil)
 
 func newBuilderQuery[T any](
+	logger *slog.Logger,
 	telemetryStore telemetrystore.TelemetryStore,
 	stmtBuilder qbtypes.StatementBuilder[T],
 	spec qbtypes.QueryBuilderQuery[T],
@@ -39,6 +43,7 @@ func newBuilderQuery[T any](
 	variables map[string]qbtypes.VariableItem,
 ) *builderQuery[T] {
 	return &builderQuery[T]{
+		logger:         logger,
 		telemetryStore: telemetryStore,
 		stmtBuilder:    stmtBuilder,
 		spec:           spec,
@@ -305,6 +310,45 @@ func (q *builderQuery[T]) executeWindowList(ctx context.Context) (*qbtypes.Resul
 	totalBytes := uint64(0)
 	start := time.Now()
 
+	// Check if filter contains trace_id(s) and optimize time range if needed
+	if q.spec.Signal == telemetrytypes.SignalTraces &&
+		q.spec.Filter != nil && q.spec.Filter.Expression != "" {
+
+		traceIDs, found := telemetrytraces.ExtractTraceIDsFromFilter(q.spec.Filter.Expression)
+		if found && len(traceIDs) > 0 {
+			finder := telemetrytraces.NewTraceTimeRangeFinder(q.telemetryStore)
+
+			traceStart, traceEnd, ok := finder.GetTraceTimeRangeMulti(ctx, traceIDs)
+			traceStartMS := uint64(traceStart) / 1_000_000
+			traceEndMS := uint64(traceEnd) / 1_000_000
+			if !ok {
+				q.logger.DebugContext(ctx, "failed to get trace time range", slog.Any("trace_ids", traceIDs))
+			} else if traceStartMS > 0 && traceEndMS > 0 {
+				// no overlap — nothing to return
+				if uint64(traceStartMS) > toMS || uint64(traceEndMS) < fromMS {
+					return &qbtypes.Result{
+						Type: qbtypes.RequestTypeRaw,
+						Value: &qbtypes.RawData{
+							QueryName: q.spec.Name,
+						},
+						Stats: qbtypes.ExecStats{
+							DurationMS: uint64(time.Since(start).Milliseconds()),
+						},
+					}, nil
+				}
+
+				// clamp window to trace time range before bucketing
+				if uint64(traceStartMS) > fromMS {
+					fromMS = uint64(traceStartMS)
+				}
+				if uint64(traceEndMS) < toMS {
+					toMS = uint64(traceEndMS)
+				}
+				q.logger.DebugContext(ctx, "optimized time range for traces", slog.Any("trace_ids", traceIDs), slog.Uint64("start", fromMS), slog.Uint64("end", toMS))
+			}
+		}
+	}
+
 	// Get buckets and reverse them for ascending order
 	buckets := makeBuckets(fromMS, toMS)
 	if isAsc {

pkg/querier/promql_query.go

@@ -36,6 +36,28 @@ var unquotedDottedNamePattern = regexp.MustCompile(`(?:^|[{,(\s])([a-zA-Z_][a-zA
 // This is a common mistake when migrating to UTF-8 syntax.
 var quotedMetricOutsideBracesPattern = regexp.MustCompile(`"([^"]+)"\s*\{`)
 
+// tryEnhancePromQLExecError attempts to convert a PromQL execution error into
+// a properly typed error. Returns nil if the error is not a recognized execution error.
+func tryEnhancePromQLExecError(execErr error) error {
+	var eqc promql.ErrQueryCanceled
+	var eqt promql.ErrQueryTimeout
+	var es promql.ErrStorage
+	switch {
+	case errors.As(execErr, &eqc):
+		return errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled").WithAdditional(eqc.Error())
+	case errors.As(execErr, &eqt):
+		return errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timed out").WithAdditional(eqt.Error())
+	case errors.Is(execErr, context.DeadlineExceeded):
+		return errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timed out")
+	case errors.Is(execErr, context.Canceled):
+		return errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled")
+	case errors.As(execErr, &es):
+		return errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", execErr)
+	default:
+		return nil
+	}
+}
+
 // enhancePromQLError adds helpful context to PromQL parse errors,
 // particularly for UTF-8 syntax migration issues where metric and label
 // names containing dots need to be quoted.
@@ -213,27 +235,20 @@ func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {
 		time.Unix(0, end),
 		q.query.Step.Duration,
 	)
-
 	if err != nil {
+		// NewRangeQuery can fail with execution errors (e.g. context deadline exceeded)
+		// during the query queue/scheduling stage, not just parse errors.
+		if err := tryEnhancePromQLExecError(err); err != nil {
+			return nil, err
+		}
+
 		return nil, enhancePromQLError(query, err)
 	}
 
 	res := qry.Exec(ctx)
 	if res.Err != nil {
-		var eqc promql.ErrQueryCanceled
-		var eqt promql.ErrQueryTimeout
-		var es promql.ErrStorage
-		switch {
-		case errors.As(res.Err, &eqc):
-			return nil, errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled")
-		case errors.As(res.Err, &eqt):
-			return nil, errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timeout")
-		case errors.As(res.Err, &es):
-			return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", res.Err)
-		}
-
-		if errors.Is(res.Err, context.Canceled) {
-			return nil, errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled")
+		if err := tryEnhancePromQLExecError(res.Err); err != nil {
+			return nil, err
 		}
 
 		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", res.Err)

pkg/querier/querier.go

@@ -353,13 +353,13 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 			case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]:
 				spec.ShiftBy = extractShiftFromBuilderQuery(spec)
 				timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
-				bq := newBuilderQuery(q.telemetryStore, q.traceStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
+				bq := newBuilderQuery(q.logger, q.telemetryStore, q.traceStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
 				queries[spec.Name] = bq
 				steps[spec.Name] = spec.StepInterval
 			case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
 				spec.ShiftBy = extractShiftFromBuilderQuery(spec)
 				timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: req.Start, To: req.End}, req.RequestType)
-				bq := newBuilderQuery(q.telemetryStore, q.logStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
+				bq := newBuilderQuery(q.logger, q.telemetryStore, q.logStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
 				queries[spec.Name] = bq
 				steps[spec.Name] = spec.StepInterval
 			case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
@@ -397,9 +397,9 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 
 				if spec.Source == telemetrytypes.SourceMeter {
 					event.Source = telemetrytypes.SourceMeter.StringValue()
-					bq = newBuilderQuery(q.telemetryStore, q.meterStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
+					bq = newBuilderQuery(q.logger, q.telemetryStore, q.meterStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
 				} else {
-					bq = newBuilderQuery(q.telemetryStore, q.metricStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
+					bq = newBuilderQuery(q.logger, q.telemetryStore, q.metricStmtBuilder, spec, timeRange, req.RequestType, tmplVars)
 				}
 
 				queries[spec.Name] = bq
@@ -509,7 +509,7 @@ func (q *querier) QueryRawStream(ctx context.Context, orgID valuer.UUID, req *qb
 		case <-tick:
 			// timestamp end is not specified here
 			timeRange := adjustTimeRangeForShift(spec, qbtypes.TimeRange{From: tsStart}, req.RequestType)
-			bq := newBuilderQuery(q.telemetryStore, q.logStmtBuilder, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
+			bq := newBuilderQuery(q.logger, q.telemetryStore, q.logStmtBuilder, spec, timeRange, req.RequestType, map[string]qbtypes.VariableItem{
 				"id": {
 					Value: updatedLogID,
 				},
@@ -801,22 +801,22 @@ func (q *querier) createRangedQuery(originalQuery qbtypes.Query, timeRange qbtyp
 		specCopy := qt.spec.Copy()
 		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
 		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
-		return newBuilderQuery(q.telemetryStore, q.traceStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
+		return newBuilderQuery(q.logger, q.telemetryStore, q.traceStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
 
 	case *builderQuery[qbtypes.LogAggregation]:
 		specCopy := qt.spec.Copy()
 		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
 		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
-		return newBuilderQuery(q.telemetryStore, q.logStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
+		return newBuilderQuery(q.logger, q.telemetryStore, q.logStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
 
 	case *builderQuery[qbtypes.MetricAggregation]:
 		specCopy := qt.spec.Copy()
 		specCopy.ShiftBy = extractShiftFromBuilderQuery(specCopy)
 		adjustedTimeRange := adjustTimeRangeForShift(specCopy, timeRange, qt.kind)
 		if qt.spec.Source == telemetrytypes.SourceMeter {
-			return newBuilderQuery(q.telemetryStore, q.meterStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
+			return newBuilderQuery(q.logger, q.telemetryStore, q.meterStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
 		}
-		return newBuilderQuery(q.telemetryStore, q.metricStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
+		return newBuilderQuery(q.logger, q.telemetryStore, q.metricStmtBuilder, specCopy, adjustedTimeRange, qt.kind, qt.variables)
 	case *traceOperatorQuery:
 		specCopy := qt.spec.Copy()
 		return &traceOperatorQuery{

pkg/query-service/app/logparsingpipeline/controller.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"slices"
 	"strings"
 
@@ -12,6 +13,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
 	"github.com/SigNoz/signoz/pkg/query-service/constants"
+	"github.com/SigNoz/signoz/pkg/query-service/interfaces"
 	"github.com/SigNoz/signoz/pkg/query-service/model"
 	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
 	"github.com/SigNoz/signoz/pkg/query-service/utils"
@@ -34,19 +36,101 @@ type LogParsingPipelineController struct {
 	Repo
 
 	GetIntegrationPipelines func(context.Context, string) ([]pipelinetypes.GettablePipeline, error)
+	// TODO(Piyush): remove with qbv5 migration
+	reader interfaces.Reader
 }
 
 func NewLogParsingPipelinesController(
 	sqlStore sqlstore.SQLStore,
 	getIntegrationPipelines func(context.Context, string) ([]pipelinetypes.GettablePipeline, error),
+	reader interfaces.Reader,
 ) (*LogParsingPipelineController, error) {
 	repo := NewRepo(sqlStore)
 	return &LogParsingPipelineController{
 		Repo:                    repo,
 		GetIntegrationPipelines: getIntegrationPipelines,
+		reader:                  reader,
 	}, nil
 }
 
+// enrichPipelinesFilters resolves the type (tag vs resource) for filter keys that are
+// missing type info, by looking them up in the store.
+//
+// TODO(Piyush): remove with qbv5 migration
+func (pc *LogParsingPipelineController) enrichPipelinesFilters(
+	ctx context.Context, pipelines []pipelinetypes.GettablePipeline,
+) ([]pipelinetypes.GettablePipeline, error) {
+	// Collect names of non-static keys that are missing type info.
+	// Static fields (body, trace_id, etc.) are intentionally Unspecified and map
+	// to top-level OTEL fields — they do not need enrichment.
+	unspecifiedNames := map[string]struct{}{}
+	for _, p := range pipelines {
+		if p.Filter != nil {
+			for _, item := range p.Filter.Items {
+				if item.Key.Type == v3.AttributeKeyTypeUnspecified {
+					// Skip static fields
+					if _, isStatic := constants.StaticFieldsLogsV3[item.Key.Key]; isStatic {
+						continue
+					}
+					// Skip enrich body.* fields
+					if strings.HasPrefix(item.Key.Key, "body.") {
+						continue
+					}
+					unspecifiedNames[item.Key.Key] = struct{}{}
+				}
+			}
+		}
+	}
+	if len(unspecifiedNames) == 0 {
+		return pipelines, nil
+	}
+
+	logFields, apiErr := pc.reader.GetLogFieldsFromNames(ctx, slices.Collect(maps.Keys(unspecifiedNames)))
+	if apiErr != nil {
+		slog.ErrorContext(ctx, "failed to fetch log fields for pipeline filter enrichment", "error", apiErr)
+		return pipelines, apiErr
+	}
+
+	// Build a simple name → AttributeKeyType map from the response.
+	fieldTypes := map[string]v3.AttributeKeyType{}
+	for _, f := range append(logFields.Selected, logFields.Interesting...) {
+		switch f.Type {
+		case constants.Resources:
+			fieldTypes[f.Name] = v3.AttributeKeyTypeResource
+		case constants.Attributes:
+			fieldTypes[f.Name] = v3.AttributeKeyTypeTag
+		}
+	}
+
+	// Set the resolved type on each untyped filter key in-place.
+	for i := range pipelines {
+		if pipelines[i].Filter != nil {
+			for j := range pipelines[i].Filter.Items {
+				key := &pipelines[i].Filter.Items[j].Key
+				if key.Type == v3.AttributeKeyTypeUnspecified {
+					// Skip static fields
+					if _, isStatic := constants.StaticFieldsLogsV3[key.Key]; isStatic {
+						continue
+					}
+					// Skip enrich body.* fields
+					if strings.HasPrefix(key.Key, "body.") {
+						continue
+					}
+
+					if t, ok := fieldTypes[key.Key]; ok {
+						key.Type = t
+					} else {
+						// default to attribute
+						key.Type = v3.AttributeKeyTypeTag
+					}
+				}
+			}
+		}
+	}
+
+	return pipelines, nil
+}
+
 // PipelinesResponse is used to prepare http response for pipelines config related requests
 type PipelinesResponse struct {
 	*opamptypes.AgentConfigVersion
@@ -256,7 +340,12 @@ func (ic *LogParsingPipelineController) PreviewLogsPipelines(
 	ctx context.Context,
 	request *PipelinesPreviewRequest,
 ) (*PipelinesPreviewResponse, error) {
-	result, collectorLogs, err := SimulatePipelinesProcessing(ctx, request.Pipelines, request.Logs)
+	pipelines, err := ic.enrichPipelinesFilters(ctx, request.Pipelines)
+	if err != nil {
+		return nil, err
+	}
+
+	result, collectorLogs, err := SimulatePipelinesProcessing(ctx, pipelines, request.Logs)
 	if err != nil {
 		return nil, err
 	}
@@ -293,10 +382,8 @@ func (pc *LogParsingPipelineController) RecommendAgentConfig(
 	if configVersion != nil {
 		pipelinesVersion = configVersion.Version
 	}
-
-	pipelinesResp, err := pc.GetPipelinesByVersion(
-		context.Background(), orgId, pipelinesVersion,
-	)
+	ctx := context.Background()
+	pipelinesResp, err := pc.GetPipelinesByVersion(ctx, orgId, pipelinesVersion)
 	if err != nil {
 		return nil, "", err
 	}
@@ -306,12 +393,17 @@ func (pc *LogParsingPipelineController) RecommendAgentConfig(
 		return nil, "", errors.WrapInternalf(err, CodeRawPipelinesMarshalFailed, "could not serialize pipelines to JSON")
 	}
 
-	if querybuilder.BodyJSONQueryEnabled {
-		// add default normalize pipeline at the beginning, only for sending to collector
-		pipelinesResp.Pipelines = append([]pipelinetypes.GettablePipeline{pc.getNormalizePipeline()}, pipelinesResp.Pipelines...)
+	enrichedPipelines, err := pc.enrichPipelinesFilters(ctx, pipelinesResp.Pipelines)
+	if err != nil {
+		return nil, "", err
 	}
 
-	updatedConf, err := GenerateCollectorConfigWithPipelines(currentConfYaml, pipelinesResp.Pipelines)
+	if querybuilder.BodyJSONQueryEnabled {
+		// add default normalize pipeline at the beginning, only for sending to collector
+		enrichedPipelines = append([]pipelinetypes.GettablePipeline{pc.getNormalizePipeline()}, enrichedPipelines...)
+	}
+
+	updatedConf, err := GenerateCollectorConfigWithPipelines(currentConfYaml, enrichedPipelines)
 	if err != nil {
 		return nil, "", err
 	}

pkg/query-service/app/querier/querier_test.go

@@ -1407,7 +1407,7 @@ func Test_querier_Traces_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1633,7 +1633,7 @@ func Test_querier_Traces_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1934,7 +1934,7 @@ func Test_querier_Logs_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -2162,7 +2162,7 @@ func Test_querier_Logs_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,

pkg/query-service/app/querier/v2/querier_test.go

@@ -1459,7 +1459,7 @@ func Test_querier_Traces_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1685,7 +1685,7 @@ func Test_querier_Traces_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -1985,7 +1985,7 @@ func Test_querier_Logs_runWindowBasedListQueryDesc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,
@@ -2213,7 +2213,7 @@ func Test_querier_Logs_runWindowBasedListQueryAsc(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Duration(time.Second),
 				nil,

pkg/query-service/app/server.go

@@ -121,6 +121,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
 	logParsingPipelineController, err := logparsingpipeline.NewLogParsingPipelinesController(
 		signoz.SQLStore,
 		integrationsController.GetPipelinesForInstalledIntegrations,
+		reader,
 	)
 	if err != nil {
 		return nil, err

pkg/query-service/app/traces/tracedetail/waterfall.go

@@ -67,7 +67,7 @@ func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string, un
 	spansFromRootToNode := []string{}
 
 	if node.SpanID == selectedSpanId {
-		if isSelectedSpanIDUnCollapsed {
+		if isSelectedSpanIDUnCollapsed && !slices.Contains(uncollapsedSpans, node.SpanID) {
 			spansFromRootToNode = append(spansFromRootToNode, node.SpanID)
 		}
 		return true, spansFromRootToNode
@@ -88,7 +88,15 @@ func getPathFromRootToSelectedSpanId(node *model.Span, selectedSpanId string, un
 	return isPresentInSubtreeForTheNode, spansFromRootToNode
 }
 
-func traverseTrace(span *model.Span, uncollapsedSpans []string, level uint64, isPartOfPreOrder bool, hasSibling bool, selectedSpanId string) []*model.Span {
+// traverseOpts holds the traversal configuration that remains constant
+// throughout the recursion. Per-call state (level, isPartOfPreOrder, etc.)
+// is passed as direct arguments.
+type traverseOpts struct {
+	uncollapsedSpans []string
+	selectedSpanID   string
+}
+
+func traverseTrace(span *model.Span, opts traverseOpts, level uint64, isPartOfPreOrder bool, hasSibling bool) []*model.Span {
 	preOrderTraversal := []*model.Span{}
 
 	// sort the children to maintain the order across requests
@@ -126,8 +134,9 @@ func traverseTrace(span *model.Span, uncollapsedSpans []string, level uint64, is
 		preOrderTraversal = append(preOrderTraversal, &nodeWithoutChildren)
 	}
 
+	isAlreadyUncollapsed := slices.Contains(opts.uncollapsedSpans, span.SpanID)
 	for index, child := range span.Children {
-		_childTraversal := traverseTrace(child, uncollapsedSpans, level+1, isPartOfPreOrder && slices.Contains(uncollapsedSpans, span.SpanID), index != (len(span.Children)-1), selectedSpanId)
+		_childTraversal := traverseTrace(child, opts, level+1, isPartOfPreOrder && isAlreadyUncollapsed, index != (len(span.Children)-1))
 		preOrderTraversal = append(preOrderTraversal, _childTraversal...)
 		nodeWithoutChildren.SubTreeNodeCount += child.SubTreeNodeCount + 1
 		span.SubTreeNodeCount += child.SubTreeNodeCount + 1
@@ -168,7 +177,11 @@ func GetSelectedSpans(uncollapsedSpans []string, selectedSpanID string, traceRoo
 			_, spansFromRootToNode := getPathFromRootToSelectedSpanId(rootNode, selectedSpanID, updatedUncollapsedSpans, isSelectedSpanIDUnCollapsed)
 			updatedUncollapsedSpans = append(updatedUncollapsedSpans, spansFromRootToNode...)
 
-			_preOrderTraversal := traverseTrace(rootNode, updatedUncollapsedSpans, 0, true, false, selectedSpanID)
+			opts := traverseOpts{
+				uncollapsedSpans: updatedUncollapsedSpans,
+				selectedSpanID:   selectedSpanID,
+			}
+			_preOrderTraversal := traverseTrace(rootNode, opts, 0, true, false)
 			_selectedSpanIndex := findIndexForSelectedSpanFromPreOrder(_preOrderTraversal, selectedSpanID)
 
 			if _selectedSpanIndex != -1 {

pkg/query-service/app/traces/tracedetail/waterfall_test.go

@@ -0,0 +1,355 @@
+// Package tracedetail tests — waterfall
+//
+// # Background
+//
+// The waterfall view renders a trace as a scrollable list of spans in
+// pre-order (parent before children, siblings left-to-right). Because a trace
+// can have thousands of spans, only a window of ~500 is returned per request.
+// The window is centred on the selected span.
+//
+// # Key concepts
+//
+// uncollapsedSpans
+//
+//	The set of span IDs the user has manually expanded in the UI.
+//	Only the direct children of an uncollapsed span are included in the
+//	output; grandchildren stay hidden until their parent is also uncollapsed.
+//	When multiple spans are uncollapsed their children are all visible at once.
+//
+// selectedSpanID
+//
+//	The span currently focused — set when the user clicks a span in the
+//	waterfall or selects one from the flamegraph.  The output window is always
+//	centred on this span.  The path from the trace root down to the selected
+//	span is automatically uncollapsed so ancestors are visible even if they are
+//	not in uncollapsedSpans.
+//
+// isSelectedSpanIDUnCollapsed
+//
+//	Controls whether the selected span's own children are shown:
+//	  true  — user expanded the span (click-to-open in waterfall or flamegraph);
+//	          direct children of the selected span are included.
+//	  false — user selected without expanding;
+//	          the span is visible but its children remain hidden.
+//
+// traceRoots
+//
+//	Root spans of the trace — spans with no parent in the current dataset.
+//	Normally one, but multiple roots are common when upstream services are
+//	not instrumented or their spans were not sampled/exported.
+
+package tracedetail
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/SigNoz/signoz/pkg/query-service/model"
+	"github.com/stretchr/testify/assert"
+)
+
+// Pre-order traversal is preserved: parent before children, siblings left-to-right.
+func TestGetSelectedSpans_PreOrderTraversal(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("child1", "svc", mkSpan("grandchild", "svc")),
+		mkSpan("child2", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{"root", "child1"}, "root", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "child1", "grandchild", "child2"}, spanIDs(spans))
+}
+
+// Multiple roots: both trees are flattened into a single pre-order list with
+// root1's subtree before root2's. Service/entry-point come from the first root.
+//
+//	root1 svc-a          ← selected
+//	  └─ child1
+//	root2 svc-b
+//	  └─ child2
+//
+// Expected output order: root1 → child1 → root2 → child2
+func TestGetSelectedSpans_MultipleRoots(t *testing.T) {
+	root1 := mkSpan("root1", "svc-a", mkSpan("child1", "svc-a"))
+	root2 := mkSpan("root2", "svc-b", mkSpan("child2", "svc-b"))
+	spanMap := buildSpanMap(root1, root2)
+
+	spans, _, svcName, entryPoint := GetSelectedSpans([]string{"root1", "root2"}, "root1", []*model.Span{root1, root2}, spanMap, false)
+
+	assert.Equal(t, []string{"root1", "child1", "root2", "child2"}, spanIDs(spans), "root1 subtree must precede root2 subtree")
+	assert.Equal(t, "svc-a", svcName, "metadata comes from first root")
+	assert.Equal(t, "root1-op", entryPoint, "metadata comes from first root")
+}
+
+// isSelectedSpanIDUnCollapsed=true opens only the selected span's direct children,
+// not deeper descendants.
+//
+//	root → selected (expanded)
+//	          ├─ child1 ✓
+//	          │    └─ grandchild ✗  (only one level opened)
+//	          └─ child2 ✓
+func TestGetSelectedSpans_ExpandedSelectedSpan(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("selected", "svc",
+			mkSpan("child1", "svc", mkSpan("grandchild", "svc")),
+			mkSpan("child2", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, true)
+
+	// root and selected are on the auto-uncollapsed path; child1/child2 are direct
+	// children of the expanded selected span; grandchild stays hidden.
+	assert.Equal(t, []string{"root", "selected", "child1", "child2"}, spanIDs(spans))
+}
+
+// Multiple spans uncollapsed simultaneously: children of all uncollapsed spans
+// are visible at once.
+//
+//	root
+//	  ├─ childA (uncollapsed) → grandchildA ✓
+//	  └─ childB (uncollapsed) → grandchildB ✓
+func TestGetSelectedSpans_MultipleUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("childA", "svc", mkSpan("grandchildA", "svc")),
+		mkSpan("childB", "svc", mkSpan("grandchildB", "svc")),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{"root", "childA", "childB"}, "root", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "childA", "grandchildA", "childB", "grandchildB"}, spanIDs(spans))
+}
+
+// Collapsing a span with other uncollapsed spans
+//
+// root
+// ├─ childA  (previously expanded — in uncollapsedSpans)
+// │    ├─ grandchild1 ✓
+// │    │    └─ greatGrandchild ✗  (grandchild1 not in uncollapsedSpans)
+// │    └─ grandchild2 ✓
+// └─ childB                        ← selected (not expanded)
+func TestGetSelectedSpans_ManualUncollapse(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("childA", "svc",
+			mkSpan("grandchild1", "svc", mkSpan("greatGrandchild", "svc")),
+			mkSpan("grandchild2", "svc"),
+		),
+		mkSpan("childB", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	// childA was expanded in a previous interaction; childB is now selected without expanding
+	spans, _, _, _ := GetSelectedSpans([]string{"childA"}, "childB", []*model.Span{root}, spanMap, false)
+
+	// path to childB auto-uncollpases root → childA and childB appear; childA is in
+	// uncollapsedSpans so its children appear; greatGrandchild stays hidden.
+	assert.Equal(t, []string{"root", "childA", "grandchild1", "grandchild2", "childB"}, spanIDs(spans))
+}
+
+// A collapsed span hides all children.
+func TestGetSelectedSpans_CollapsedSpan(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("child1", "svc"),
+		mkSpan("child2", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{}, "root", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root"}, spanIDs(spans))
+}
+
+// Selecting a span auto-uncollpases the path from root to that span so it is visible.
+//
+//	root → parent → selected
+func TestGetSelectedSpans_PathToSelectedIsUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("parent", "svc",
+			mkSpan("selected", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	// no manually uncollapsed spans — path should still be opened
+	spans, _, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "parent", "selected"}, spanIDs(spans))
+}
+
+// The path-to-selected spans are returned in updatedUncollapsedSpans.
+func TestGetSelectedSpans_PathReturnedInUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("parent", "svc",
+			mkSpan("selected", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	spans, uncollapsed, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, []string{"root", "parent"}, uncollapsed)
+	assert.Equal(t, []string{"root", "parent", "selected"}, spanIDs(spans))
+}
+
+// Siblings of ancestors are rendered as collapsed nodes but their subtrees
+// must NOT be expanded.
+//
+//	root
+//	  ├─ unrelated → unrelated-child (✗)
+//	  └─ parent → selected
+func TestGetSelectedSpans_SiblingsNotExpanded(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("unrelated", "svc", mkSpan("unrelated-child", "svc")),
+		mkSpan("parent", "svc",
+			mkSpan("selected", "svc"),
+		),
+	)
+	spanMap := buildSpanMap(root)
+	spans, uncollapsed, _, _ := GetSelectedSpans([]string{}, "selected", []*model.Span{root}, spanMap, false)
+
+	// children of root sort alphabetically: parent < unrelated; unrelated-child stays hidden
+	assert.Equal(t, []string{"root", "parent", "selected", "unrelated"}, spanIDs(spans))
+	// only the path nodes are tracked as uncollapsed — unrelated is not
+	assert.Equal(t, []string{"root", "parent"}, uncollapsed)
+}
+
+// An unknown selectedSpanID must not panic; returns a window from index 0.
+func TestGetSelectedSpans_UnknownSelectedSpan(t *testing.T) {
+	root := mkSpan("root", "svc", mkSpan("child", "svc"))
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{}, "nonexistent", []*model.Span{root}, spanMap, false)
+	assert.Equal(t, []string{"root"}, spanIDs(spans))
+}
+
+// Test to check if Level, HasChildren, HasSiblings, and SubTreeNodeCount are populated correctly.
+//
+//	root          level=0, hasChildren=true,  hasSiblings=false, subTree=4
+//	  child1      level=1, hasChildren=true,  hasSiblings=true,  subTree=2
+//	    grandchild level=2, hasChildren=false, hasSiblings=false, subTree=1
+//	  child2      level=1, hasChildren=false, hasSiblings=false, subTree=1
+func TestGetSelectedSpans_SpanMetadata(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("child1", "svc", mkSpan("grandchild", "svc")),
+		mkSpan("child2", "svc"),
+	)
+	spanMap := buildSpanMap(root)
+	spans, _, _, _ := GetSelectedSpans([]string{"root", "child1"}, "root", []*model.Span{root}, spanMap, false)
+
+	byID := map[string]*model.Span{}
+	for _, s := range spans {
+		byID[s.SpanID] = s
+	}
+
+	assert.Equal(t, uint64(0), byID["root"].Level)
+	assert.Equal(t, uint64(1), byID["child1"].Level)
+	assert.Equal(t, uint64(1), byID["child2"].Level)
+	assert.Equal(t, uint64(2), byID["grandchild"].Level)
+
+	assert.True(t, byID["root"].HasChildren)
+	assert.True(t, byID["child1"].HasChildren)
+	assert.False(t, byID["child2"].HasChildren)
+	assert.False(t, byID["grandchild"].HasChildren)
+
+	assert.False(t, byID["root"].HasSiblings, "root has no siblings")
+	assert.True(t, byID["child1"].HasSiblings, "child1 has sibling child2")
+	assert.False(t, byID["child2"].HasSiblings, "child2 is the last child")
+	assert.False(t, byID["grandchild"].HasSiblings, "grandchild has no siblings")
+
+	assert.Equal(t, uint64(4), byID["root"].SubTreeNodeCount)
+	assert.Equal(t, uint64(2), byID["child1"].SubTreeNodeCount)
+	assert.Equal(t, uint64(1), byID["grandchild"].SubTreeNodeCount)
+	assert.Equal(t, uint64(1), byID["child2"].SubTreeNodeCount)
+}
+
+// If the selected span is already in uncollapsedSpans AND isSelectedSpanIDUnCollapsed=true,
+func TestGetSelectedSpans_DuplicateInUncollapsed(t *testing.T) {
+	root := mkSpan("root", "svc",
+		mkSpan("selected", "svc", mkSpan("child", "svc")),
+	)
+	spanMap := buildSpanMap(root)
+	_, uncollapsed, _, _ := GetSelectedSpans(
+		[]string{"selected"}, // already present
+		"selected",
+		[]*model.Span{root}, spanMap,
+		true,
+	)
+
+	count := 0
+	for _, id := range uncollapsed {
+		if id == "selected" {
+			count++
+		}
+	}
+	assert.Equal(t, 1, count, "should appear once")
+}
+
+// makeChain builds a linear trace: span0 → span1 → … → span(n-1).
+// All span IDs are "span0", "span1", … so the caller can reference them by index.
+func makeChain(n int) (*model.Span, map[string]*model.Span, []string) {
+	spans := make([]*model.Span, n)
+	for i := n - 1; i >= 0; i-- {
+		if i == n-1 {
+			spans[i] = mkSpan(fmt.Sprintf("span%d", i), "svc")
+		} else {
+			spans[i] = mkSpan(fmt.Sprintf("span%d", i), "svc", spans[i+1])
+		}
+	}
+	uncollapsed := make([]string, n)
+	for i := range spans {
+		uncollapsed[i] = fmt.Sprintf("span%d", i)
+	}
+	return spans[0], buildSpanMap(spans[0]), uncollapsed
+}
+
+// The selected span is centred: 200 spans before it, 300 after (0.4 / 0.6 split).
+func TestGetSelectedSpans_WindowCentredOnSelected(t *testing.T) {
+	root, spanMap, uncollapsed := makeChain(600)
+	spans, _, _, _ := GetSelectedSpans(uncollapsed, "span300", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, 500, len(spans), "window should be 500 spans")
+	// window is [100, 600): span300 lands at position 200 (300 - 100)
+	assert.Equal(t, "span100", spans[0].SpanID, "window starts 200 before selected")
+	assert.Equal(t, "span300", spans[200].SpanID, "selected span at position 200 in window")
+	assert.Equal(t, "span599", spans[499].SpanID, "window ends 300 after selected")
+}
+
+// When the selected span is near the start, the window shifts right so no
+// negative index is used — the result is still 500 spans.
+func TestGetSelectedSpans_WindowShiftsAtStart(t *testing.T) {
+	root, spanMap, uncollapsed := makeChain(600)
+	spans, _, _, _ := GetSelectedSpans(uncollapsed, "span10", []*model.Span{root}, spanMap, false)
+
+	assert.Equal(t, 500, len(spans))
+	assert.Equal(t, "span0", spans[0].SpanID, "window clamped to start of trace")
+	assert.Equal(t, "span10", spans[10].SpanID, "selected span still in window")
+}
+
+func mkSpan(id, service string, children ...*model.Span) *model.Span {
+	return &model.Span{
+		SpanID:      id,
+		ServiceName: service,
+		Name:        id + "-op",
+		Children:    children,
+	}
+}
+
+// spanIDs returns SpanIDs in order.
+func spanIDs(spans []*model.Span) []string {
+	ids := make([]string, len(spans))
+	for i, s := range spans {
+		ids[i] = s.SpanID
+	}
+	return ids
+}
+
+// buildSpanMap indexes every span in a set of trees by SpanID.
+func buildSpanMap(roots ...*model.Span) map[string]*model.Span {
+	m := map[string]*model.Span{}
+	var walk func(*model.Span)
+	walk = func(s *model.Span) {
+		m[s.SpanID] = s
+		for _, c := range s.Children {
+			walk(c)
+		}
+	}
+	for _, r := range roots {
+		walk(r)
+	}
+	return m
+}

pkg/query-service/rules/base_rule_test.go

@@ -698,7 +698,7 @@ func TestBaseRule_FilterNewSeries(t *testing.T) {
 				slog.Default(),
 				nil,
 				telemetryStore,
-				prometheustest.New(context.Background(), settings, prometheus.Config{}, telemetryStore),
+				prometheustest.New(context.Background(), settings, prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore),
 				"",
 				time.Second,
 				nil,

pkg/query-service/rules/manager_test.go

@@ -253,7 +253,7 @@ func TestManager_TestNotification_SendUnmatched_PromRule(t *testing.T) {
 						WillReturnRows(samplesRows)
 
 					// Create Prometheus provider for this test
-					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, store)
+					promProvider = prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, store)
 				},
 				ManagerOptionsHook: func(opts *ManagerOptions) {
 					// Set Prometheus provider for PromQL queries

pkg/query-service/rules/managertestfactory.go

@@ -99,7 +99,7 @@ func NewTestManager(t *testing.T, testOpts *TestManagerOptions) *Manager {
 
 	options := clickhouseReader.NewOptions("", "", "archiveNamespace")
 	providerSettings := instrumentationtest.New().ToProviderSettings()
-	prometheus := prometheustest.New(context.Background(), providerSettings, prometheus.Config{}, telemetryStore)
+	prometheus := prometheustest.New(context.Background(), providerSettings, prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 	reader := clickhouseReader.NewReader(
 		instrumentationtest.New().Logger(),
 		nil,

pkg/query-service/rules/prom_rule_test.go

@@ -940,7 +940,7 @@ func TestPromRuleUnitCombinations(t *testing.T) {
 			).
 			WillReturnRows(samplesRows)
 
-		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
 		postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
 		postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
@@ -1061,7 +1061,7 @@ func _Enable_this_after_9146_issue_fix_is_merged_TestPromRuleNoData(t *testing.T
 			WithArgs("test_metric", "__name__", "test_metric").
 			WillReturnRows(fingerprintRows)
 
-		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
 		var target float64 = 0
 		postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{
@@ -1281,7 +1281,7 @@ func TestMultipleThresholdPromRule(t *testing.T) {
 			).
 			WillReturnRows(samplesRows)
 
-		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+		promProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 
 		postableRule.RuleCondition.CompareOp = ruletypes.CompareOp(c.compareOp)
 		postableRule.RuleCondition.MatchType = ruletypes.MatchType(c.matchType)
@@ -1441,7 +1441,7 @@ func TestPromRule_NoData(t *testing.T) {
 			promProvider := prometheustest.New(
 				context.Background(),
 				instrumentationtest.New().ToProviderSettings(),
-				prometheus.Config{},
+				prometheus.Config{Timeout: 2 * time.Minute},
 				telemetryStore,
 			)
 			defer func() {
@@ -1590,7 +1590,7 @@ func TestPromRule_NoData_AbsentFor(t *testing.T) {
 			promProvider := prometheustest.New(
 				context.Background(),
 				instrumentationtest.New().ToProviderSettings(),
-				prometheus.Config{},
+				prometheus.Config{Timeout: 2 * time.Minute},
 				telemetryStore,
 			)
 			defer func() {
@@ -1748,7 +1748,7 @@ func TestPromRuleEval_RequireMinPoints(t *testing.T) {
 			promProvider := prometheustest.New(
 				context.Background(),
 				instrumentationtest.New().ToProviderSettings(),
-				prometheus.Config{LookbackDelta: lookBackDelta},
+				prometheus.Config{Timeout: 2 * time.Minute, LookbackDelta: lookBackDelta},
 				telemetryStore,
 			)
 			defer func() {

pkg/query-service/rules/threshold_rule_test.go

@@ -779,7 +779,7 @@ func TestThresholdRuleUnitCombinations(t *testing.T) {
 			},
 		)
 		require.NoError(t, err)
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
 			"signoz_calls_total": {
@@ -894,7 +894,7 @@ func TestThresholdRuleNoData(t *testing.T) {
 		)
 		assert.NoError(t, err)
 		options := clickhouseReader.NewOptions("", "", "archiveNamespace")
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, readerCache, options)
 
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
@@ -1014,7 +1014,7 @@ func TestThresholdRuleTracesLink(t *testing.T) {
 		}
 
 		options := clickhouseReader.NewOptions("", "", "archiveNamespace")
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
 
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
@@ -1151,7 +1151,7 @@ func TestThresholdRuleLogsLink(t *testing.T) {
 		}
 
 		options := clickhouseReader.NewOptions("", "", "archiveNamespace")
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Duration(time.Second), nil, nil, options)
 
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
@@ -1418,7 +1418,7 @@ func TestMultipleThresholdRule(t *testing.T) {
 			},
 		)
 		require.NoError(t, err)
-		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore), "", time.Second, nil, readerCache, options)
+		reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore), "", time.Second, nil, readerCache, options)
 		rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, reader, nil, logger)
 		rule.TemporalityMap = map[string]map[v3.Temporality]bool{
 			"signoz_calls_total": {
@@ -2220,7 +2220,7 @@ func TestThresholdEval_RequireMinPoints(t *testing.T) {
 			)
 			require.NoError(t, err)
 
-			prometheusProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{}, telemetryStore)
+			prometheusProvider := prometheustest.New(context.Background(), instrumentationtest.New().ToProviderSettings(), prometheus.Config{Timeout: 2 * time.Minute}, telemetryStore)
 			reader := clickhouseReader.NewReader(slog.Default(), nil, telemetryStore, prometheusProvider, "", time.Second, nil, readerCache, options)
 
 			rule, err := NewThresholdRule("some-id", valuer.GenerateUUID(), &postableRule, reader, nil, logger)

pkg/signoz/handler.go

@@ -12,6 +12,8 @@ import (
 	"github.com/SigNoz/signoz/pkg/licensing"
 	"github.com/SigNoz/signoz/pkg/modules/apdex"
 	"github.com/SigNoz/signoz/pkg/modules/apdex/implapdex"
+	"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
+	"github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
 	"github.com/SigNoz/signoz/pkg/modules/fields"
@@ -38,24 +40,25 @@ import (
 )
 
 type Handlers struct {
-	SavedView             savedview.Handler
-	Apdex                 apdex.Handler
-	Dashboard             dashboard.Handler
-	QuickFilter           quickfilter.Handler
-	TraceFunnel           tracefunnel.Handler
-	RawDataExport         rawdataexport.Handler
-	SpanPercentile        spanpercentile.Handler
-	Services              services.Handler
-	MetricsExplorer       metricsexplorer.Handler
-	Global                global.Handler
-	FlaggerHandler        flagger.Handler
-	GatewayHandler        gateway.Handler
-	Fields                fields.Handler
-	AuthzHandler          authz.Handler
-	ZeusHandler           zeus.Handler
-	QuerierHandler        querier.Handler
-	ServiceAccountHandler serviceaccount.Handler
-	RegistryHandler       factory.Handler
+	SavedView               savedview.Handler
+	Apdex                   apdex.Handler
+	Dashboard               dashboard.Handler
+	QuickFilter             quickfilter.Handler
+	TraceFunnel             tracefunnel.Handler
+	RawDataExport           rawdataexport.Handler
+	SpanPercentile          spanpercentile.Handler
+	Services                services.Handler
+	MetricsExplorer         metricsexplorer.Handler
+	Global                  global.Handler
+	FlaggerHandler          flagger.Handler
+	GatewayHandler          gateway.Handler
+	Fields                  fields.Handler
+	AuthzHandler            authz.Handler
+	ZeusHandler             zeus.Handler
+	QuerierHandler          querier.Handler
+	ServiceAccountHandler   serviceaccount.Handler
+	RegistryHandler         factory.Handler
+	CloudIntegrationHandler cloudintegration.Handler
 }
 
 func NewHandlers(
@@ -73,23 +76,24 @@ func NewHandlers(
 	registryHandler factory.Handler,
 ) Handlers {
 	return Handlers{
-		SavedView:             implsavedview.NewHandler(modules.SavedView),
-		Apdex:                 implapdex.NewHandler(modules.Apdex),
-		Dashboard:             impldashboard.NewHandler(modules.Dashboard, providerSettings),
-		QuickFilter:           implquickfilter.NewHandler(modules.QuickFilter),
-		TraceFunnel:           impltracefunnel.NewHandler(modules.TraceFunnel),
-		RawDataExport:         implrawdataexport.NewHandler(modules.RawDataExport),
-		Services:              implservices.NewHandler(modules.Services),
-		MetricsExplorer:       implmetricsexplorer.NewHandler(modules.MetricsExplorer),
-		SpanPercentile:        implspanpercentile.NewHandler(modules.SpanPercentile),
-		Global:                signozglobal.NewHandler(global),
-		FlaggerHandler:        flagger.NewHandler(flaggerService),
-		GatewayHandler:        gateway.NewHandler(gatewayService),
-		Fields:                implfields.NewHandler(providerSettings, telemetryMetadataStore),
-		AuthzHandler:          signozauthzapi.NewHandler(authz),
-		ZeusHandler:           zeus.NewHandler(zeusService, licensing),
-		QuerierHandler:        querierHandler,
-		ServiceAccountHandler: implserviceaccount.NewHandler(modules.ServiceAccount),
-		RegistryHandler:       registryHandler,
+		SavedView:               implsavedview.NewHandler(modules.SavedView),
+		Apdex:                   implapdex.NewHandler(modules.Apdex),
+		Dashboard:               impldashboard.NewHandler(modules.Dashboard, providerSettings),
+		QuickFilter:             implquickfilter.NewHandler(modules.QuickFilter),
+		TraceFunnel:             impltracefunnel.NewHandler(modules.TraceFunnel),
+		RawDataExport:           implrawdataexport.NewHandler(modules.RawDataExport),
+		Services:                implservices.NewHandler(modules.Services),
+		MetricsExplorer:         implmetricsexplorer.NewHandler(modules.MetricsExplorer),
+		SpanPercentile:          implspanpercentile.NewHandler(modules.SpanPercentile),
+		Global:                  signozglobal.NewHandler(global),
+		FlaggerHandler:          flagger.NewHandler(flaggerService),
+		GatewayHandler:          gateway.NewHandler(gatewayService),
+		Fields:                  implfields.NewHandler(providerSettings, telemetryMetadataStore),
+		AuthzHandler:            signozauthzapi.NewHandler(authz),
+		ZeusHandler:             zeus.NewHandler(zeusService, licensing),
+		QuerierHandler:          querierHandler,
+		ServiceAccountHandler:   implserviceaccount.NewHandler(modules.ServiceAccount),
+		RegistryHandler:         registryHandler,
+		CloudIntegrationHandler: implcloudintegration.NewHandler(),
 	}
 }

pkg/signoz/openapi.go

@@ -17,6 +17,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/http/handler"
 	"github.com/SigNoz/signoz/pkg/instrumentation"
 	"github.com/SigNoz/signoz/pkg/modules/authdomain"
+	"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard"
 	"github.com/SigNoz/signoz/pkg/modules/fields"
 	"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
@@ -65,6 +66,7 @@ func NewOpenAPI(ctx context.Context, instrumentation instrumentation.Instrumenta
 		struct{ querier.Handler }{},
 		struct{ serviceaccount.Handler }{},
 		struct{ factory.Handler }{},
+		struct{ cloudintegration.Handler }{},
 	).New(ctx, instrumentation.ToProviderSettings(), apiserver.Config{})
 	if err != nil {
 		return nil, err

pkg/signoz/provider.go

@@ -279,6 +279,7 @@ func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.Au
 			handlers.QuerierHandler,
 			handlers.ServiceAccountHandler,
 			handlers.RegistryHandler,
+			handlers.CloudIntegrationHandler,
 		),
 	)
 }

pkg/telemetrytraces/statement_builder.go

@@ -111,23 +111,6 @@ func (b *traceQueryStatementBuilder) Build(
 
 	query = b.adjustKeys(ctx, keys, query, requestType)
 
-	// Check if filter contains trace_id(s) and optimize time range if needed
-	if query.Filter != nil && query.Filter.Expression != "" && b.telemetryStore != nil {
-		traceIDs, found := ExtractTraceIDsFromFilter(query.Filter.Expression)
-		if found && len(traceIDs) > 0 {
-			finder := NewTraceTimeRangeFinder(b.telemetryStore)
-
-			traceStart, traceEnd, ok := finder.GetTraceTimeRangeMulti(ctx, traceIDs)
-			if !ok {
-				b.logger.DebugContext(ctx, "failed to get trace time range", slog.Any("trace_ids", traceIDs))
-			} else if traceStart > 0 && traceEnd > 0 {
-				start = uint64(traceStart)
-				end = uint64(traceEnd)
-				b.logger.DebugContext(ctx, "optimized time range for traces", slog.Any("trace_ids", traceIDs), slog.Uint64("start", start), slog.Uint64("end", end))
-			}
-		}
-	}
-
 	// Create SQL builder
 	q := sqlbuilder.NewSelectBuilder()
 

pkg/types/cloudintegrationtypes/account.go

@@ -10,34 +10,35 @@ import (
 type Account struct {
 	types.Identifiable
 	types.TimeAuditable
-	ProviderAccountId *string           `json:"providerAccountID,omitempty"`
-	Provider          CloudProviderType `json:"provider"`
-	RemovedAt         *time.Time        `json:"removedAt,omitempty"`
-	AgentReport       *AgentReport      `json:"agentReport,omitempty"`
-	OrgID             valuer.UUID       `json:"orgID"`
-	Config            *AccountConfig    `json:"config,omitempty"`
+	ProviderAccountID *string           `json:"providerAccountId" required:"true" nullable:"true"`
+	Provider          CloudProviderType `json:"provider" required:"true"`
+	RemovedAt         *time.Time        `json:"removedAt" required:"true" nullable:"true"`
+	AgentReport       *AgentReport      `json:"agentReport" required:"true" nullable:"true"`
+	OrgID             valuer.UUID       `json:"orgId" required:"true"`
+	Config            *AccountConfig    `json:"config" required:"true" nullable:"false"`
 }
 
 // AgentReport represents heartbeats sent by the agent.
 type AgentReport struct {
-	TimestampMillis int64          `json:"timestampMillis"`
-	Data            map[string]any `json:"data"`
+	TimestampMillis int64          `json:"timestampMillis" required:"true"`
+	Data            map[string]any `json:"data" required:"true" nullable:"true"`
+}
+
+type AccountConfig struct {
+	// required till new providers are added
+	AWS *AWSAccountConfig `json:"aws" required:"true" nullable:"false"`
 }
 
 type GettableAccounts struct {
-	Accounts []*Account `json:"accounts"`
+	Accounts []*Account `json:"accounts" required:"true" nullable:"false"`
 }
 
 type GettableAccount = Account
 
 type UpdatableAccount struct {
-	Config *AccountConfig `json:"config"`
-}
-
-type AccountConfig struct {
-	AWS *AWSAccountConfig `json:"aws,omitempty"`
+	Config *AccountConfig `json:"config" required:"true" nullable:"false"`
 }
 
 type AWSAccountConfig struct {
-	Regions []string `json:"regions"`
+	Regions []string `json:"regions" required:"true" nullable:"false"`
 }

pkg/types/cloudintegrationtypes/connection.go

@@ -1,88 +1,81 @@
 package cloudintegrationtypes
 
-import "github.com/SigNoz/signoz/pkg/types/integrationtypes"
+import (
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
 
 type ConnectionArtifactRequest struct {
-	Aws *AWSConnectionArtifactRequest `json:"aws"`
+	// required till new providers are added
+	Aws *AWSConnectionArtifactRequest `json:"aws" required:"true" nullable:"false"`
 }
 
 type AWSConnectionArtifactRequest struct {
-	DeploymentRegion string   `json:"deploymentRegion"`
-	Regions          []string `json:"regions"`
+	DeploymentRegion string   `json:"deploymentRegion" required:"true"`
+	Regions          []string `json:"regions" required:"true" nullable:"false"`
 }
 
 type PostableConnectionArtifact = ConnectionArtifactRequest
 
 type ConnectionArtifact struct {
-	Aws *AWSConnectionArtifact `json:"aws"`
+	// required till new providers are added
+	Aws *AWSConnectionArtifact `json:"aws" required:"true" nullable:"false"`
 }
 
 type AWSConnectionArtifact struct {
-	ConnectionUrl string `json:"connectionURL"`
+	ConnectionURL string `json:"connectionURL" required:"true"`
 }
 
-type GettableConnectionArtifact = ConnectionArtifact
-
-type AccountStatus struct {
-	Id                string                         `json:"id"`
-	ProviderAccountId *string                        `json:"providerAccountID,omitempty"`
-	Status            integrationtypes.AccountStatus `json:"status"`
+type GettableAccountWithArtifact struct {
+	ID       valuer.UUID         `json:"id" required:"true"`
+	Artifact *ConnectionArtifact `json:"connectionArtifact" required:"true"`
 }
 
-type GettableAccountStatus = AccountStatus
-
 type AgentCheckInRequest struct {
-	// older backward compatible fields are mapped to new fields
-	// CloudIntegrationId string `json:"cloudIntegrationId"`
-	// AccountId          string `json:"accountId"`
+	ProviderAccountID  string `json:"providerAccountId" required:"false"`
+	CloudIntegrationID string `json:"cloudIntegrationId" required:"false"`
 
-	// New fields
-	ProviderAccountId string `json:"providerAccountId"`
-	CloudAccountId    string `json:"cloudAccountId"`
-
-	Data map[string]any `json:"data,omitempty"`
+	Data map[string]any `json:"data" required:"true" nullable:"true"`
 }
 
 type PostableAgentCheckInRequest struct {
 	AgentCheckInRequest
 	// following are backward compatible fields for older running agents
 	// which gets mapped to new fields in AgentCheckInRequest
-	CloudIntegrationId string `json:"cloud_integration_id"`
-	CloudAccountId     string `json:"cloud_account_id"`
-}
-
-type GettableAgentCheckInResponse struct {
-	AgentCheckInResponse
-
-	// For backward compatibility
-	CloudIntegrationId string `json:"cloud_integration_id"`
-	AccountId          string `json:"account_id"`
+	ID        string `json:"account_id" required:"false"`       // => CloudIntegrationID
+	AccountID string `json:"cloud_account_id" required:"false"` // => ProviderAccountID
 }
 
 type AgentCheckInResponse struct {
-	// Older fields for backward compatibility are mapped to new fields below
-	// CloudIntegrationId string `json:"cloud_integration_id"`
-	// AccountId          string `json:"account_id"`
-
-	// New fields
-	ProviderAccountId string `json:"providerAccountId"`
-	CloudAccountId    string `json:"cloudAccountId"`
-
-	// IntegrationConfig populates data related to integration that is required for an agent
-	// to start collecting telemetry data
-	// keeping JSON key snake_case for backward compatibility
-	IntegrationConfig *IntegrationConfig `json:"integration_config,omitempty"`
+	CloudIntegrationID string                     `json:"cloudIntegrationId" required:"true"`
+	ProviderAccountID  string                     `json:"providerAccountId" required:"true"`
+	IntegrationConfig  *ProviderIntegrationConfig `json:"integrationConfig" required:"true"`
+	RemovedAt          *time.Time                 `json:"removedAt" required:"true" nullable:"true"`
 }
 
-type IntegrationConfig struct {
-	EnabledRegions []string               `json:"enabledRegions"`      // backward compatible
-	Telemetry      *AWSCollectionStrategy `json:"telemetry,omitempty"` // backward compatible
+type GettableAgentCheckInResponse struct {
+	// Older fields for backward compatibility with existing AWS agents
+	AccountID              string             `json:"account_id" required:"true"`
+	CloudAccountID         string             `json:"cloud_account_id" required:"true"`
+	OlderIntegrationConfig *IntegrationConfig `json:"integration_config" required:"true" nullable:"true"`
+	OlderRemovedAt         *time.Time         `json:"removed_at" required:"true" nullable:"true"`
 
-	// new fields
-	AWS *AWSIntegrationConfig `json:"aws,omitempty"`
+	AgentCheckInResponse
+}
+
+// IntegrationConfig older integration config struct for backward compatibility,
+// this will be eventually removed once agents are updated to use new struct.
+type IntegrationConfig struct {
+	EnabledRegions []string               `json:"enabled_regions" required:"true" nullable:"false"` // backward compatible
+	Telemetry      *AWSCollectionStrategy `json:"telemetry" required:"true" nullable:"false"`       // backward compatible
+}
+
+type ProviderIntegrationConfig struct {
+	AWS *AWSIntegrationConfig `json:"aws" required:"true" nullable:"false"`
 }
 
 type AWSIntegrationConfig struct {
-	EnabledRegions []string               `json:"enabledRegions"`
-	Telemetry      *AWSCollectionStrategy `json:"telemetry,omitempty"`
+	EnabledRegions []string               `json:"enabledRegions" required:"true" nullable:"false"`
+	Telemetry      *AWSCollectionStrategy `json:"telemetry" required:"true" nullable:"false"`
 }

pkg/types/cloudintegrationtypes/service.go

@@ -10,20 +10,19 @@ import (
 	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
-var (
-	S3Sync = valuer.NewString("s3sync")
-	// ErrCodeInvalidServiceID is the error code for invalid service id.
-	ErrCodeInvalidServiceID = errors.MustNewCode("invalid_service_id")
-)
-
-type ServiceID struct{ valuer.String }
+var ErrCodeInvalidServiceID = errors.MustNewCode("invalid_service_id")
 
 type CloudIntegrationService struct {
 	types.Identifiable
 	types.TimeAuditable
 	Type               ServiceID      `json:"type"`
 	Config             *ServiceConfig `json:"config"`
-	CloudIntegrationID valuer.UUID    `json:"cloudIntegrationID"`
+	CloudIntegrationID valuer.UUID    `json:"cloudIntegrationId"`
+}
+
+type ServiceConfig struct {
+	// required till new providers are added
+	AWS *AWSServiceConfig `json:"aws" required:"true" nullable:"false"`
 }
 
 // ServiceMetadata helps to quickly list available services and whether it is enabled or not.
@@ -32,26 +31,56 @@ type CloudIntegrationService struct {
 type ServiceMetadata struct {
 	ServiceDefinitionMetadata
 	// if the service is enabled for the account
-	Enabled bool `json:"enabled"`
+	Enabled bool `json:"enabled" required:"true"`
+}
+
+// ServiceDefinitionMetadata represents service definition metadata. This is useful for showing service tab in frontend.
+type ServiceDefinitionMetadata struct {
+	ID    string `json:"id" required:"true"`
+	Title string `json:"title" required:"true"`
+	Icon  string `json:"icon" required:"true"`
 }
 
 type GettableServicesMetadata struct {
-	Services []*ServiceMetadata `json:"services"`
+	Services []*ServiceMetadata `json:"services" required:"true" nullable:"false"`
 }
 
 type Service struct {
 	ServiceDefinition
-	ServiceConfig *ServiceConfig `json:"serviceConfig"`
+	ServiceConfig *ServiceConfig `json:"serviceConfig" required:"false" nullable:"false"`
 }
 
 type GettableService = Service
 
 type UpdatableService struct {
-	Config *ServiceConfig `json:"config"`
+	Config *ServiceConfig `json:"config" required:"true" nullable:"false"`
 }
 
-type ServiceConfig struct {
-	AWS *AWSServiceConfig `json:"aws,omitempty"`
+type ServiceDefinition struct {
+	ServiceDefinitionMetadata
+	Overview         string              `json:"overview" required:"true"` // markdown
+	Assets           Assets              `json:"assets" required:"true"`
+	SupportedSignals SupportedSignals    `json:"supported_signals" required:"true"`
+	DataCollected    DataCollected       `json:"dataCollected" required:"true"`
+	Strategy         *CollectionStrategy `json:"telemetryCollectionStrategy" required:"true" nullable:"false"`
+}
+
+// SupportedSignals for cloud provider's service.
+type SupportedSignals struct {
+	Logs    bool `json:"logs"`
+	Metrics bool `json:"metrics"`
+}
+
+// DataCollected is curated static list of metrics and logs, this is shown as part of service overview.
+type DataCollected struct {
+	Logs    []CollectedLogAttribute `json:"logs"`
+	Metrics []CollectedMetric       `json:"metrics"`
+}
+
+// CollectionStrategy is cloud provider specific configuration for signal collection,
+// this is used by agent to understand the nitty-gritty for collecting telemetry for the cloud provider.
+type CollectionStrategy struct {
+	AWS *AWSCollectionStrategy `json:"aws" required:"true" nullable:"false"`
 }
 
 type AWSServiceConfig struct {
@@ -70,45 +99,11 @@ type AWSServiceMetricsConfig struct {
 	Enabled bool `json:"enabled"`
 }
 
-// ServiceDefinitionMetadata represents service definition metadata. This is useful for showing service tab in frontend.
-type ServiceDefinitionMetadata struct {
-	Id    string `json:"id"`
-	Title string `json:"title"`
-	Icon  string `json:"icon"`
-}
-
-type ServiceDefinition struct {
-	ServiceDefinitionMetadata
-	Overview         string              `json:"overview"` // markdown
-	Assets           Assets              `json:"assets"`
-	SupportedSignals SupportedSignals    `json:"supported_signals"`
-	DataCollected    DataCollected       `json:"dataCollected"`
-	Strategy         *CollectionStrategy `json:"telemetryCollectionStrategy"`
-}
-
-// CollectionStrategy is cloud provider specific configuration for signal collection,
-// this is used by agent to understand the nitty-gritty for collecting telemetry for the cloud provider.
-type CollectionStrategy struct {
-	AWS *AWSCollectionStrategy `json:"aws,omitempty"`
-}
-
 // Assets represents the collection of dashboards.
 type Assets struct {
 	Dashboards []Dashboard `json:"dashboards"`
 }
 
-// SupportedSignals for cloud provider's service.
-type SupportedSignals struct {
-	Logs    bool `json:"logs"`
-	Metrics bool `json:"metrics"`
-}
-
-// DataCollected is curated static list of metrics and logs, this is shown as part of service overview.
-type DataCollected struct {
-	Logs    []CollectedLogAttribute `json:"logs"`
-	Metrics []CollectedMetric       `json:"metrics"`
-}
-
 // CollectedLogAttribute represents a log attribute that is present in all log entries for a service,
 // this is shown as part of service overview.
 type CollectedLogAttribute struct {
@@ -169,56 +164,23 @@ type AWSLogsStrategy struct {
 // This is used to show available pre-made dashboards for a service,
 // hence has additional fields like id, title and description
 type Dashboard struct {
-	Id          string                               `json:"id"`
+	ID          string                               `json:"id"`
 	Title       string                               `json:"title"`
 	Description string                               `json:"description"`
 	Definition  dashboardtypes.StorableDashboardData `json:"definition,omitempty"`
 }
 
-// SupportedServices is the map of supported services for each cloud provider.
-var SupportedServices = map[CloudProviderType][]ServiceID{
-	CloudProviderTypeAWS: {
-		{valuer.NewString("alb")},
-		{valuer.NewString("api-gateway")},
-		{valuer.NewString("dynamodb")},
-		{valuer.NewString("ec2")},
-		{valuer.NewString("ecs")},
-		{valuer.NewString("eks")},
-		{valuer.NewString("elasticache")},
-		{valuer.NewString("lambda")},
-		{valuer.NewString("msk")},
-		{valuer.NewString("rds")},
-		{valuer.NewString("s3sync")},
-		{valuer.NewString("sns")},
-		{valuer.NewString("sqs")},
-	},
-}
-
-// NewServiceID returns a new ServiceID from a string, validated against the supported services for the given cloud provider.
-func NewServiceID(provider CloudProviderType, service string) (ServiceID, error) {
-	services, ok := SupportedServices[provider]
-	if !ok {
-		return ServiceID{}, errors.NewInvalidInputf(ErrCodeInvalidServiceID, "no services defined for cloud provider: %s", provider)
-	}
-	for _, s := range services {
-		if s.StringValue() == service {
-			return s, nil
-		}
-	}
-	return ServiceID{}, errors.NewInvalidInputf(ErrCodeInvalidServiceID, "invalid service id %q for cloud provider %s", service, provider)
-}
-
 // UTILS
 
 // GetCloudIntegrationDashboardID returns the dashboard id for a cloud integration, given the cloud provider, service id, and dashboard id.
 // This is used to generate unique dashboard ids for cloud integration, and also to parse the dashboard id to get the cloud provider and service id when needed.
-func GetCloudIntegrationDashboardID(cloudProvider CloudProviderType, svcId, dashboardId string) string {
-	return fmt.Sprintf("cloud-integration--%s--%s--%s", cloudProvider, svcId, dashboardId)
+func GetCloudIntegrationDashboardID(cloudProvider CloudProviderType, svcID, dashboardID string) string {
+	return fmt.Sprintf("cloud-integration--%s--%s--%s", cloudProvider, svcID, dashboardID)
 }
 
 // GetDashboardsFromAssets returns the list of dashboards for the cloud provider service from definition.
 func GetDashboardsFromAssets(
-	svcId string,
+	svcID string,
 	orgID valuer.UUID,
 	cloudProvider CloudProviderType,
 	createdAt time.Time,
@@ -229,7 +191,7 @@ func GetDashboardsFromAssets(
 	for _, d := range assets.Dashboards {
 		author := fmt.Sprintf("%s-integration", cloudProvider)
 		dashboards = append(dashboards, &dashboardtypes.Dashboard{
-			ID:     GetCloudIntegrationDashboardID(cloudProvider, svcId, d.Id),
+			ID:     GetCloudIntegrationDashboardID(cloudProvider, svcID, d.ID),
 			Locked: true,
 			OrgID:  orgID,
 			Data:   d.Definition,

pkg/types/cloudintegrationtypes/serviceid.go

@@ -0,0 +1,75 @@
+package cloudintegrationtypes
+
+import (
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+type ServiceID struct{ valuer.String }
+
+var (
+	AWSServiceALB         = ServiceID{valuer.NewString("alb")}
+	AWSServiceAPIGateway  = ServiceID{valuer.NewString("api-gateway")}
+	AWSServiceDynamoDB    = ServiceID{valuer.NewString("dynamodb")}
+	AWSServiceEC2         = ServiceID{valuer.NewString("ec2")}
+	AWSServiceECS         = ServiceID{valuer.NewString("ecs")}
+	AWSServiceEKS         = ServiceID{valuer.NewString("eks")}
+	AWSServiceElastiCache = ServiceID{valuer.NewString("elasticache")}
+	AWSServiceLambda      = ServiceID{valuer.NewString("lambda")}
+	AWSServiceMSK         = ServiceID{valuer.NewString("msk")}
+	AWSServiceRDS         = ServiceID{valuer.NewString("rds")}
+	AWSServiceS3Sync      = ServiceID{valuer.NewString("s3sync")}
+	AWSServiceSNS         = ServiceID{valuer.NewString("sns")}
+	AWSServiceSQS         = ServiceID{valuer.NewString("sqs")}
+)
+
+func (ServiceID) Enum() []any {
+	return []any{
+		AWSServiceALB,
+		AWSServiceAPIGateway,
+		AWSServiceDynamoDB,
+		AWSServiceEC2,
+		AWSServiceECS,
+		AWSServiceEKS,
+		AWSServiceElastiCache,
+		AWSServiceLambda,
+		AWSServiceMSK,
+		AWSServiceRDS,
+		AWSServiceS3Sync,
+		AWSServiceSNS,
+		AWSServiceSQS,
+	}
+}
+
+// SupportedServices is the map of supported services for each cloud provider.
+var SupportedServices = map[CloudProviderType][]ServiceID{
+	CloudProviderTypeAWS: {
+		AWSServiceALB,
+		AWSServiceAPIGateway,
+		AWSServiceDynamoDB,
+		AWSServiceEC2,
+		AWSServiceECS,
+		AWSServiceEKS,
+		AWSServiceElastiCache,
+		AWSServiceLambda,
+		AWSServiceMSK,
+		AWSServiceRDS,
+		AWSServiceS3Sync,
+		AWSServiceSNS,
+		AWSServiceSQS,
+	},
+}
+
+// NewServiceID returns a new ServiceID from a string, validated against the supported services for the given cloud provider.
+func NewServiceID(provider CloudProviderType, service string) (ServiceID, error) {
+	services, ok := SupportedServices[provider]
+	if !ok {
+		return ServiceID{}, errors.NewInvalidInputf(ErrCodeInvalidServiceID, "no services defined for cloud provider: %s", provider)
+	}
+	for _, s := range services {
+		if s.StringValue() == service {
+			return s, nil
+		}
+	}
+	return ServiceID{}, errors.NewInvalidInputf(ErrCodeInvalidServiceID, "invalid service id %q for cloud provider %s", service, provider)
+}

tests/integration/src/querier/04_traces.py

@@ -696,7 +696,6 @@ def test_traces_list_with_corrupt_data(
     assert response.status_code == status_code
 
     if response.status_code == HTTPStatus.OK:
-
         if not results(traces):
             # No results expected
             assert response.json()["data"]["data"]["results"][0]["rows"] is None
@@ -2026,3 +2025,136 @@ def test_traces_fill_zero_formula_with_group_by(
             expected_by_ts=expectations[service_name],
             context=f"traces/fillZero/F1/{service_name}",
         )
+
+
+def test_traces_list_filter_by_trace_id(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[List[Traces]], None],
+) -> None:
+    """
+    Tests that filtering by trace_id:
+    1. Returns the matching span (narrow window, single bucket).
+    2. Does not return duplicate spans when the query window spans multiple
+       exponential buckets (>1 h)
+    3. Returns no results when the query window does not contain the trace.
+    """
+    target_trace_id = TraceIdGenerator.trace_id()
+    other_trace_id = TraceIdGenerator.trace_id()
+    span_id_root = TraceIdGenerator.span_id()
+    other_span_id = TraceIdGenerator.span_id()
+
+    now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
+
+    common_resources = {
+        "deployment.environment": "production",
+        "service.name": "trace-filter-service",
+        "cloud.provider": "integration",
+    }
+
+    insert_traces(
+        [
+            Traces(
+                timestamp=now - timedelta(seconds=10),
+                duration=timedelta(seconds=5),
+                trace_id=target_trace_id,
+                span_id=span_id_root,
+                parent_span_id="",
+                name="root-span",
+                kind=TracesKind.SPAN_KIND_SERVER,
+                status_code=TracesStatusCode.STATUS_CODE_OK,
+                status_message="",
+                resources=common_resources,
+                attributes={"http.request.method": "GET"},
+            ),
+            # span from a different trace — must not appear in results
+            Traces(
+                timestamp=now - timedelta(seconds=5),
+                duration=timedelta(seconds=1),
+                trace_id=other_trace_id,
+                span_id=other_span_id,
+                parent_span_id="",
+                name="other-root-span",
+                kind=TracesKind.SPAN_KIND_SERVER,
+                status_code=TracesStatusCode.STATUS_CODE_OK,
+                status_message="",
+                resources=common_resources,
+                attributes={},
+            ),
+        ]
+    )
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    trace_filter = f"trace_id = '{target_trace_id}'"
+
+    def _query(start_ms: int, end_ms: int) -> List:
+        response = make_query_request(
+            signoz,
+            token,
+            start_ms=start_ms,
+            end_ms=end_ms,
+            request_type="raw",
+            queries=[
+                {
+                    "type": "builder_query",
+                    "spec": {
+                        "name": "A",
+                        "signal": "traces",
+                        "disabled": False,
+                        "limit": 100,
+                        "offset": 0,
+                        "filter": {"expression": trace_filter},
+                        "order": [{"key": {"name": "timestamp"}, "direction": "desc"}],
+                        "selectFields": [
+                            {
+                                "name": "name",
+                                "fieldDataType": "string",
+                                "fieldContext": "span",
+                                "signal": "traces",
+                            }
+                        ],
+                        "having": {"expression": ""},
+                        "aggregations": [{"expression": "count()"}],
+                    },
+                }
+            ],
+        )
+        assert response.status_code == HTTPStatus.OK
+        assert response.json()["status"] == "success"
+        return response.json()["data"]["data"]["results"][0]["rows"] or []
+
+    now_ms = int(now.timestamp() * 1000)
+
+    # --- Test 1: narrow window (single bucket, <1 h) ---
+    narrow_start_ms = int((now - timedelta(minutes=5)).timestamp() * 1000)
+    narrow_rows = _query(narrow_start_ms, now_ms)
+
+    assert (
+        len(narrow_rows) == 1
+    ), f"Expected 1 span for trace_id filter (narrow window), got {len(narrow_rows)}"
+    assert narrow_rows[0]["data"]["span_id"] == span_id_root
+    assert narrow_rows[0]["data"]["trace_id"] == target_trace_id
+
+    # --- Test 2: wide window (>1 h, triggers multiple exponential buckets) ---
+    # should just return 1 span, not duplicate
+    wide_start_ms = int((now - timedelta(hours=12)).timestamp() * 1000)
+    wide_rows = _query(wide_start_ms, now_ms)
+
+    assert len(wide_rows) == 1, (
+        f"Expected 1 span for trace_id filter (wide window, multi-bucket), "
+        f"got {len(wide_rows)} — possible duplicate-span regression"
+    )
+    assert wide_rows[0]["data"]["span_id"] == span_id_root
+    assert wide_rows[0]["data"]["trace_id"] == target_trace_id
+
+    # --- Test 3: window that does not contain the trace returns no results ---
+    past_start_ms = int((now - timedelta(hours=6)).timestamp() * 1000)
+    past_end_ms = int((now - timedelta(hours=2)).timestamp() * 1000)
+    past_rows = _query(past_start_ms, past_end_ms)
+
+    assert len(past_rows) == 0, (
+        f"Expected 0 spans for trace_id filter outside time window, "
+        f"got {len(past_rows)}"
+    )