fix(apm): stop the service drilldown crashing on an empty operations list

The service detail page rebuilds its widget queries with `operation in []`
whenever the top-level operations list is momentarily empty, and the filter
grammar rejects that outright. The drilldown menu resolved that query through
/substitute_vars on a floating promise, so the 400 surfaced as an unhandled
rejection.

- catch the resolve failure in the drilldown menu and fall back to the
  unresolved query
- skip the /substitute_vars round-trip when there are no variables to
  substitute, which is every non-dashboard surface
- keep the previous operations list across a time-range change so the widget
  queries are never built with an empty list

docs/api/openapi.yml

@@ -8805,6 +8805,7 @@ components:
       - metric
       - log
       - span
+      - trace
       - resource
       - attribute
       - body
@@ -9115,6 +9116,158 @@ info:
   version: ""
 openapi: 3.0.3
 paths:
+  /api/v1/ai_observability/fields/keys:
+    get:
+      deprecated: false
+      description: This endpoint returns the field keys the AI observability explorer
+        can filter on, including the computed per-trace aggregates
+      operationId: GetAIObservabilityFieldsKeys
+      parameters:
+      - in: query
+        name: searchText
+        schema:
+          type: string
+      - in: query
+        name: fieldContext
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesFieldContext'
+      - in: query
+        name: fieldDataType
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesFieldDataType'
+      - in: query
+        name: startUnixMilli
+        schema:
+          format: int64
+          type: integer
+      - in: query
+        name: endUnixMilli
+        schema:
+          format: int64
+          type: integer
+      - in: query
+        name: limit
+        schema:
+          type: integer
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/TelemetrytypesGettableFieldKeys'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - VIEWER
+      - tokenizer:
+        - VIEWER
+      summary: Get AI observability field keys
+      tags:
+      - ai_observability
+  /api/v1/ai_observability/fields/values:
+    get:
+      deprecated: false
+      description: This endpoint returns the values the AI observability explorer
+        can filter a field key on
+      operationId: GetAIObservabilityFieldsValues
+      parameters:
+      - in: query
+        name: searchText
+        schema:
+          type: string
+      - in: query
+        name: fieldContext
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesFieldContext'
+      - in: query
+        name: fieldDataType
+        schema:
+          $ref: '#/components/schemas/TelemetrytypesFieldDataType'
+      - in: query
+        name: startUnixMilli
+        schema:
+          format: int64
+          type: integer
+      - in: query
+        name: endUnixMilli
+        schema:
+          format: int64
+          type: integer
+      - in: query
+        name: limit
+        schema:
+          type: integer
+      - in: query
+        name: name
+        schema:
+          type: string
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                properties:
+                  data:
+                    $ref: '#/components/schemas/TelemetrytypesGettableFieldValues'
+                  status:
+                    type: string
+                required:
+                - status
+                - data
+                type: object
+          description: OK
+        "401":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Unauthorized
+        "403":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Forbidden
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/RenderErrorResponse'
+          description: Internal Server Error
+      security:
+      - api_key:
+        - VIEWER
+      - tokenizer:
+        - VIEWER
+      summary: Get AI observability field values
+      tags:
+      - ai_observability
   /api/v1/alerts:
     get:
       deprecated: false

frontend/package.json

@@ -88,6 +88,7 @@
     "jest": "30.2.0",
     "js-base64": "^3.7.2",
     "lodash-es": "^4.17.21",
+    "monaco-editor": "0.55.1",
     "motion": "12.4.13",
     "nuqs": "2.8.8",
     "overlayscrollbars": "^2.16.0",
@@ -237,4 +238,4 @@
     "tmp@<0.2.6": ">=0.2.6 <0.3.0",
     "yaml@>=1.0.0 <1.10.3": ">=1.10.3 <2"
   }
-}
+}

frontend/pnpm-lock.yaml

@@ -180,6 +180,9 @@ importers:
       lodash-es:
         specifier: ^4.17.21
         version: 4.18.1
+      monaco-editor:
+        specifier: 0.55.1
+        version: 0.55.1
       motion:
         specifier: 12.4.13
         version: 12.4.13(@emotion/is-prop-valid@1.2.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
@@ -1755,105 +1758,89 @@ packages:
     resolution: {integrity: sha512-C0SqjoFKnszqa44EQ7xoaT48nnO0lOyXEULfXMWi8krrjOPGYkeK30Okzla6ATbBYsyZ0ySinK0FVkpv3DwzfQ==}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-libvips-linux-arm@1.3.0':
     resolution: {integrity: sha512-A8UpHoUDW4DwnXoV6+q3C1s7QLRAHtPDEjWuNZjwHMyoCNZnm0GeNN8ls9f/bsEYTRQRW96C/n34XJQHJ2fT7A==}
     cpu: [arm]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-libvips-linux-ppc64@1.3.0':
     resolution: {integrity: sha512-WOpkVxAjFd369iaIzEgNRreFD+gWdUMIGD5zplhNKNeqS6mm5dac3q2AFyCBmzYoAdouzZvRBgxy4z8QHZb4/A==}
     cpu: [ppc64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-libvips-linux-riscv64@1.3.0':
     resolution: {integrity: sha512-DRWw0mOHusrCCuw2rqP87oLg6PGlkomVDFqw2hIwsSfwWpu4k3XLcBPaKKl6ct/GtL/cwNkgwjV/tc0Mqht3VA==}
     cpu: [riscv64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-libvips-linux-s390x@1.3.0':
     resolution: {integrity: sha512-9APy+nFWhHS+kzLgWZfLcyrUd7YqnAQVa4BPOo4xkoHpdoktOAPG4cEr9+Jpl0TtqfVmcMJimNL5qNTyyOHZNA==}
     cpu: [s390x]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-libvips-linux-x64@1.3.0':
     resolution: {integrity: sha512-y9RNUYDe2A1UAdhLyfeOodGRszQdaEoe4nfOpp/sNVPl2CWIcUyFaDoCh4vPLPxu19803j2naLqZup2WxDXCLA==}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-libvips-linuxmusl-arm64@1.3.0':
     resolution: {integrity: sha512-cC1wkC0Mlucd0KSiGrLkJnB/ZqPvZCntc/Lk7ZnYO5ZSbF2euNek4Xvxafojq+wN1q/W0eprdpUIjUr/EV2PBg==}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@img/sharp-libvips-linuxmusl-x64@1.3.0':
     resolution: {integrity: sha512-LiYMhUZicB1QG//+RvmYZpXJO8fYRENfp+MZUCnG9aw+AKvGAy9gPaCnuwsPcBFs8EV66M0NNxj9VHcNklE8zw==}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@img/sharp-linux-arm64@0.35.0':
     resolution: {integrity: sha512-4+4XHLNT5wDT0roYlHTEmH9lDKt0acf9Tv+3hM3iceOirkxrR404/3WjAYZ9F9CkHrxeRcGLJXbi4vluMZ9O+A==}
     engines: {node: '>=20.9.0'}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-linux-arm@0.35.0':
     resolution: {integrity: sha512-VVlpEWwizEFIOom0zdoeKuO5nuTswzVE5uHcBNvHzmeHUpNFajY3HFfbQ+zIH4E2kVaZ/yVxmsShW56TtEy4uA==}
     engines: {node: '>=20.9.0'}
     cpu: [arm]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-linux-ppc64@0.35.0':
     resolution: {integrity: sha512-N3hzbEpUTJC8pWpPVJvgzGxM+so/MAXc8O2s/53B0LL9ZGpfXpME7Wizkc5d/8fRBlBtkDjzoZGDCqqNDHqLEw==}
     engines: {node: '>=20.9.0'}
     cpu: [ppc64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-linux-riscv64@0.35.0':
     resolution: {integrity: sha512-l6vmKVPnbS0RhVMbyxP5meAARsbhCnBN4fy31qz0+3a6Rv4jEqfzDrT89y6ZPkCi0AJGnwp2En528yXo401Hpw==}
     engines: {node: '>=20.9.0'}
     cpu: [riscv64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-linux-s390x@0.35.0':
     resolution: {integrity: sha512-MYlMiPFiv/EKPAHnp3yNZ9AAWFsxga9c5Bkc6wkar6bqzHLlkGVJHRm0u1ei+VXnZxp3Mz9MG9ZIsI8vSOf3sQ==}
     engines: {node: '>=20.9.0'}
     cpu: [s390x]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-linux-x64@0.35.0':
     resolution: {integrity: sha512-TYaItB5oj1ioXjhyn2xrR208vf+YuIIcHptQWRRaBmFhvIvL9D72DXN8w75xup0KXA8UdEAhQ9Qb2S49FD/9Cw==}
     engines: {node: '>=20.9.0'}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@img/sharp-linuxmusl-arm64@0.35.0':
     resolution: {integrity: sha512-DSTb6ijQzqe6DdAaOBVqJ/SYf1vO8EW5bK6X6LRXufEBebf2722VCdvBUtZ3rtV0x2ApfPNDy/p7LrrjaWjiyQ==}
     engines: {node: '>=20.9.0'}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@img/sharp-linuxmusl-x64@0.35.0':
     resolution: {integrity: sha512-K7ykQ+26Rt6+4BTU80AuGgTPIYX86UxiAKT4rcXX/WNTo7k1ZxpKz+TguHnwVpCqQK3B5PK0vZ0ZBe6nz/ib1w==}
     engines: {node: '>=20.9.0'}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@img/sharp-wasm32@0.35.0':
     resolution: {integrity: sha512-9woLIFORERCr+6cWu87dQ22J34EExkhc73U1kZW0c+RclQqWetoodByp4dWZ/hN8/KVmTRAx2HOnUwib8AwZdA==}
@@ -2234,56 +2221,48 @@ packages:
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@oxfmt/binding-linux-arm64-musl@0.54.0':
     resolution: {integrity: sha512-QrwJlBFFKnxOd95TAaszpMbZBLzMoYMpGaQTZF8oibacnF5rv8l12IhILhQRPmksWiBqg0YSe2Mnl7ayeJAHSA==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@oxfmt/binding-linux-ppc64-gnu@0.54.0':
     resolution: {integrity: sha512-WILatiol/TUHTlhod7R09+7Az/XlhKwmY1MHfLZNmewltPWNN/EwxP2rQSHahibZ/cB8gmckEBjBOByD+5bYsQ==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [ppc64]
     os: [linux]
-    libc: [glibc]
 
   '@oxfmt/binding-linux-riscv64-gnu@0.54.0':
     resolution: {integrity: sha512-f05YMG4BH4G8S4ME6UM6fi1MnJ9094mrnvO5Pa4SJlMfWlUM+1/ZWMEF4NnjM7shZAvbHsHRuVYpUo0PHC4P9Q==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [riscv64]
     os: [linux]
-    libc: [glibc]
 
   '@oxfmt/binding-linux-riscv64-musl@0.54.0':
     resolution: {integrity: sha512-UfL+2hj1ClNqcCRT9s8vBU4axDpjxgVxX96G+9DYAYjoc5b0u15CJtn2jgsi9iM+EbGNc5CW1HVRgwVu76UsSA==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [riscv64]
     os: [linux]
-    libc: [musl]
 
   '@oxfmt/binding-linux-s390x-gnu@0.54.0':
     resolution: {integrity: sha512-3/XZe931Hka+J6NjnaqJzYpsWWxDTuRdUdwSQHnOuJEgbC+SehIMFJS8hsEjV7LBhVSL2OCnRLvbVW8O97XIyw==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [s390x]
     os: [linux]
-    libc: [glibc]
 
   '@oxfmt/binding-linux-x64-gnu@0.54.0':
     resolution: {integrity: sha512-Ik93RlObtu43GbxApafayFjwYE06L6Xr08cSwpBPYbDrLp2ReZx0Jm1DqwRyYRnukUJy+rK2WaEvUQOxdytU9Q==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@oxfmt/binding-linux-x64-musl@0.54.0':
     resolution: {integrity: sha512-yZcakmPlD86CNymknd7KfW+FH+qfbqJH+i0h69CYfV1+KMoVeM9UED+8+TDVoU4haxI0NxY7RPCvRLy3Sqd2Qg==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@oxfmt/binding-openharmony-arm64@0.54.0':
     resolution: {integrity: sha512-GiVBZNnEZnKu00f1jTg49nomv187d0GQX+O+ocykoLeiaALuEO+swoTehHn9TehTfi7V8H0i0e/yvUjCqnwk1w==}
@@ -2386,56 +2365,48 @@ packages:
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@oxlint/binding-linux-arm64-musl@1.69.0':
     resolution: {integrity: sha512-xEPpNppTfN1l/nM7gYSf9iocscu/as+p/7vxkLeLEKnYU+09Dm+5V6IhDYDh+Uz6FajEupWwCLt5SOG0y1PCKg==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@oxlint/binding-linux-ppc64-gnu@1.69.0':
     resolution: {integrity: sha512-Ug0+eU7HJBlek+SjklYH62IlOMirEJsdxpihH0kSqX0XdrDD4NdHpQc10fK1JC35yn6KrrcN+uYzlHD38XAf8Q==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [ppc64]
     os: [linux]
-    libc: [glibc]
 
   '@oxlint/binding-linux-riscv64-gnu@1.69.0':
     resolution: {integrity: sha512-iEyI3GIg0l/s3G4qy2TlaaWKdzj4PJJStwtlocpDTC00PY9hZueotf6OKUj9+yfQh0lrpBW/pLMgTztbAHKJEg==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [riscv64]
     os: [linux]
-    libc: [glibc]
 
   '@oxlint/binding-linux-riscv64-musl@1.69.0':
     resolution: {integrity: sha512-NjHjpiI4WIKSMwuoJSZi5VToPeoYOS1FR52HLIDG6lidMdqquusgtODb4iLk0+lb1q3Z0nv2/aPRcC/olmpQGg==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [riscv64]
     os: [linux]
-    libc: [musl]
 
   '@oxlint/binding-linux-s390x-gnu@1.69.0':
     resolution: {integrity: sha512-Ai/prDewoItkDXbp38gwGZi41DycZbUTZJ3UidwoHgQC0/DaqC2TGdtBTQLJ6hSD+SAxASzh8+/eSBPmxfOacA==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [s390x]
     os: [linux]
-    libc: [glibc]
 
   '@oxlint/binding-linux-x64-gnu@1.69.0':
     resolution: {integrity: sha512-Gt3KHgp46mRKz4sJeaASmKvD8ayXookRw07RMf+NowhEztGGDZ7VrXpoW96XuKJLjFukWizOFVNjmYb/u7caNQ==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@oxlint/binding-linux-x64-musl@1.69.0':
     resolution: {integrity: sha512-7tQhJ2+p/oHv1zcfnjYI7YVzC/7iBaVOfIvFYtxdJ5F45mWgEdrCyXZXZGfiLey5t/5JhOhsaMnnv1kAzckd7g==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@oxlint/binding-openharmony-arm64@1.69.0':
     resolution: {integrity: sha512-vmWz6TKp/3hfA4lksR0zHBv/6xuX1jhym6eqOjdH2DXsDDHZWcp2f0KG0VCAnlVbIrjk29G4wAWMXb/Hn1YobA==}
@@ -2490,42 +2461,36 @@ packages:
     engines: {node: '>= 10.0.0'}
     cpu: [arm]
     os: [linux]
-    libc: [glibc]
 
   '@parcel/watcher-linux-arm-musl@2.5.1':
     resolution: {integrity: sha512-6E+m/Mm1t1yhB8X412stiKFG3XykmgdIOqhjWj+VL8oHkKABfu/gjFj8DvLrYVHSBNC+/u5PeNrujiSQ1zwd1Q==}
     engines: {node: '>= 10.0.0'}
     cpu: [arm]
     os: [linux]
-    libc: [musl]
 
   '@parcel/watcher-linux-arm64-glibc@2.5.1':
     resolution: {integrity: sha512-LrGp+f02yU3BN9A+DGuY3v3bmnFUggAITBGriZHUREfNEzZh/GO06FF5u2kx8x+GBEUYfyTGamol4j3m9ANe8w==}
     engines: {node: '>= 10.0.0'}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@parcel/watcher-linux-arm64-musl@2.5.1':
     resolution: {integrity: sha512-cFOjABi92pMYRXS7AcQv9/M1YuKRw8SZniCDw0ssQb/noPkRzA+HBDkwmyOJYp5wXcsTrhxO0zq1U11cK9jsFg==}
     engines: {node: '>= 10.0.0'}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@parcel/watcher-linux-x64-glibc@2.5.1':
     resolution: {integrity: sha512-GcESn8NZySmfwlTsIur+49yDqSny2IhPeZfXunQi48DMugKeZ7uy1FX83pO0X22sHntJ4Ub+9k34XQCX+oHt2A==}
     engines: {node: '>= 10.0.0'}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@parcel/watcher-linux-x64-musl@2.5.1':
     resolution: {integrity: sha512-n0E2EQbatQ3bXhcH2D1XIAANAcTZkQICBPVaxMeaCVBtOpBZpWJuf7LwyWPSBDITb7In8mqQgJ7gH8CILCURXg==}
     engines: {node: '>= 10.0.0'}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@parcel/watcher-win32-arm64@2.5.1':
     resolution: {integrity: sha512-RFzklRvmc3PkjKjry3hLF9wD7ppR4AKcWNzH7kXR7GUe0Igb3Nz8fyPwtZCSquGrhU5HhUNDr/mKBqj7tqA2Vw==}
@@ -3142,28 +3107,24 @@ packages:
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@rolldown/binding-linux-arm64-musl@1.0.0-beta.53':
     resolution: {integrity: sha512-bGe5EBB8FVjHBR1mOLOPEFg1Lp3//7geqWkU5NIhxe+yH0W8FVrQ6WRYOap4SUTKdklD/dC4qPLREkMMQ855FA==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@rolldown/binding-linux-x64-gnu@1.0.0-beta.53':
     resolution: {integrity: sha512-qL+63WKVQs1CMvFedlPt0U9PiEKJOAL/bsHMKUDS6Vp2Q+YAv/QLPu8rcvkfIMvQ0FPU2WL0aX4eWwF6e/GAnA==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@rolldown/binding-linux-x64-musl@1.0.0-beta.53':
     resolution: {integrity: sha512-VGl9JIGjoJh3H8Mb+7xnVqODajBmrdOOb9lxWXdcmxyI+zjB2sux69br0hZJDTyLJfvBoYm439zPACYbCjGRmw==}
     engines: {node: ^20.19.0 || >=22.12.0}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@rolldown/binding-openharmony-arm64@1.0.0-beta.53':
     resolution: {integrity: sha512-B4iIserJXuSnNzA5xBLFUIjTfhNy7d9sq4FUMQY3GhQWGVhS2RWWzzDnkSU6MUt7/aHUrep0CdQfXUJI9D3W7A==}
@@ -3824,49 +3785,41 @@ packages:
     resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@unrs/resolver-binding-linux-arm64-musl@1.11.1':
     resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1':
     resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==}
     cpu: [ppc64]
     os: [linux]
-    libc: [glibc]
 
   '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1':
     resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==}
     cpu: [riscv64]
     os: [linux]
-    libc: [glibc]
 
   '@unrs/resolver-binding-linux-riscv64-musl@1.11.1':
     resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==}
     cpu: [riscv64]
     os: [linux]
-    libc: [musl]
 
   '@unrs/resolver-binding-linux-s390x-gnu@1.11.1':
     resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==}
     cpu: [s390x]
     os: [linux]
-    libc: [glibc]
 
   '@unrs/resolver-binding-linux-x64-gnu@1.11.1':
     resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@unrs/resolver-binding-linux-x64-musl@1.11.1':
     resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@unrs/resolver-binding-wasm32-wasi@1.11.1':
     resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==}
@@ -6275,28 +6228,24 @@ packages:
     engines: {node: '>= 12.0.0'}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   lightningcss-linux-arm64-musl@1.31.1:
     resolution: {integrity: sha512-mVZ7Pg2zIbe3XlNbZJdjs86YViQFoJSpc41CbVmKBPiGmC4YrfeOyz65ms2qpAobVd7WQsbW4PdsSJEMymyIMg==}
     engines: {node: '>= 12.0.0'}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   lightningcss-linux-x64-gnu@1.31.1:
     resolution: {integrity: sha512-xGlFWRMl+0KvUhgySdIaReQdB4FNudfUTARn7q0hh/V67PVGCs3ADFjw+6++kG1RNd0zdGRlEKa+T13/tQjPMA==}
     engines: {node: '>= 12.0.0'}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   lightningcss-linux-x64-musl@1.31.1:
     resolution: {integrity: sha512-eowF8PrKHw9LpoZii5tdZwnBcYDxRw2rRCyvAXLi34iyeYfqCQNA9rmUM0ce62NlPhCvof1+9ivRaTY6pSKDaA==}
     engines: {node: '>= 12.0.0'}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   lightningcss-win32-arm64-msvc@1.31.1:
     resolution: {integrity: sha512-aJReEbSEQzx1uBlQizAOBSjcmr9dCdL3XuC/6HLXAxmtErsj2ICo5yYggg1qOODQMtnjNQv2UHb9NpOuFtYe4w==}

frontend/src/api/generated/services/ai-observability/index.ts

@@ -0,0 +1,236 @@
+/**
+ * ! Do not edit manually
+ * * The file has been auto-generated using Orval for SigNoz
+ * * regenerate with 'pnpm generate:api'
+ * SigNoz
+ */
+import { useQuery } from 'react-query';
+import type {
+	InvalidateOptions,
+	QueryClient,
+	QueryFunction,
+	QueryKey,
+	UseQueryOptions,
+	UseQueryResult,
+} from 'react-query';
+
+import type {
+	GetAIObservabilityFieldsKeys200,
+	GetAIObservabilityFieldsKeysParams,
+	GetAIObservabilityFieldsValues200,
+	GetAIObservabilityFieldsValuesParams,
+	RenderErrorResponseDTO,
+} from '../sigNoz.schemas';
+
+import { GeneratedAPIInstance } from '../../../generatedAPIInstance';
+import type { ErrorType } from '../../../generatedAPIInstance';
+
+/**
+ * This endpoint returns the field keys the AI observability explorer can filter on, including the computed per-trace aggregates
+ * @summary Get AI observability field keys
+ */
+export const getAIObservabilityFieldsKeys = (
+	params?: GetAIObservabilityFieldsKeysParams,
+	signal?: AbortSignal,
+) => {
+	return GeneratedAPIInstance<GetAIObservabilityFieldsKeys200>({
+		url: `/api/v1/ai_observability/fields/keys`,
+		method: 'GET',
+		params,
+		signal,
+	});
+};
+
+export const getGetAIObservabilityFieldsKeysQueryKey = (
+	params?: GetAIObservabilityFieldsKeysParams,
+) => {
+	return [
+		`/api/v1/ai_observability/fields/keys`,
+		...(params ? [params] : []),
+	] as const;
+};
+
+export const getGetAIObservabilityFieldsKeysQueryOptions = <
+	TData = Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+	TError = ErrorType<RenderErrorResponseDTO>,
+>(
+	params?: GetAIObservabilityFieldsKeysParams,
+	options?: {
+		query?: UseQueryOptions<
+			Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+			TError,
+			TData
+		>;
+	},
+) => {
+	const { query: queryOptions } = options ?? {};
+
+	const queryKey =
+		queryOptions?.queryKey ?? getGetAIObservabilityFieldsKeysQueryKey(params);
+
+	const queryFn: QueryFunction<
+		Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>
+	> = ({ signal }) => getAIObservabilityFieldsKeys(params, signal);
+
+	return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
+		Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+		TError,
+		TData
+	> & { queryKey: QueryKey };
+};
+
+export type GetAIObservabilityFieldsKeysQueryResult = NonNullable<
+	Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>
+>;
+export type GetAIObservabilityFieldsKeysQueryError =
+	ErrorType<RenderErrorResponseDTO>;
+
+/**
+ * @summary Get AI observability field keys
+ */
+
+export function useGetAIObservabilityFieldsKeys<
+	TData = Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+	TError = ErrorType<RenderErrorResponseDTO>,
+>(
+	params?: GetAIObservabilityFieldsKeysParams,
+	options?: {
+		query?: UseQueryOptions<
+			Awaited<ReturnType<typeof getAIObservabilityFieldsKeys>>,
+			TError,
+			TData
+		>;
+	},
+): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
+	const queryOptions = getGetAIObservabilityFieldsKeysQueryOptions(
+		params,
+		options,
+	);
+
+	const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
+		queryKey: QueryKey;
+	};
+
+	return { ...query, queryKey: queryOptions.queryKey };
+}
+
+/**
+ * @summary Get AI observability field keys
+ */
+export const invalidateGetAIObservabilityFieldsKeys = async (
+	queryClient: QueryClient,
+	params?: GetAIObservabilityFieldsKeysParams,
+	options?: InvalidateOptions,
+): Promise<QueryClient> => {
+	await queryClient.invalidateQueries(
+		{ queryKey: getGetAIObservabilityFieldsKeysQueryKey(params) },
+		options,
+	);
+
+	return queryClient;
+};
+
+/**
+ * This endpoint returns the values the AI observability explorer can filter a field key on
+ * @summary Get AI observability field values
+ */
+export const getAIObservabilityFieldsValues = (
+	params?: GetAIObservabilityFieldsValuesParams,
+	signal?: AbortSignal,
+) => {
+	return GeneratedAPIInstance<GetAIObservabilityFieldsValues200>({
+		url: `/api/v1/ai_observability/fields/values`,
+		method: 'GET',
+		params,
+		signal,
+	});
+};
+
+export const getGetAIObservabilityFieldsValuesQueryKey = (
+	params?: GetAIObservabilityFieldsValuesParams,
+) => {
+	return [
+		`/api/v1/ai_observability/fields/values`,
+		...(params ? [params] : []),
+	] as const;
+};
+
+export const getGetAIObservabilityFieldsValuesQueryOptions = <
+	TData = Awaited<ReturnType<typeof getAIObservabilityFieldsValues>>,
+	TError = ErrorType<RenderErrorResponseDTO>,
+>(
+	params?: GetAIObservabilityFieldsValuesParams,
+	options?: {
+		query?: UseQueryOptions<
+			Awaited<ReturnType<typeof getAIObservabilityFieldsValues>>,
+			TError,
+			TData
+		>;
+	},
+) => {
+	const { query: queryOptions } = options ?? {};
+
+	const queryKey =
+		queryOptions?.queryKey ?? getGetAIObservabilityFieldsValuesQueryKey(params);
+
+	const queryFn: QueryFunction<
+		Awaited<ReturnType<typeof getAIObservabilityFieldsValues>>
+	> = ({ signal }) => getAIObservabilityFieldsValues(params, signal);
+
+	return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
+		Awaited<ReturnType<typeof getAIObservabilityFieldsValues>>,
+		TError,
+		TData
+	> & { queryKey: QueryKey };
+};
+
+export type GetAIObservabilityFieldsValuesQueryResult = NonNullable<
+	Awaited<ReturnType<typeof getAIObservabilityFieldsValues>>
+>;
+export type GetAIObservabilityFieldsValuesQueryError =
+	ErrorType<RenderErrorResponseDTO>;
+
+/**
+ * @summary Get AI observability field values
+ */
+
+export function useGetAIObservabilityFieldsValues<
+	TData = Awaited<ReturnType<typeof getAIObservabilityFieldsValues>>,
+	TError = ErrorType<RenderErrorResponseDTO>,
+>(
+	params?: GetAIObservabilityFieldsValuesParams,
+	options?: {
+		query?: UseQueryOptions<
+			Awaited<ReturnType<typeof getAIObservabilityFieldsValues>>,
+			TError,
+			TData
+		>;
+	},
+): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
+	const queryOptions = getGetAIObservabilityFieldsValuesQueryOptions(
+		params,
+		options,
+	);
+
+	const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
+		queryKey: QueryKey;
+	};
+
+	return { ...query, queryKey: queryOptions.queryKey };
+}
+
+/**
+ * @summary Get AI observability field values
+ */
+export const invalidateGetAIObservabilityFieldsValues = async (
+	queryClient: QueryClient,
+	params?: GetAIObservabilityFieldsValuesParams,
+	options?: InvalidateOptions,
+): Promise<QueryClient> => {
+	await queryClient.invalidateQueries(
+		{ queryKey: getGetAIObservabilityFieldsValuesQueryKey(params) },
+		options,
+	);
+
+	return queryClient;
+};

frontend/src/api/generated/services/sigNoz.schemas.ts

@@ -3490,6 +3490,7 @@ export enum TelemetrytypesFieldContextDTO {
 	metric = 'metric',
 	log = 'log',
 	span = 'span',
+	trace = 'trace',
 	resource = 'resource',
 	attribute = 'attribute',
 	body = 'body',
@@ -10175,6 +10176,93 @@ export interface ZeustypesPostableProfileDTO {
 	where_did_you_discover_signoz: string;
 }
 
+export type GetAIObservabilityFieldsKeysParams = {
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	searchText?: string;
+	/**
+	 * @description undefined
+	 */
+	fieldContext?: TelemetrytypesFieldContextDTO;
+	/**
+	 * @description undefined
+	 */
+	fieldDataType?: TelemetrytypesFieldDataTypeDTO;
+	/**
+	 * @type integer
+	 * @format int64
+	 * @description undefined
+	 */
+	startUnixMilli?: number;
+	/**
+	 * @type integer
+	 * @format int64
+	 * @description undefined
+	 */
+	endUnixMilli?: number;
+	/**
+	 * @type integer
+	 * @description undefined
+	 */
+	limit?: number;
+};
+
+export type GetAIObservabilityFieldsKeys200 = {
+	data: TelemetrytypesGettableFieldKeysDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
+export type GetAIObservabilityFieldsValuesParams = {
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	searchText?: string;
+	/**
+	 * @description undefined
+	 */
+	fieldContext?: TelemetrytypesFieldContextDTO;
+	/**
+	 * @description undefined
+	 */
+	fieldDataType?: TelemetrytypesFieldDataTypeDTO;
+	/**
+	 * @type integer
+	 * @format int64
+	 * @description undefined
+	 */
+	startUnixMilli?: number;
+	/**
+	 * @type integer
+	 * @format int64
+	 * @description undefined
+	 */
+	endUnixMilli?: number;
+	/**
+	 * @type integer
+	 * @description undefined
+	 */
+	limit?: number;
+	/**
+	 * @type string
+	 * @description undefined
+	 */
+	name?: string;
+};
+
+export type GetAIObservabilityFieldsValues200 = {
+	data: TelemetrytypesGettableFieldValuesDTO;
+	/**
+	 * @type string
+	 */
+	status: string;
+};
+
 export type GetAlerts200 = {
 	/**
 	 * @type array

frontend/src/container/AlertHistory/Timeline/Table/utils.ts

@@ -9,6 +9,7 @@ const fieldContextToSuggestionMap: Record<
 	[TelemetrytypesFieldContextDTO.span]: 'span',
 	[TelemetrytypesFieldContextDTO.attribute]: 'attribute',
 	// no maps for the following values on suggestion context
+	[TelemetrytypesFieldContextDTO.trace]: undefined,
 	[TelemetrytypesFieldContextDTO.body]: undefined,
 	[TelemetrytypesFieldContextDTO.metric]: undefined,
 	[TelemetrytypesFieldContextDTO.log]: undefined,

frontend/src/container/GridCardLayout/__tests__/useResolveQuery.test.tsx

@@ -0,0 +1,91 @@
+import { renderHook } from '@testing-library/react';
+import { PANEL_TYPES } from 'constants/queryBuilder';
+import MockQueryClientProvider from 'providers/test/MockQueryClientProvider';
+import { Query } from 'types/api/queryBuilder/queryBuilderData';
+
+import useUpdatedQuery from '../useResolveQuery';
+
+const mockGetSubstituteVars = jest.fn();
+const mockDynamicVariables: unknown[] = [];
+
+jest.mock('api/dashboard/substitute_vars', () => ({
+	getSubstituteVars: (...args: unknown[]): unknown =>
+		mockGetSubstituteVars(...args),
+}));
+
+jest.mock('api/v5/v5', () => ({
+	prepareQueryRangePayloadV5: (): { queryPayload: unknown } => ({
+		queryPayload: { start: 0, end: 1 },
+	}),
+}));
+
+jest.mock(
+	'lib/newQueryBuilder/queryBuilderMappers/mapQueryDataFromApi',
+	() => ({
+		mapQueryDataFromApi: (): Query => ({ resolved: true }) as unknown as Query,
+	}),
+);
+
+jest.mock('hooks/dashboard/useDashboardVariablesByType', () => ({
+	useDashboardVariablesByType: (): unknown[] => mockDynamicVariables,
+}));
+
+jest.mock('react-redux', () => ({
+	...jest.requireActual('react-redux'),
+	useSelector: (): unknown => ({
+		selectedTime: 'GLOBAL_TIME',
+	}),
+}));
+
+const QUERY = { builder: { queryData: [] } } as unknown as Query;
+
+const WIDGET_CONFIG = {
+	query: QUERY,
+	panelTypes: PANEL_TYPES.TIME_SERIES,
+	timePreferance: 'GLOBAL_TIME' as const,
+};
+
+describe('useResolveQuery', () => {
+	beforeEach(() => {
+		jest.clearAllMocks();
+		mockDynamicVariables.length = 0;
+	});
+
+	it('skips the substitute_vars round-trip when there are no variables', async () => {
+		const { result } = renderHook(() => useUpdatedQuery(), {
+			wrapper: MockQueryClientProvider,
+		});
+
+		const resolved = await result.current.getUpdatedQuery({
+			widgetConfig: WIDGET_CONFIG,
+		});
+
+		expect(mockGetSubstituteVars).not.toHaveBeenCalled();
+		expect(resolved).toBe(QUERY);
+	});
+
+	it('resolves through substitute_vars when the dashboard has variables', async () => {
+		mockGetSubstituteVars.mockResolvedValue({
+			httpStatusCode: 200,
+			data: { compositeQuery: {} },
+		});
+
+		const { result } = renderHook(() => useUpdatedQuery(), {
+			wrapper: MockQueryClientProvider,
+		});
+
+		const resolved = await result.current.getUpdatedQuery({
+			widgetConfig: WIDGET_CONFIG,
+			dashboardData: {
+				data: {
+					variables: {
+						env: { name: 'env', selectedValue: 'prod' },
+					},
+				},
+			},
+		});
+
+		expect(mockGetSubstituteVars).toHaveBeenCalledTimes(1);
+		expect(resolved).toStrictEqual({ resolved: true });
+	});
+});

frontend/src/container/GridCardLayout/useResolveQuery.ts

@@ -2,6 +2,7 @@ import { useCallback } from 'react';
 import { useMutation } from 'react-query';
 // eslint-disable-next-line no-restricted-imports
 import { useSelector } from 'react-redux';
+import { isEmpty } from 'lodash-es';
 import { getSubstituteVars } from 'api/dashboard/substitute_vars';
 import { prepareQueryRangePayloadV5 } from 'api/v5/v5';
 import { PANEL_TYPES } from 'constants/queryBuilder';
@@ -46,13 +47,21 @@ function useUpdatedQuery(): UseUpdatedQueryResult {
 			widgetConfig,
 			dashboardData,
 		}: UseUpdatedQueryOptions): Promise<Query> => {
+			const variables = getDashboardVariables(dashboardData?.data?.variables);
+
+			// `/substitute_vars` only rewrites `$variable` references, so on surfaces with no
+			// dashboard behind them (APM, Celery, API monitoring) the round-trip is a no-op.
+			if (isEmpty(variables) && isEmpty(dashboardDynamicVariables)) {
+				return widgetConfig.query;
+			}
+
 			// Prepare query payload with resolved variables
 			const { queryPayload } = prepareQueryRangePayloadV5({
 				query: widgetConfig.query,
 				graphType: getGraphType(widgetConfig.panelTypes),
 				selectedTime: widgetConfig.timePreferance,
 				globalSelectedInterval,
-				variables: getDashboardVariables(dashboardData?.data?.variables),
+				variables,
 				originalGraphType: widgetConfig.panelTypes,
 				dynamicVariables: dashboardDynamicVariables,
 			});

frontend/src/container/MetricsApplication/Tabs/Overview.tsx

@@ -124,6 +124,9 @@ function Application(): JSX.Element {
 				start: minTime,
 				end: maxTime,
 			}),
+		// the time range is part of the key, so without this every window change blanks the
+		// operations list and the widgets below are rebuilt with an empty `operation in []`
+		keepPreviousData: true,
 	});
 
 	const selectedTraceTags: string = JSON.stringify(

frontend/src/container/QueryTable/Drilldown/__tests__/useBaseAggregateOptions.test.tsx

@@ -0,0 +1,88 @@
+import { renderHook, waitFor } from '@testing-library/react';
+import { Query } from 'types/api/queryBuilder/queryBuilderData';
+
+import useBaseAggregateOptions from '../useBaseAggregateOptions';
+
+const mockGetUpdatedQuery = jest.fn();
+const mockNotificationsError = jest.fn();
+
+jest.mock('container/GridCardLayout/useResolveQuery', () => ({
+	__esModule: true,
+	default: (): unknown => ({
+		getUpdatedQuery: mockGetUpdatedQuery,
+		isLoading: false,
+	}),
+}));
+
+jest.mock('hooks/useNotifications', () => ({
+	useNotifications: (): unknown => ({
+		notifications: { error: mockNotificationsError },
+	}),
+}));
+
+jest.mock('providers/Dashboard/store/useDashboardStore', () => ({
+	useDashboardStore: (): unknown => ({ dashboardData: undefined }),
+}));
+
+jest.mock('hooks/dashboard/useContextVariables', () => ({
+	__esModule: true,
+	default: (): unknown => ({ processedVariables: {} }),
+}));
+
+jest.mock('hooks/useSafeNavigate', () => ({
+	useSafeNavigate: (): unknown => ({ safeNavigate: jest.fn() }),
+}));
+
+jest.mock('react-router-dom', () => ({
+	...jest.requireActual('react-router-dom'),
+	useLocation: (): { pathname: string } => ({ pathname: '/services/socky-api' }),
+}));
+
+const QUERY = {
+	builder: {
+		queryData: [{ queryName: 'A', dataSource: 'traces', aggregations: [] }],
+	},
+} as unknown as Query;
+
+const AGGREGATE_DATA = { queryName: 'A', filters: [] };
+
+const renderOptions = (): ReturnType<typeof renderHook> =>
+	renderHook(() =>
+		useBaseAggregateOptions({
+			query: QUERY,
+			onClose: jest.fn(),
+			subMenu: '',
+			setSubMenu: jest.fn(),
+			aggregateData: AGGREGATE_DATA,
+			fieldVariables: {},
+		}),
+	);
+
+describe('useBaseAggregateOptions', () => {
+	beforeEach(() => {
+		jest.clearAllMocks();
+	});
+
+	it('notifies and keeps the unresolved query when variable resolution fails', async () => {
+		mockGetUpdatedQuery.mockRejectedValue(
+			new Error('syntax errors in expression'),
+		);
+
+		renderOptions();
+
+		await waitFor(() =>
+			expect(mockNotificationsError).toHaveBeenCalledWith({
+				message: 'Unable to resolve variables',
+			}),
+		);
+	});
+
+	it('does not notify when variable resolution succeeds', async () => {
+		mockGetUpdatedQuery.mockResolvedValue(QUERY);
+
+		renderOptions();
+
+		await waitFor(() => expect(mockGetUpdatedQuery).toHaveBeenCalled());
+		expect(mockNotificationsError).not.toHaveBeenCalled();
+	});
+});

frontend/src/container/QueryTable/Drilldown/useBaseAggregateOptions.tsx

@@ -6,6 +6,7 @@ import { PANEL_TYPES } from 'constants/queryBuilder';
 import useUpdatedQuery from 'container/GridCardLayout/useResolveQuery';
 import { processContextLinks } from 'container/NewWidget/RightContainer/ContextLinks/utils';
 import useContextVariables from 'hooks/dashboard/useContextVariables';
+import { useNotifications } from 'hooks/useNotifications';
 import ContextMenu from 'periscope/components/ContextMenu';
 import { useDashboardStore } from 'providers/Dashboard/store/useDashboardStore';
 import { ContextLinksData } from 'types/api/dashboard/getAll';
@@ -50,23 +51,25 @@ const useBaseAggregateOptions = ({
 	const { getUpdatedQuery, isLoading: isResolveQueryLoading } =
 		useUpdatedQuery();
 	const { dashboardData } = useDashboardStore();
+	const { notifications } = useNotifications();
 
 	useEffect(() => {
 		if (!aggregateData) {
 			return;
 		}
-		const resolveQuery = async (): Promise<void> => {
-			const updatedQuery = await getUpdatedQuery({
-				widgetConfig: {
-					query,
-					panelTypes: panelType || PANEL_TYPES.TIME_SERIES,
-					timePreferance: 'GLOBAL_TIME',
-				},
-				dashboardData,
+		getUpdatedQuery({
+			widgetConfig: {
+				query,
+				panelTypes: panelType || PANEL_TYPES.TIME_SERIES,
+				timePreferance: 'GLOBAL_TIME',
+			},
+			dashboardData,
+		})
+			.then(setResolvedQuery)
+			.catch(() => {
+				setResolvedQuery(query);
+				notifications.error({ message: 'Unable to resolve variables' });
 			});
-			setResolvedQuery(updatedQuery);
-		};
-		resolveQuery();
 		// eslint-disable-next-line react-hooks/exhaustive-deps
 	}, [query, aggregateData, panelType]);
 

frontend/src/index.tsx

@@ -15,6 +15,8 @@ import store from 'store';
 import APIError from 'types/api/error';
 import { installTranslationResilience } from 'translation-resilience';
 
+import 'lib/monaco/setup';
+
 import './ReactI18';
 
 import 'styles.scss';

frontend/src/lib/monaco/setup.ts

@@ -0,0 +1,21 @@
+import { loader } from '@monaco-editor/react';
+import * as monaco from 'monaco-editor';
+import EditorWorker from 'monaco-editor/esm/vs/editor/editor.worker?worker';
+import JsonWorker from 'monaco-editor/esm/vs/language/json/json.worker?worker';
+
+// Serve Monaco's workers from our own origin instead of letting @monaco-editor/loader
+// fetch them from cdn.jsdelivr.net at runtime. The CDN default breaks the editor in
+// air-gapped/on-prem installs, CDN-blocking corporate networks, and regions where
+// jsdelivr is unreachable. Ref: engineering-pod#5871, SIGNOZ-UI-5G0.
+self.MonacoEnvironment = {
+	getWorker(_workerId: string, label: string): Worker {
+		if (label === 'json') {
+			return new JsonWorker(); // JSON language service
+		}
+		// SigNoz editors use JSON + a hand-registered ClickHouse tokenizer (no worker),
+		// so the base editor worker covers everything else.
+		return new EditorWorker(); // base worker — sql, yaml, plaintext etc.
+	},
+};
+
+loader.config({ monaco });

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/kinds/NumberPanel/__tests__/Renderer.test.tsx

@@ -125,6 +125,37 @@ describe('NumberPanelRenderer', () => {
 		expect(queryByText('3.14159')).not.toBeInTheDocument();
 	});
 
+	// #7669: large scalars are unreadable as an undelimited digit run.
+	it('groups large values into thousands', () => {
+		const { getByText, queryByText } = renderPanel({
+			panel: panelWith({}),
+			data: dataWith('1234567'),
+		});
+
+		expect(getByText('1,234,567')).toBeInTheDocument();
+		expect(queryByText('1234567')).not.toBeInTheDocument();
+	});
+
+	it('groups the value while keeping its unit separate', () => {
+		const { getByText } = renderPanel({
+			panel: panelWith({ formatting: { unit: 'percent' } }),
+			data: dataWith('1234567'),
+		});
+
+		expect(getByText('1,234,567')).toBeInTheDocument();
+		expect(getByText('%')).toBeInTheDocument();
+	});
+
+	it('leaves a unit-scaled value ungrouped', () => {
+		const { getByText } = renderPanel({
+			panel: panelWith({ formatting: { unit: 'bytes' } }),
+			data: dataWith('1234567'),
+		});
+
+		expect(getByText('1.18')).toBeInTheDocument();
+		expect(getByText('MiB')).toBeInTheDocument();
+	});
+
 	it('renders No Data when the response has no scalar results', () => {
 		const { getByTestId } = renderPanel({ data: emptyData });
 

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/kinds/TablePanel/__tests__/Renderer.test.tsx

@@ -93,6 +93,15 @@ describe('TablePanelRenderer', () => {
 		expect(getByText('cartservice')).toBeInTheDocument();
 	});
 
+	// Value cells share `formatPanelValue`, so they group like the Number panel.
+	it('groups large value cells into thousands', () => {
+		const { getByText } = renderPanel({
+			data: dataWith([['frontend', 1234567]]),
+		});
+
+		expect(getByText('1,234,567')).toBeInTheDocument();
+	});
+
 	it('renders No Data when the response has no scalar results', () => {
 		const { getByTestId } = renderPanel({ data: emptyData });
 

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/__tests__/formatPanelValue.test.ts

@@ -30,4 +30,14 @@ describe('formatPanelValue', () => {
 	it('renders whole numbers without a trailing decimal', () => {
 		expect(formatPanelValue(5, undefined, 2)).toBe('5');
 	});
+
+	it('groups the integer part into thousands', () => {
+		expect(formatPanelValue(1234567, undefined, 2)).toBe('1,234,567');
+		expect(formatPanelValue(1234567, 'percent', 2)).toBe('1,234,567%');
+		expect(formatPanelValue(1234567.891, undefined, 2)).toBe('1,234,567.89');
+	});
+
+	it('leaves unit-scaled values ungrouped', () => {
+		expect(formatPanelValue(1234567, 'bytes', 2)).toBe('1.18 MiB');
+	});
 });

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/__tests__/groupThousands.test.ts

@@ -0,0 +1,55 @@
+import { groupThousands } from '../groupThousands';
+
+describe('groupThousands', () => {
+	it('groups the integer digits of a plain number', () => {
+		expect(groupThousands('1234567')).toBe('1,234,567');
+		expect(groupThousands('1000')).toBe('1,000');
+		expect(groupThousands('1000000000000000000000')).toBe(
+			'1,000,000,000,000,000,000,000',
+		);
+	});
+
+	it('leaves values below a thousand alone', () => {
+		expect(groupThousands('0')).toBe('0');
+		expect(groupThousands('999')).toBe('999');
+		expect(groupThousands('295.43')).toBe('295.43');
+	});
+
+	it('groups only the integer part', () => {
+		expect(groupThousands('1234567.891')).toBe('1,234,567.891');
+		expect(groupThousands('1234.0001234')).toBe('1,234.0001234');
+	});
+
+	it('keeps the sign outside the first group', () => {
+		expect(groupThousands('-1234567')).toBe('-1,234,567');
+		expect(groupThousands('-1234567.891')).toBe('-1,234,567.891');
+	});
+
+	it('preserves suffix and prefix unit decoration', () => {
+		expect(groupThousands('1234567 ms')).toBe('1,234,567 ms');
+		expect(groupThousands('1234567%')).toBe('1,234,567%');
+		expect(groupThousands('$ 1234567')).toBe('$ 1,234,567');
+	});
+
+	it('leaves formatter-scaled values untouched', () => {
+		expect(groupThousands('1.18 MiB')).toBe('1.18 MiB');
+		expect(groupThousands('1.23 Mil')).toBe('1.23 Mil');
+		expect(groupThousands('20.58 mins')).toBe('20.58 mins');
+	});
+
+	it('leaves exponent notation untouched', () => {
+		expect(groupThousands('1.234567e+21')).toBe('1.234567e+21');
+		expect(groupThousands('1234567e-8')).toBe('1234567e-8');
+	});
+
+	it('returns non-numeric output unchanged', () => {
+		expect(groupThousands('∞')).toBe('∞');
+		expect(groupThousands('-∞')).toBe('-∞');
+		expect(groupThousands('NaN')).toBe('NaN');
+		expect(groupThousands('')).toBe('');
+	});
+
+	it('is idempotent on already-grouped input', () => {
+		expect(groupThousands(groupThousands('1234567'))).toBe('1,234,567');
+	});
+});

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/__tests__/parseFormattedValue.test.ts

@@ -0,0 +1,60 @@
+import { parseFormattedValue } from '../parseFormattedValue';
+
+describe('parseFormattedValue', () => {
+	it('splits a trailing unit label off the numeric core', () => {
+		expect(parseFormattedValue('295.43 ms')).toStrictEqual({
+			numericValue: '295.43',
+			prefixUnit: '',
+			suffixUnit: 'ms',
+		});
+	});
+
+	it('splits a leading currency symbol off the numeric core', () => {
+		expect(parseFormattedValue('$ 1.2K')).toStrictEqual({
+			numericValue: '1.2K',
+			prefixUnit: '$',
+			suffixUnit: '',
+		});
+	});
+
+	// Regression: the numeric core used to reject `,`, so a grouped value fell
+	// through to the whole-string fallback and lost its unit split.
+	it('keeps the unit split for grouped values', () => {
+		expect(parseFormattedValue('1,234,567 ms')).toStrictEqual({
+			numericValue: '1,234,567',
+			prefixUnit: '',
+			suffixUnit: 'ms',
+		});
+		expect(parseFormattedValue('1,234,567%')).toStrictEqual({
+			numericValue: '1,234,567',
+			prefixUnit: '',
+			suffixUnit: '%',
+		});
+		expect(parseFormattedValue('$ 1,234,567.89')).toStrictEqual({
+			numericValue: '1,234,567.89',
+			prefixUnit: '$',
+			suffixUnit: '',
+		});
+	});
+
+	it('treats a unitless value as the numeric core', () => {
+		expect(parseFormattedValue('1,234,567')).toStrictEqual({
+			numericValue: '1,234,567',
+			prefixUnit: '',
+			suffixUnit: '',
+		});
+	});
+
+	it('falls back to the whole string when nothing matches', () => {
+		expect(parseFormattedValue('∞')).toStrictEqual({
+			numericValue: '∞',
+			prefixUnit: '',
+			suffixUnit: '',
+		});
+		expect(parseFormattedValue('NaN')).toStrictEqual({
+			numericValue: 'NaN',
+			prefixUnit: '',
+			suffixUnit: '',
+		});
+	});
+});

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/formatPanelValue.ts

@@ -1,16 +1,21 @@
 import type { PrecisionOption } from 'components/Graph/types';
 import { getYAxisFormattedValue } from 'components/Graph/yAxisConfig';
 
+import { groupThousands } from './groupThousands';
+
 /**
- * Formats a scalar for display in a V2 panel, honoring decimal precision. The
- * single seam through which panels touch `getYAxisFormattedValue`. Unitless
- * values format through the `'none'` unit, which still respects precision — so
- * precision isn't silently dropped when no unit is set.
+ * Formats a scalar for display in a V2 panel, honoring decimal precision and
+ * grouping the integer part into thousands. The single seam through which panels
+ * touch `getYAxisFormattedValue`. Unitless values format through the `'none'`
+ * unit, which still respects precision — so precision isn't silently dropped
+ * when no unit is set.
  */
 export function formatPanelValue(
 	value: number,
 	unit?: string,
 	precision?: PrecisionOption,
 ): string {
-	return getYAxisFormattedValue(String(value), unit || 'none', precision);
+	return groupThousands(
+		getYAxisFormattedValue(String(value), unit || 'none', precision),
+	);
 }

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/groupThousands.ts

@@ -0,0 +1,22 @@
+const THOUSANDS_BOUNDARY = /(\d)(?=(?:\d{3})+$)/g;
+
+/** Leading number of a formatted value; unit decoration falls outside the match. */
+const NUMERIC_TOKEN = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/;
+
+/**
+ * Inserts thousand separators into the integer part of an already-formatted value,
+ * so large scalars read as `1,234,567`. Fractions, unit labels and formatter-scaled
+ * values (`1.18 MiB`) are left as-is, as is exponent notation.
+ */
+export function groupThousands(formatted: string): string {
+	return formatted.replace(NUMERIC_TOKEN, (token) => {
+		if (token.includes('e') || token.includes('E')) {
+			return token;
+		}
+
+		const [integerPart, fraction] = token.split('.');
+		const grouped = integerPart.replace(THOUSANDS_BOUNDARY, '$1,');
+
+		return fraction === undefined ? grouped : `${grouped}.${fraction}`;
+	});
+}

frontend/src/pages/DashboardPageV2/DashboardContainer/Panels/utils/parseFormattedValue.ts

@@ -1,5 +1,5 @@
 export interface ParsedFormattedValue {
-	/** The numeric portion (e.g. "295.43", "1.2K"). */
+	/** The numeric portion (e.g. "295.43", "1,234,567", "1.2K"). */
 	numericValue: string;
 	/** A leading unit symbol such as a currency prefix, if any. */
 	prefixUnit: string;
@@ -8,13 +8,14 @@ export interface ParsedFormattedValue {
 }
 
 /**
- * Splits a formatted value (e.g. "$ 1.2K", "295.43 ms") into its numeric core
- * and prefix/suffix unit for independent styling. Non-matching input falls back
- * to the whole string as the numeric value.
+ * Splits a formatted value (e.g. "$ 1.2K", "295.43 ms", "1,234,567") into its
+ * numeric core and prefix/suffix unit for independent styling. The core accepts
+ * thousand separators, so a grouped value keeps its unit split. Non-matching
+ * input falls back to the whole string as the numeric value.
  */
 export function parseFormattedValue(value: string): ParsedFormattedValue {
 	const matches = value.match(
-		/^([^\d.]*)?([\d.]+(?:[eE][+-]?[\d]+)?[KMB]?)([^\d.]*)?$/,
+		/^([^\d.]*)?([\d.,]+(?:[eE][+-]?[\d]+)?[KMB]?)([^\d.]*)?$/,
 	);
 
 	return {

frontend/src/pages/TraceDetailsV3/TraceWaterfall/TraceWaterfallStates/Success/Success.tsx

@@ -690,7 +690,7 @@ function Success(props: ISuccessProps): JSX.Element {
 				urlQuery.set('spanId', span?.span_id);
 			}
 
-			safeNavigate({ search: urlQuery.toString() });
+			safeNavigate({ search: urlQuery.toString() }, { replace: true });
 		},
 		[setSelectedSpan, urlQuery, safeNavigate],
 	);

frontend/src/pages/TraceDetailsV3/TraceWaterfall/TraceWaterfallStates/Success/__tests__/UnifiedSpanClick.test.tsx

@@ -260,11 +260,13 @@ describe('Span Click User Flows', () => {
 		) as HTMLElement;
 		await user.click(spanElement);
 
-		// Verify URL was updated with spanId
 		expect(mockUrlQuery.get('spanId')).toBe('span-1');
-		expect(mockSafeNavigate).toHaveBeenCalledWith({
-			search: expect.stringContaining('spanId=span-1'),
-		});
+		expect(mockSafeNavigate).toHaveBeenCalledWith(
+			{
+				search: expect.stringContaining('spanId=span-1'),
+			},
+			{ replace: true },
+		);
 	});
 
 	it('clicking span duration visually selects the span', async () => {
@@ -430,10 +432,13 @@ describe('Span Click User Flows', () => {
 		expect(mockUrlQuery.get('anotherParam')).toBe('anotherValue');
 		expect(mockUrlQuery.get('spanId')).toBe('span-1');
 
-		expect(mockSafeNavigate).toHaveBeenCalledWith({
-			search: expect.stringMatching(
-				/existingParam=existingValue.*anotherParam=anotherValue.*spanId=span-1/,
-			),
-		});
+		expect(mockSafeNavigate).toHaveBeenCalledWith(
+			{
+				search: expect.stringMatching(
+					/existingParam=existingValue.*anotherParam=anotherValue.*spanId=span-1/,
+				),
+			},
+			{ replace: true },
+		);
 	});
 });

frontend/src/pages/TraceDetailsV3/index.tsx

@@ -80,7 +80,7 @@ function TraceDetailsV3(): JSX.Element {
 
 	const handleSpanDetailsClose = useCallback((): void => {
 		urlQuery.delete('spanId');
-		safeNavigate({ search: urlQuery.toString() });
+		safeNavigate({ search: urlQuery.toString() }, { replace: true });
 	}, [urlQuery, safeNavigate]);
 
 	const handleFilteredSpansChange = useCallback(

pkg/alertmanager/alertmanagernotify/email/email_test.go

@@ -223,9 +223,7 @@ func TestEmailNotifyWithErrors(t *testing.T) {
 	}
 
 	c, err := loadEmailTestConfiguration(cfgFile)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	for _, tc := range []struct {
 		title     string
@@ -288,11 +286,6 @@ func TestEmailNotifyWithErrors(t *testing.T) {
 		},
 	} {
 		t.Run(tc.title, func(t *testing.T) {
-			if len(tc.errMsg) == 0 {
-				t.Fatal("please define the expected error message")
-				return
-			}
-
 			emailCfg := &config.EmailConfig{
 				Smarthost: c.Smarthost,
 				To:        emailTo,
@@ -309,15 +302,15 @@ func TestEmailNotifyWithErrors(t *testing.T) {
 
 			_, retry, err := notifyEmail(t, emailCfg, c.Server)
 			require.Error(t, err)
-			require.Contains(t, err.Error(), tc.errMsg)
-			require.False(t, retry)
+			assert.Contains(t, err.Error(), tc.errMsg)
+			assert.False(t, retry)
 
 			e, err := c.Server.getLastEmail(t)
 			require.NoError(t, err)
 			if tc.hasEmail {
-				require.NotNil(t, e)
+				assert.NotNil(t, e)
 			} else {
-				require.Nil(t, e)
+				assert.Nil(t, e)
 			}
 		})
 	}
@@ -331,9 +324,7 @@ func TestEmailNotifyWithDoneContext(t *testing.T) {
 	}
 
 	c, err := loadEmailTestConfiguration(cfgFile)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
@@ -350,7 +341,7 @@ func TestEmailNotifyWithDoneContext(t *testing.T) {
 		c.Server,
 	)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "establish connection to server")
+	assert.Contains(t, err.Error(), "establish connection to server")
 }
 
 // TestEmailNotifyWithoutAuthentication sends an email to an instance of
@@ -363,9 +354,7 @@ func TestEmailNotifyWithoutAuthentication(t *testing.T) {
 	}
 
 	c, err := loadEmailTestConfiguration(cfgFile)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	mail, _, err := notifyEmail(
 		t,
@@ -390,7 +379,7 @@ func TestEmailNotifyWithoutAuthentication(t *testing.T) {
 		}
 		headers = append(headers, k)
 	}
-	require.True(t, foundMsgID, "Couldn't find 'message-id' in %v", headers)
+	assert.True(t, foundMsgID, "Couldn't find 'message-id' in %v", headers)
 }
 
 // TestEmailNotifyWithSTARTTLS connects to the server, upgrades the connection
@@ -406,9 +395,7 @@ func TestEmailNotifyWithSTARTTLS(t *testing.T) {
 	}
 
 	c, err := loadEmailTestConfiguration(cfgFile)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	trueVar := true
 	_, _, err = notifyEmail(
@@ -437,9 +424,7 @@ func TestEmailNotifyWithAuthentication(t *testing.T) {
 	}
 
 	c, err := loadEmailTestConfiguration(cfgFile)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	td := t.TempDir()
 	fileWithCorrectPassword, err := os.CreateTemp(td, "smtp-password-correct")
@@ -583,13 +568,13 @@ func TestEmailNotifyWithAuthentication(t *testing.T) {
 			e, retry, err := notifyEmail(t, emailCfg, c.Server)
 			if len(tc.errMsg) > 0 {
 				require.Error(t, err)
-				require.Contains(t, err.Error(), tc.errMsg)
-				require.Equal(t, tc.retry, retry)
+				assert.Contains(t, err.Error(), tc.errMsg)
+				assert.Equal(t, tc.retry, retry)
 				return
 			}
 			require.NoError(t, err)
 
-			require.Equal(t, "1 firing alert(s)", e.Subject)
+			assert.Equal(t, "1 firing alert(s)", e.Subject)
 
 			getAddresses := func(addresses []map[string]string) []string {
 				res := make([]string, 0, len(addresses))
@@ -600,19 +585,21 @@ func TestEmailNotifyWithAuthentication(t *testing.T) {
 			}
 			to := getAddresses(e.To)
 			from := getAddresses(e.From)
-			require.Equal(t, strings.Split(emailCfg.To, ","), to)
-			require.Equal(t, strings.Split(emailCfg.From, ","), from)
+			assert.Equal(t, strings.Split(emailCfg.To, ","), to)
+			assert.Equal(t, strings.Split(emailCfg.From, ","), from)
 
 			if len(emailCfg.HTML) > 0 {
-				require.Equal(t, emailCfg.HTML, *e.HTML)
+				require.NotNil(t, e.HTML)
+				assert.Equal(t, emailCfg.HTML, *e.HTML)
 			} else {
-				require.Nil(t, e.HTML)
+				assert.Nil(t, e.HTML)
 			}
 
 			if len(emailCfg.Text) > 0 {
-				require.Equal(t, emailCfg.Text, *e.Text)
+				require.NotNil(t, e.Text)
+				assert.Equal(t, emailCfg.Text, *e.Text)
 			} else {
-				require.Nil(t, e.Text)
+				assert.Nil(t, e.Text)
 			}
 		})
 	}
@@ -624,7 +611,7 @@ func TestEmailConfigNoAuthMechs(t *testing.T) {
 	}
 	_, err := email.auth("")
 	require.Error(t, err)
-	require.Equal(t, "unknown auth mechanism: ", err.Error())
+	assert.Equal(t, "unknown auth mechanism: ", err.Error())
 }
 
 func TestEmailConfigMissingAuthParam(t *testing.T) {
@@ -634,19 +621,19 @@ func TestEmailConfigMissingAuthParam(t *testing.T) {
 	}
 	_, err := email.auth("CRAM-MD5")
 	require.Error(t, err)
-	require.Equal(t, "missing secret for CRAM-MD5 auth mechanism", err.Error())
+	assert.Equal(t, "missing secret for CRAM-MD5 auth mechanism", err.Error())
 
 	_, err = email.auth("PLAIN")
 	require.Error(t, err)
-	require.Equal(t, "missing password for PLAIN auth mechanism", err.Error())
+	assert.Equal(t, "missing password for PLAIN auth mechanism", err.Error())
 
 	_, err = email.auth("LOGIN")
 	require.Error(t, err)
-	require.Equal(t, "missing password for LOGIN auth mechanism", err.Error())
+	assert.Equal(t, "missing password for LOGIN auth mechanism", err.Error())
 
 	_, err = email.auth("PLAIN LOGIN")
 	require.Error(t, err)
-	require.Equal(t, "missing password for PLAIN auth mechanism\nmissing password for LOGIN auth mechanism", err.Error())
+	assert.Equal(t, "missing password for PLAIN auth mechanism\nmissing password for LOGIN auth mechanism", err.Error())
 }
 
 func TestEmailNoUsernameCustomError(t *testing.T) {
@@ -655,7 +642,7 @@ func TestEmailNoUsernameCustomError(t *testing.T) {
 	}
 	a, err := email.auth("CRAM-MD5")
 	require.ErrorIs(t, err, errNoAuthUsernameConfigured)
-	require.Nil(t, a)
+	assert.Nil(t, a)
 }
 
 // TestEmailRejected simulates the failure of an otherwise valid message submission which fails at a later point than
@@ -720,7 +707,7 @@ func TestEmailRejected(t *testing.T) {
 	// Send the alert to mock SMTP server.
 	retry, err := e.Notify(context.Background(), firingAlert)
 	require.ErrorContains(t, err, "501 5.5.4 Rejected!")
-	require.True(t, retry)
+	assert.True(t, retry)
 	require.NoError(t, srv.Shutdown(ctx))
 
 	require.Eventuallyf(t, func() bool {
@@ -789,9 +776,7 @@ func TestEmailNotifyWithThreading(t *testing.T) {
 	}
 
 	c, err := loadEmailTestConfiguration(cfgFile)
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	for _, tc := range []struct {
 		name         string
@@ -836,22 +821,22 @@ func TestEmailNotifyWithThreading(t *testing.T) {
 			referencesValue := mail.Headers["references"]
 			inReplyToValue := mail.Headers["in-reply-to"]
 
-			require.NotEmpty(t, referencesValue, "References header not found in %v", mail.Headers)
-			require.NotEmpty(t, inReplyToValue, "In-Reply-To header not found in %v", mail.Headers)
+			assert.NotEmpty(t, referencesValue, "References header not found in %v", mail.Headers)
+			assert.NotEmpty(t, inReplyToValue, "In-Reply-To header not found in %v", mail.Headers)
 
-			require.Equal(t, referencesValue, inReplyToValue, "References and In-Reply-To should match")
+			assert.Equal(t, referencesValue, inReplyToValue, "References and In-Reply-To should match")
 
 			// Verify the format: <alert-HASH-DATE@alertmanager>
-			require.Contains(t, referencesValue, "<alert-")
-			require.Contains(t, referencesValue, "@alertmanager>")
+			assert.Contains(t, referencesValue, "<alert-")
+			assert.Contains(t, referencesValue, "@alertmanager>")
 
 			if tc.wantDatePart {
 				today := time.Now().Format("2006-01-02")
-				require.Contains(t, referencesValue, today, "threading header should contain today's date")
+				assert.Contains(t, referencesValue, today, "threading header should contain today's date")
 			} else {
 				// With thread_by_date: none, there should be no date
 				// (empty string between hash and @).
-				require.Contains(t, referencesValue, "-@alertmanager>", "threading header should have empty date part")
+				assert.Contains(t, referencesValue, "-@alertmanager>", "threading header should have empty date part")
 			}
 		})
 	}
@@ -904,14 +889,14 @@ func TestEmailGetPassword(t *testing.T) {
 				require.Error(t, err)
 				if errors.Asc(err, errors.CodeInternal) {
 					_, _, errMsg, _, _, _ := errors.Unwrapb(err)
-					require.Contains(t, errMsg, tc.errMsg)
+					assert.Contains(t, errMsg, tc.errMsg)
 				} else {
-					require.Contains(t, err.Error(), tc.errMsg)
+					assert.Contains(t, err.Error(), tc.errMsg)
 				}
-				require.Empty(t, password)
+				assert.Empty(t, password)
 			} else {
 				require.NoError(t, err)
-				require.Equal(t, "secret", password)
+				assert.Equal(t, "secret", password)
 			}
 		})
 	}
@@ -962,11 +947,11 @@ func TestEmailGetSecret(t *testing.T) {
 			secret, err := email.getAuthSecret()
 			if len(tc.errMsg) > 0 {
 				require.Error(t, err)
-				require.Contains(t, err.Error(), tc.errMsg)
-				require.Empty(t, secret)
+				assert.Contains(t, err.Error(), tc.errMsg)
+				assert.Empty(t, secret)
 			} else {
 				require.NoError(t, err)
-				require.Equal(t, "secret", secret)
+				assert.Equal(t, "secret", secret)
 			}
 		})
 	}
@@ -1032,7 +1017,7 @@ func TestEmailImplicitTLS(t *testing.T) {
 				useImplicitTLS = cfg.Smarthost.Port == "465"
 			}
 
-			require.Equal(t, tt.expectImplicit, useImplicitTLS,
+			assert.Equal(t, tt.expectImplicit, useImplicitTLS,
 				"Expected useImplicitTLS=%v for port=%s with forceImplicitTLS=%v",
 				tt.expectImplicit, tt.port, tt.forceImplicitTLS)
 		})
@@ -1074,8 +1059,8 @@ func TestPrepareContent(t *testing.T) {
 		ctx := context.Background()
 		subject, htmlBody, err := n.prepareContent(ctx, alerts)
 		require.NoError(t, err)
-		require.Equal(t, "subj", subject)
-		require.Equal(t, "<div><p>line one</p>\n</div><div><p>line two</p>\n</div>", htmlBody)
+		assert.Equal(t, "subj", subject)
+		assert.Equal(t, "<div><p>line one</p>\n</div><div><p>line two</p>\n</div>", htmlBody)
 	})
 
 	t.Run("custom title template; default body HTML template", func(t *testing.T) {
@@ -1103,8 +1088,8 @@ func TestPrepareContent(t *testing.T) {
 		ctx := context.Background()
 		subject, htmlBody, err := n.prepareContent(ctx, alerts)
 		require.NoError(t, err)
-		require.Equal(t, "Status: firing", htmlBody)
-		require.Equal(t, "fixed from firing", subject)
+		assert.Equal(t, "Status: firing", htmlBody)
+		assert.Equal(t, "fixed from firing", subject)
 	})
 
 	t.Run("default template without HTML", func(t *testing.T) {
@@ -1125,8 +1110,8 @@ func TestPrepareContent(t *testing.T) {
 		ctx := context.Background()
 		subject, htmlBody, err := n.prepareContent(ctx, alerts)
 		require.NoError(t, err)
-		require.Equal(t, "", htmlBody)
-		require.Equal(t, "the email subject", subject)
+		assert.Equal(t, "", htmlBody)
+		assert.Equal(t, "the email subject", subject)
 	})
 
 	t.Run("custom title template; custom body template", func(t *testing.T) {
@@ -1160,11 +1145,11 @@ func TestPrepareContent(t *testing.T) {
 		ctx := context.Background()
 		subject, htmlBody, err := n.prepareContent(ctx, alerts)
 		require.NoError(t, err)
-		require.Contains(t, htmlBody, "<!DOCTYPE html>")
-		require.Contains(t, htmlBody, "<p>line two</p>")
-		require.NotContains(t, htmlBody, "Well, what are you?")
-		require.Equal(t, subject, "fixed from firing")
-		require.NotContains(t, subject, "subject")
+		assert.Contains(t, htmlBody, "<!DOCTYPE html>")
+		assert.Contains(t, htmlBody, "<p>line two</p>")
+		assert.NotContains(t, htmlBody, "Well, what are you?")
+		assert.Equal(t, "fixed from firing", subject)
+		assert.NotContains(t, subject, "subject")
 	})
 }
 

pkg/alertmanager/alertmanagernotify/msteamsv2/msteamsv2_test.go

@@ -22,6 +22,7 @@ import (
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	test "github.com/SigNoz/signoz/pkg/alertmanager/alertmanagernotify/alertmanagernotifytest"
@@ -54,7 +55,7 @@ func TestMSTeamsV2Retry(t *testing.T) {
 
 	for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
 		actual, _ := notifier.retrier.Check(statusCode, nil)
-		require.Equal(t, expected, actual, "retry - error on status %d", statusCode)
+		assert.Equal(t, expected, actual, "retry - error on status %d", statusCode)
 	}
 }
 
@@ -110,7 +111,7 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
 			} else {
 				var reasonError *notify.ErrorWithReason
 				require.ErrorAs(t, err, &reasonError)
-				require.Equal(t, tt.expectedReason, reasonError.Reason)
+				assert.Equal(t, tt.expectedReason, reasonError.Reason)
 			}
 		})
 	}
@@ -133,7 +134,6 @@ func TestMSTeamsV2Templating(t *testing.T) {
 		cfg       *config.MSTeamsV2Config
 		titleLink string
 
-		retry  bool
 		errMsg string
 	}{
 		{
@@ -143,7 +143,6 @@ func TestMSTeamsV2Templating(t *testing.T) {
 				Text:  `{{ template "msteams.default.text" . }}`,
 			},
 			titleLink: `{{ template "msteamsv2.default.titleLink" . }}`,
-			retry:     false,
 		},
 		{
 			title: "title with templating errors",
@@ -185,12 +184,12 @@ func TestMSTeamsV2Templating(t *testing.T) {
 				},
 			}...)
 			if tc.errMsg == "" {
-				require.NoError(t, err)
+				assert.NoError(t, err)
 			} else {
 				require.Error(t, err)
-				require.Contains(t, err.Error(), tc.errMsg)
+				assert.Contains(t, err.Error(), tc.errMsg)
 			}
-			require.Equal(t, tc.retry, ok)
+			assert.False(t, ok)
 		})
 	}
 }
@@ -250,14 +249,14 @@ func TestPrepareContent(t *testing.T) {
 		}
 		blocks, err := notifier.prepareContent(ctx, alerts)
 		require.NoError(t, err)
-		require.NotEmpty(t, blocks)
+		require.Len(t, blocks, 2)
 		// First block should be the title with color (firing = red)
-		require.Equal(t, "Bolder", blocks[0].Weight)
-		require.Equal(t, colorRed, blocks[0].Color)
+		assert.Equal(t, "Bolder", blocks[0].Weight)
+		assert.Equal(t, colorRed, blocks[0].Color)
 		// verify title text
-		require.Equal(t, "Alertname: test", blocks[0].Text)
+		assert.Equal(t, "Alertname: test", blocks[0].Text)
 		// verify body text
-		require.Equal(t, "Firing alert: test", blocks[1].Text)
+		assert.Equal(t, "Firing alert: test", blocks[1].Text)
 	})
 
 	t.Run("custom template - per-alert color", func(t *testing.T) {
@@ -305,16 +304,15 @@ func TestPrepareContent(t *testing.T) {
 		}
 		blocks, err := notifier.prepareContent(ctx, alerts)
 		require.NoError(t, err)
-		require.NotEmpty(t, blocks)
 		// total 3 blocks: title and 2 body blocks
-		require.True(t, len(blocks) == 3)
+		require.Len(t, blocks, 3)
 		// First block: title color is overall color of the alerts
-		require.Equal(t, colorRed, blocks[0].Color)
+		assert.Equal(t, colorRed, blocks[0].Color)
 		// verify title text
-		require.Equal(t, "Custom Title", blocks[0].Text)
+		assert.Equal(t, "Custom Title", blocks[0].Text)
 		// Body blocks should have per-alert color
-		require.Equal(t, colorRed, blocks[1].Color)   // firing
-		require.Equal(t, colorGreen, blocks[2].Color) // resolved
+		assert.Equal(t, colorRed, blocks[1].Color)   // firing
+		assert.Equal(t, colorGreen, blocks[2].Color) // resolved
 	})
 }
 

pkg/alertmanager/alertmanagernotify/opsgenie/opsgenie_test.go

@@ -21,6 +21,7 @@ import (
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/alertmanager/config"
@@ -49,7 +50,7 @@ func TestOpsGenieRetry(t *testing.T) {
 	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
 	for statusCode, expected := range test.RetryTests(retryCodes) {
 		actual, _ := notifier.retrier.Check(statusCode, nil)
-		require.Equal(t, expected, actual, "error on status %d", statusCode)
+		assert.Equal(t, expected, actual, "error on status %d", statusCode)
 	}
 }
 
@@ -103,9 +104,7 @@ func TestGettingOpsGegineApikeyFromFile(t *testing.T) {
 
 func TestOpsGenie(t *testing.T) {
 	u, err := url.Parse("https://opsgenie/api")
-	if err != nil {
-		t.Fatalf("failed to parse URL: %v", err)
-	}
+	require.NoError(t, err)
 	logger := promslog.NewNopLogger()
 	tmpl := test.CreateTmpl(t)
 
@@ -236,10 +235,10 @@ func TestOpsGenie(t *testing.T) {
 			req, retry, err := notifier.createRequests(ctx, alert1)
 			require.NoError(t, err)
 			require.Len(t, req, 1)
-			require.True(t, retry)
-			require.Equal(t, expectedURL, req[0].URL)
-			require.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
-			require.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
+			assert.True(t, retry)
+			assert.Equal(t, expectedURL, req[0].URL)
+			assert.Equal(t, "GenieKey http://am", req[0].Header.Get("Authorization"))
+			assert.Equal(t, tc.expectedEmptyAlertBody, readBody(t, req[0]))
 
 			// Fully defined alert.
 			alert2 := &types.Alert{
@@ -266,15 +265,15 @@ func TestOpsGenie(t *testing.T) {
 			}
 			req, retry, err = notifier.createRequests(ctx, alert2)
 			require.NoError(t, err)
-			require.True(t, retry)
+			assert.True(t, retry)
 			require.Len(t, req, 1)
-			require.Equal(t, tc.expectedBody, readBody(t, req[0]))
+			assert.Equal(t, tc.expectedBody, readBody(t, req[0]))
 
 			// Broken API Key Template.
 			tc.cfg.APIKey = "{{ kaput "
 			_, _, err = notifier.createRequests(ctx, alert2)
 			require.Error(t, err)
-			require.Equal(t, "template: :1: function \"kaput\" not defined", err.Error())
+			assert.Equal(t, "template: :1: function \"kaput\" not defined", err.Error())
 		})
 	}
 }
@@ -307,7 +306,7 @@ func TestOpsGenieWithUpdate(t *testing.T) {
 	require.NoError(t, err)
 	requests, retry, err := notifierWithUpdate.createRequests(ctx, alert)
 	require.NoError(t, err)
-	require.True(t, retry)
+	assert.True(t, retry)
 	require.Len(t, requests, 3)
 
 	body0 := readBody(t, requests[0])
@@ -316,13 +315,13 @@ func TestOpsGenieWithUpdate(t *testing.T) {
 	key, _ := notify.ExtractGroupKey(ctx)
 	alias := key.Hash()
 
-	require.Equal(t, "https://test-opsgenie-url/v2/alerts", requests[0].URL.String())
-	require.NotEmpty(t, body0)
+	assert.Equal(t, "https://test-opsgenie-url/v2/alerts", requests[0].URL.String())
+	assert.NotEmpty(t, body0)
 
-	require.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
-	require.JSONEq(t, `{"message":"new message"}`, body1)
-	require.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
-	require.JSONEq(t, `{"description":"new description"}`, body2)
+	assert.Equal(t, requests[1].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/message?identifierType=alias", alias))
+	assert.JSONEq(t, `{"message":"new message"}`, body1)
+	assert.Equal(t, requests[2].URL.String(), fmt.Sprintf("https://test-opsgenie-url/v2/alerts/%s/description?identifierType=alias", alias))
+	assert.JSONEq(t, `{"description":"new description"}`, body2)
 }
 
 func TestOpsGenieApiKeyFile(t *testing.T) {
@@ -341,7 +340,8 @@ func TestOpsGenieApiKeyFile(t *testing.T) {
 	require.NoError(t, err)
 	requests, _, err := notifierWithUpdate.createRequests(ctx)
 	require.NoError(t, err)
-	require.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
+	require.Len(t, requests, 1)
+	assert.Equal(t, "GenieKey my_secret_api_key", requests[0].Header.Get("Authorization"))
 }
 
 func TestPrepareContent(t *testing.T) {
@@ -377,8 +377,8 @@ func TestPrepareContent(t *testing.T) {
 
 		title, desc, prepErr := notifier.prepareContent(ctx, alerts)
 		require.NoError(t, prepErr)
-		require.Equal(t, "Firing alert: test", title)
-		require.Equal(t, "Check runbook for more details", desc)
+		assert.Equal(t, "Firing alert: test", title)
+		assert.Equal(t, "Check runbook for more details", desc)
 	})
 
 	t.Run("custom template", func(t *testing.T) {
@@ -431,9 +431,9 @@ func TestPrepareContent(t *testing.T) {
 
 		title, desc, err := notifier.prepareContent(ctx, alerts)
 		require.NoError(t, err)
-		require.Equal(t, "High request throughput for payment", title)
+		assert.Equal(t, "High request throughput for payment", title)
 		// Each alert body wrapped in <div>, separated by <hr>
-		require.Equal(t, "<div><p>Alert firing in NS: potter-the-harry</p>\n</div><hr><div><p>Alert firing in NS: smart-the-rat</p>\n</div>", desc)
+		assert.Equal(t, "<div><p>Alert firing in NS: potter-the-harry</p>\n</div><hr><div><p>Alert firing in NS: smart-the-rat</p>\n</div>", desc)
 	})
 }
 

pkg/alertmanager/alertmanagernotify/pagerduty/pagerduty_test.go

@@ -25,6 +25,7 @@ import (
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/alertmanager/config"
@@ -54,7 +55,7 @@ func TestPagerDutyRetryV1(t *testing.T) {
 	retryCodes := append(test.DefaultRetryCodes(), http.StatusForbidden)
 	for statusCode, expected := range test.RetryTests(retryCodes) {
 		actual, _ := notifier.retrier.Check(statusCode, nil)
-		require.Equal(t, expected, actual, "retryv1 - error on status %d", statusCode)
+		assert.Equal(t, expected, actual, "retryv1 - error on status %d", statusCode)
 	}
 }
 
@@ -74,7 +75,7 @@ func TestPagerDutyRetryV2(t *testing.T) {
 	retryCodes := append(test.DefaultRetryCodes(), http.StatusTooManyRequests)
 	for statusCode, expected := range test.RetryTests(retryCodes) {
 		actual, _ := notifier.retrier.Check(statusCode, nil)
-		require.Equal(t, expected, actual, "retryv2 - error on status %d", statusCode)
+		assert.Equal(t, expected, actual, "retryv2 - error on status %d", statusCode)
 	}
 }
 
@@ -349,12 +350,12 @@ func TestPagerDutyTemplating(t *testing.T) {
 				require.Error(t, err)
 				if errors.Asc(err, errors.CodeInternal) {
 					_, _, errMsg, _, _, _ := errors.Unwrapb(err)
-					require.Contains(t, errMsg, tc.errMsg)
+					assert.Contains(t, errMsg, tc.errMsg)
 				} else {
-					require.Contains(t, err.Error(), tc.errMsg)
+					assert.Contains(t, err.Error(), tc.errMsg)
 				}
 			}
-			require.Equal(t, tc.retry, ok)
+			assert.Equal(t, tc.retry, ok)
 		})
 	}
 }
@@ -393,7 +394,7 @@ func TestErrDetails(t *testing.T) {
 	} {
 		t.Run("", func(t *testing.T) {
 			err := errDetails(tc.status, tc.body)
-			require.Contains(t, err, tc.exp)
+			assert.Contains(t, err, tc.exp)
 		})
 	}
 }
@@ -427,7 +428,7 @@ func TestEventSizeEnforcement(t *testing.T) {
 
 	encodedV1, err := notifierV1.encodeMessage(context.Background(), msgV1)
 	require.NoError(t, err)
-	require.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+	assert.Contains(t, encodedV1.String(), `"details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
 
 	// V2 Messages
 	msgV2 := &pagerDutyMessage{
@@ -451,7 +452,7 @@ func TestEventSizeEnforcement(t *testing.T) {
 
 	encodedV2, err := notifierV2.encodeMessage(context.Background(), msgV2)
 	require.NoError(t, err)
-	require.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
+	assert.Contains(t, encodedV2.String(), `"custom_details":{"error":"Custom details have been removed because the original event exceeds the maximum size of 512KB"}`)
 }
 
 func TestPagerDutyEmptySrcHref(t *testing.T) {
@@ -543,8 +544,9 @@ func TestPagerDutyEmptySrcHref(t *testing.T) {
 				}
 			}
 
-			require.Equal(t, expectedImages, event.Images)
-			require.Equal(t, expectedLinks, event.Links)
+			// Handler runs on the server's goroutine — require is illegal here.
+			assert.Equal(t, expectedImages, event.Images)
+			assert.Equal(t, expectedLinks, event.Links)
 		},
 	))
 	defer server.Close()
@@ -644,7 +646,7 @@ func TestPagerDutyTimeout(t *testing.T) {
 				},
 			}
 			_, err = pd.Notify(ctx, alert)
-			require.Equal(t, tt.wantErr, err != nil)
+			assert.Equal(t, tt.wantErr, err != nil)
 		})
 	}
 }
@@ -899,11 +901,12 @@ func TestRenderDetails(t *testing.T) {
 				tmpl: test.CreateTmpl(t),
 			}
 			got, err := n.renderDetails(tt.args.data)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("renderDetails() error = %v, wantErr %v", err, tt.wantErr)
-				return
+			if tt.wantErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
 			}
-			require.Equal(t, tt.want, got)
+			assert.Equal(t, tt.want, got)
 		})
 	}
 }
@@ -944,7 +947,7 @@ func TestPrepareContent(t *testing.T) {
 
 		title, err := notifier.prepareTitle(ctx, alerts)
 		require.NoError(t, err)
-		require.Equal(t, "HighCPU for Payment service (FIRING)", title)
+		assert.Equal(t, "HighCPU for Payment service (FIRING)", title)
 	})
 
 	t.Run("custom template uses $variable annotation for title", func(t *testing.T) {
@@ -980,6 +983,6 @@ func TestPrepareContent(t *testing.T) {
 
 		title, err := notifier.prepareTitle(ctx, alerts)
 		require.NoError(t, err)
-		require.Equal(t, "HighCPU on api-server is in resolved state", title)
+		assert.Equal(t, "HighCPU on api-server is in resolved state", title)
 	})
 }

pkg/alertmanager/alertmanagernotify/slack/slack_test.go

@@ -23,6 +23,7 @@ import (
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/alertmanager/config"
@@ -50,7 +51,7 @@ func TestSlackRetry(t *testing.T) {
 
 	for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
 		actual, _ := notifier.retrier.Check(statusCode, nil)
-		require.Equal(t, expected, actual, "error on status %d", statusCode)
+		assert.Equal(t, expected, actual, "error on status %d", statusCode)
 	}
 }
 
@@ -232,15 +233,15 @@ func TestNotifier_Notify_WithReason(t *testing.T) {
 				},
 			}
 			retry, err := notifier.Notify(ctx, alert1)
-			require.Equal(t, tt.expectedRetry, retry)
+			assert.Equal(t, tt.expectedRetry, retry)
 			if tt.noError {
 				require.NoError(t, err)
 			} else {
 				var reasonError *notify.ErrorWithReason
 				require.ErrorAs(t, err, &reasonError)
-				require.Equal(t, tt.expectedReason, reasonError.Reason)
-				require.Contains(t, err.Error(), tt.expectedErr)
-				require.Contains(t, err.Error(), "channelname")
+				assert.Equal(t, tt.expectedReason, reasonError.Reason)
+				assert.Contains(t, err.Error(), tt.expectedErr)
+				assert.Contains(t, err.Error(), "channelname")
 			}
 		})
 	}
@@ -296,7 +297,7 @@ func TestSlackTimeout(t *testing.T) {
 				},
 			}
 			_, err = notifier.Notify(ctx, alert)
-			require.Equal(t, tt.wantErr, err != nil)
+			assert.Equal(t, tt.wantErr, err != nil)
 		})
 	}
 }
@@ -350,14 +351,14 @@ func TestPrepareContent(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, atts, 1)
 
-		require.Equal(t, "HighCPU (FIRING)", atts[0].Title)
-		require.Equal(t, "Alert: HighCPU - severity critical", atts[0].Text)
+		assert.Equal(t, "HighCPU (FIRING)", atts[0].Title)
+		assert.Equal(t, "Alert: HighCPU - severity critical", atts[0].Text)
 		// Color is templated — firing alert should be "danger"
-		require.Equal(t, "danger", atts[0].Color)
+		assert.Equal(t, "danger", atts[0].Color)
 		// No BlockKit blocks for default template
-		require.Nil(t, atts[0].Blocks)
+		assert.Nil(t, atts[0].Blocks)
 		// Default markdownIn when config has none
-		require.Equal(t, []string{"fallback", "pretext", "text"}, atts[0].MrkdwnIn)
+		assert.Equal(t, []string{"fallback", "pretext", "text"}, atts[0].MrkdwnIn)
 	})
 
 	t.Run("custom template produces 1+N attachments with per-alert color", func(t *testing.T) {
@@ -428,10 +429,10 @@ func TestPrepareContent(t *testing.T) {
 		require.Len(t, atts, 3)
 
 		// First attachment: title-only, no color, no blocks
-		require.Equal(t, "[firing] HighCPU — api-server", atts[0].Title)
-		require.Empty(t, atts[0].Color)
-		require.Nil(t, atts[0].Blocks)
-		require.Equal(t, "https://alertmanager.signoz.com", atts[0].TitleLink)
+		assert.Equal(t, "[firing] HighCPU — api-server", atts[0].Title)
+		assert.Empty(t, atts[0].Color)
+		assert.Nil(t, atts[0].Blocks)
+		assert.Equal(t, "https://alertmanager.signoz.com", atts[0].TitleLink)
 
 		expectedFiringBody := "*HighCPU*\n\n" +
 			"*Service:* _api-server_\n*Instance:* _i-0abc123_\n*Region:* _us-east-1_\n*Method:* _GET_\n\n" +
@@ -446,16 +447,16 @@ func TestPrepareContent(t *testing.T) {
 			"*Status:* resolved | *Severity:* critical\n\n"
 
 		// Second attachment: firing alert body rendered as slack mrkdwn text, red color
-		require.Nil(t, atts[1].Blocks)
-		require.Equal(t, "#FF0000", atts[1].Color)
-		require.Equal(t, []string{"text"}, atts[1].MrkdwnIn)
-		require.Equal(t, expectedFiringBody, atts[1].Text)
+		assert.Nil(t, atts[1].Blocks)
+		assert.Equal(t, "#FF0000", atts[1].Color)
+		assert.Equal(t, []string{"text"}, atts[1].MrkdwnIn)
+		assert.Equal(t, expectedFiringBody, atts[1].Text)
 
 		// Third attachment: resolved alert body rendered as slack mrkdwn text, green color
-		require.Nil(t, atts[2].Blocks)
-		require.Equal(t, "#00FF00", atts[2].Color)
-		require.Equal(t, []string{"text"}, atts[2].MrkdwnIn)
-		require.Equal(t, expectedResolvedBody, atts[2].Text)
+		assert.Nil(t, atts[2].Blocks)
+		assert.Equal(t, "#00FF00", atts[2].Color)
+		assert.Equal(t, []string{"text"}, atts[2].MrkdwnIn)
+		assert.Equal(t, expectedResolvedBody, atts[2].Text)
 	})
 
 	t.Run("default template with fields and actions", func(t *testing.T) {
@@ -498,49 +499,45 @@ func TestPrepareContent(t *testing.T) {
 
 		// prepareContent does not populate fields/actions — that's done by
 		// addFieldsAndActions which is called from Notify.
-		require.Nil(t, atts[0].Fields)
-		require.Nil(t, atts[0].Actions)
+		assert.Nil(t, atts[0].Fields)
+		assert.Nil(t, atts[0].Actions)
 
 		// Simulate what Notify does after prepareContent
 		notifier.addFieldsAndActions(&atts[0], tmplText)
 
 		// Verify fields
 		require.Len(t, atts[0].Fields, 2)
-		require.Equal(t, "Severity", atts[0].Fields[0].Title)
-		require.Equal(t, "critical", atts[0].Fields[0].Value)
-		require.True(t, *atts[0].Fields[0].Short)
-		require.Equal(t, "Service", atts[0].Fields[1].Title)
-		require.Equal(t, "api-server", atts[0].Fields[1].Value)
+		assert.Equal(t, "Severity", atts[0].Fields[0].Title)
+		assert.Equal(t, "critical", atts[0].Fields[0].Value)
+		require.NotNil(t, atts[0].Fields[0].Short)
+		assert.True(t, *atts[0].Fields[0].Short)
+		assert.Equal(t, "Service", atts[0].Fields[1].Title)
+		assert.Equal(t, "api-server", atts[0].Fields[1].Value)
 
 		// Verify actions
 		require.Len(t, atts[0].Actions, 1)
-		require.Equal(t, "button", atts[0].Actions[0].Type)
-		require.Equal(t, "View Alert", atts[0].Actions[0].Text)
-		require.Equal(t, "https://alertmanager.signoz.com", atts[0].Actions[0].URL)
+		assert.Equal(t, "button", atts[0].Actions[0].Type)
+		assert.Equal(t, "View Alert", atts[0].Actions[0].Text)
+		assert.Equal(t, "https://alertmanager.signoz.com", atts[0].Actions[0].URL)
 	})
 }
 
 func TestSlackMessageField(t *testing.T) {
-	// 1. Setup a fake Slack server
+	// 1. Setup a fake Slack server. The handler runs on the server's
+	// goroutine, so only assert (never require) is safe here.
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		var body map[string]any
-		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
-			t.Fatal(err)
-		}
+		assert.NoError(t, json.NewDecoder(r.Body).Decode(&body))
 
 		// 2. VERIFY: Top-level text exists
-		if body["text"] != "My Top Level Message" {
-			t.Errorf("Expected top-level 'text' to be 'My Top Level Message', got %v", body["text"])
-		}
+		assert.Equal(t, "My Top Level Message", body["text"])
 
 		// 3. VERIFY: Old attachments still exist
 		attachments, ok := body["attachments"].([]any)
-		if !ok || len(attachments) == 0 {
-			t.Errorf("Expected attachments to exist")
-		} else {
-			first := attachments[0].(map[string]any)
-			if first["title"] != "Old Attachment Title" {
-				t.Errorf("Expected attachment title 'Old Attachment Title', got %v", first["title"])
+		if assert.True(t, ok, "expected attachments to exist") && assert.NotEmpty(t, attachments) {
+			first, ok := attachments[0].(map[string]any)
+			if assert.True(t, ok, "expected attachment to be an object") {
+				assert.Equal(t, "Old Attachment Title", first["title"])
 			}
 		}
 
@@ -561,21 +558,16 @@ func TestSlackMessageField(t *testing.T) {
 	}
 
 	tmpl, err := template.FromGlobs([]string{})
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 	tmpl.ExternalURL = u
 
 	logger := slog.New(slog.DiscardHandler)
 	notifier, err := New(conf, tmpl, logger, newTestTemplater(tmpl))
-	if err != nil {
-		t.Fatal(err)
-	}
+	require.NoError(t, err)
 
 	ctx := context.Background()
 	ctx = notify.WithGroupKey(ctx, "test-group-key")
 
-	if _, err := notifier.Notify(ctx); err != nil {
-		t.Fatal("Notify failed:", err)
-	}
+	_, err = notifier.Notify(ctx)
+	require.NoError(t, err, "Notify failed")
 }

pkg/alertmanager/alertmanagernotify/webhook/webhook_test.go

@@ -19,6 +19,7 @@ import (
 	commoncfg "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/promslog"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/SigNoz/signoz/pkg/alertmanager/alertmanagertemplate"
@@ -39,14 +40,12 @@ func TestWebhookRetry(t *testing.T) {
 		promslog.NewNopLogger(),
 		alertmanagertemplate.New(tmpl, slog.Default()),
 	)
-	if err != nil {
-		require.NoError(t, err)
-	}
+	require.NoError(t, err)
 
 	t.Run("test retry status code", func(t *testing.T) {
 		for statusCode, expected := range test.RetryTests(test.DefaultRetryCodes()) {
 			actual, _ := notifier.retrier.Check(statusCode, nil)
-			require.Equal(t, expected, actual, "error on status %d", statusCode)
+			assert.Equal(t, expected, actual, "error on status %d", statusCode)
 		}
 	})
 
@@ -73,7 +72,8 @@ func TestWebhookRetry(t *testing.T) {
 		} {
 			t.Run("", func(t *testing.T) {
 				_, err = notifier.retrier.Check(tc.status, tc.body)
-				require.Equal(t, tc.exp, err.Error())
+				require.Error(t, err)
+				assert.Equal(t, tc.exp, err.Error())
 			})
 		}
 	})
@@ -83,16 +83,16 @@ func TestWebhookTruncateAlerts(t *testing.T) {
 	alerts := make([]*types.Alert, 10)
 
 	truncatedAlerts, numTruncated := truncateAlerts(0, alerts)
-	require.Len(t, truncatedAlerts, 10)
-	require.EqualValues(t, 0, numTruncated)
+	assert.Len(t, truncatedAlerts, 10)
+	assert.EqualValues(t, 0, numTruncated)
 
 	truncatedAlerts, numTruncated = truncateAlerts(4, alerts)
-	require.Len(t, truncatedAlerts, 4)
-	require.EqualValues(t, 6, numTruncated)
+	assert.Len(t, truncatedAlerts, 4)
+	assert.EqualValues(t, 6, numTruncated)
 
 	truncatedAlerts, numTruncated = truncateAlerts(100, alerts)
-	require.Len(t, truncatedAlerts, 10)
-	require.EqualValues(t, 0, numTruncated)
+	assert.Len(t, truncatedAlerts, 10)
+	assert.EqualValues(t, 0, numTruncated)
 }
 
 func TestWebhookRedactedURL(t *testing.T) {
@@ -219,10 +219,10 @@ func TestWebhookURLTemplating(t *testing.T) {
 
 			if tc.expectError {
 				require.Error(t, err)
-				require.Contains(t, err.Error(), tc.expectedErrMsg)
+				assert.Contains(t, err.Error(), tc.expectedErrMsg)
 			} else {
 				require.NoError(t, err)
-				require.Equal(t, tc.expectedPath, calledURL)
+				assert.Equal(t, tc.expectedPath, calledURL)
 			}
 		})
 	}

pkg/apiserver/signozapiserver/aiobservability.go

@@ -0,0 +1,51 @@
+package signozapiserver
+
+import (
+	"net/http"
+
+	"github.com/SigNoz/signoz/pkg/http/handler"
+	"github.com/SigNoz/signoz/pkg/types"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/gorilla/mux"
+)
+
+func (provider *provider) addAIObservabilityRoutes(router *mux.Router) error {
+	if err := router.Handle("/api/v1/ai_observability/fields/keys", handler.New(provider.authzMiddleware.ViewAccess(provider.aiObservabilityHandler.GetFieldsKeys), handler.OpenAPIDef{
+		ID:                  "GetAIObservabilityFieldsKeys",
+		Tags:                []string{"ai_observability"},
+		Summary:             "Get AI observability field keys",
+		Description:         "This endpoint returns the field keys the AI observability explorer can filter on, including the computed per-trace aggregates",
+		Request:             nil,
+		RequestQuery:        new(aiobservabilitytypes.PostableFieldKeysParams),
+		RequestContentType:  "",
+		Response:            new(telemetrytypes.GettableFieldKeys),
+		ResponseContentType: "application/json",
+		SuccessStatusCode:   http.StatusOK,
+		ErrorStatusCodes:    []int{},
+		Deprecated:          false,
+		SecuritySchemes:     newSecuritySchemes(types.RoleViewer),
+	})).Methods(http.MethodGet).GetError(); err != nil {
+		return err
+	}
+
+	if err := router.Handle("/api/v1/ai_observability/fields/values", handler.New(provider.authzMiddleware.ViewAccess(provider.aiObservabilityHandler.GetFieldsValues), handler.OpenAPIDef{
+		ID:                  "GetAIObservabilityFieldsValues",
+		Tags:                []string{"ai_observability"},
+		Summary:             "Get AI observability field values",
+		Description:         "This endpoint returns the values the AI observability explorer can filter a field key on",
+		Request:             nil,
+		RequestQuery:        new(aiobservabilitytypes.PostableFieldValueParams),
+		RequestContentType:  "",
+		Response:            new(telemetrytypes.GettableFieldValues),
+		ResponseContentType: "application/json",
+		SuccessStatusCode:   http.StatusOK,
+		ErrorStatusCodes:    []int{},
+		Deprecated:          false,
+		SecuritySchemes:     newSecuritySchemes(types.RoleViewer),
+	})).Methods(http.MethodGet).GetError(); err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/apiserver/signozapiserver/provider.go

@@ -12,6 +12,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/global"
 	"github.com/SigNoz/signoz/pkg/http/handler"
 	"github.com/SigNoz/signoz/pkg/http/middleware"
+	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
 	"github.com/SigNoz/signoz/pkg/modules/authdomain"
 	"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard"
@@ -61,6 +62,7 @@ type provider struct {
 	infraMonitoringHandler     inframonitoring.Handler
 	gatewayHandler             gateway.Handler
 	fieldsHandler              fields.Handler
+	aiObservabilityHandler     aiobservability.Handler
 	authzHandler               authz.Handler
 	rawDataExportHandler       rawdataexport.Handler
 	zeusHandler                zeus.Handler
@@ -97,6 +99,7 @@ func NewFactory(
 	infraMonitoringHandler inframonitoring.Handler,
 	gatewayHandler gateway.Handler,
 	fieldsHandler fields.Handler,
+	aiObservabilityHandler aiobservability.Handler,
 	authzHandler authz.Handler,
 	rawDataExportHandler rawdataexport.Handler,
 	zeusHandler zeus.Handler,
@@ -136,6 +139,7 @@ func NewFactory(
 			infraMonitoringHandler,
 			gatewayHandler,
 			fieldsHandler,
+			aiObservabilityHandler,
 			authzHandler,
 			rawDataExportHandler,
 			zeusHandler,
@@ -177,6 +181,7 @@ func newProvider(
 	infraMonitoringHandler inframonitoring.Handler,
 	gatewayHandler gateway.Handler,
 	fieldsHandler fields.Handler,
+	aiObservabilityHandler aiobservability.Handler,
 	authzHandler authz.Handler,
 	rawDataExportHandler rawdataexport.Handler,
 	zeusHandler zeus.Handler,
@@ -217,6 +222,7 @@ func newProvider(
 		infraMonitoringHandler:     infraMonitoringHandler,
 		gatewayHandler:             gatewayHandler,
 		fieldsHandler:              fieldsHandler,
+		aiObservabilityHandler:     aiObservabilityHandler,
 		authzHandler:               authzHandler,
 		rawDataExportHandler:       rawDataExportHandler,
 		zeusHandler:                zeusHandler,
@@ -313,6 +319,10 @@ func (provider *provider) AddToRouter(router *mux.Router) error {
 		return err
 	}
 
+	if err := provider.addAIObservabilityRoutes(router); err != nil {
+		return err
+	}
+
 	if err := provider.addRawDataExportRoutes(router); err != nil {
 		return err
 	}

pkg/modules/aiobservability/handler.go

@@ -0,0 +1,11 @@
+package aiobservability
+
+import "net/http"
+
+type Handler interface {
+	// Gets the fields keys the AI observability explorer can filter on
+	GetFieldsKeys(http.ResponseWriter, *http.Request)
+
+	// Gets the values the AI observability explorer can filter a field key on
+	GetFieldsValues(http.ResponseWriter, *http.Request)
+}

pkg/modules/aiobservability/implaiobservability/handler.go

@@ -0,0 +1,105 @@
+package implaiobservability
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/http/binding"
+	"github.com/SigNoz/signoz/pkg/http/render"
+	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
+	"github.com/SigNoz/signoz/pkg/types/authtypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/valuer"
+)
+
+type handler struct {
+	telemetryMetadataStore telemetrytypes.MetadataStore
+}
+
+func NewHandler(telemetryMetadataStore telemetrytypes.MetadataStore) aiobservability.Handler {
+	return &handler{
+		telemetryMetadataStore: telemetryMetadataStore,
+	}
+}
+
+func (handler *handler) GetFieldsKeys(rw http.ResponseWriter, req *http.Request) {
+	ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second)
+	defer cancel()
+
+	var params aiobservabilitytypes.PostableFieldKeysParams
+	if err := binding.Query.BindQuery(req.URL.Query(), &params); err != nil {
+		render.Error(rw, err)
+		return
+	}
+
+	claims, err := authtypes.ClaimsFromContext(ctx)
+	if err != nil {
+		render.Error(rw, err)
+		return
+	}
+	orgID := valuer.MustNewUUID(claims.OrgID)
+
+	fieldKeySelector := aiobservabilitytypes.NewFieldKeySelectorFromPostableFieldKeysParams(params)
+
+	keys := make(map[string][]*telemetrytypes.TelemetryFieldKey)
+	complete := true
+	// the trace context names the computed per-trace aggregates, which no scan can serve
+	if fieldKeySelector.FieldContext != telemetrytypes.FieldContextTrace {
+		keys, complete, err = handler.telemetryMetadataStore.GetKeys(ctx, orgID, fieldKeySelector)
+		if err != nil {
+			render.Error(rw, err)
+			return
+		}
+	}
+
+	render.Success(rw, http.StatusOK, &telemetrytypes.GettableFieldKeys{
+		Keys:     aitelemetryschema.FieldKeys(keys, fieldKeySelector),
+		Complete: complete,
+	})
+}
+
+func (handler *handler) GetFieldsValues(rw http.ResponseWriter, req *http.Request) {
+	ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second)
+	defer cancel()
+
+	// binding ignores query params the struct does not declare, so an unsupported
+	// existingQuery would silently return values it did not narrow
+	if req.URL.Query().Has("existingQuery") {
+		render.Error(rw, errors.New(errors.TypeInvalidInput, errors.CodeInvalidInput, "existingQuery is not supported"))
+		return
+	}
+
+	var params aiobservabilitytypes.PostableFieldValueParams
+	if err := binding.Query.BindQuery(req.URL.Query(), &params); err != nil {
+		render.Error(rw, err)
+		return
+	}
+
+	claims, err := authtypes.ClaimsFromContext(ctx)
+	if err != nil {
+		render.Error(rw, err)
+		return
+	}
+
+	fieldValueSelector := aiobservabilitytypes.NewFieldValueSelectorFromPostableFieldValueParams(params)
+
+	values := &telemetrytypes.TelemetryFieldValues{}
+	complete := true
+	// the trace context names the computed per-trace aggregates, which are never ingested
+	if fieldValueSelector.FieldContext != telemetrytypes.FieldContextTrace {
+		values, complete, err = handler.telemetryMetadataStore.GetAllValues(ctx, valuer.MustNewUUID(claims.OrgID), fieldValueSelector)
+		if err != nil {
+			render.Error(rw, err)
+			return
+		}
+	}
+
+	render.Success(rw, http.StatusOK, &telemetrytypes.GettableFieldValues{
+		Values:   values,
+		Complete: complete,
+	})
+}

pkg/modules/llmpricingrule/impllmpricingrule/module.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/modules/llmpricingrule"
 	"github.com/SigNoz/signoz/pkg/querier"
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
 	"github.com/SigNoz/signoz/pkg/types/featuretypes"
 	"github.com/SigNoz/signoz/pkg/types/llmpricingruletypes"
 	"github.com/SigNoz/signoz/pkg/types/opamptypes"
@@ -213,18 +214,18 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 					Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
 						Name:   "A",
 						Signal: telemetrytypes.SignalTraces,
-						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", telemetrytypes.GenAIRequestModel)},
+						Filter: &qbtypes.Filter{Expression: fmt.Sprintf("%s EXISTS", aiobservabilitytypes.GenAIRequestModel)},
 						Aggregations: []qbtypes.TraceAggregation{
 							{Expression: "count()", Alias: "spanCount"},
 						},
 						GroupBy: []qbtypes.GroupByKey{
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          telemetrytypes.GenAIRequestModel,
+								Name:          aiobservabilitytypes.GenAIRequestModel,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
 							{TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
-								Name:          telemetrytypes.GenAIProviderName,
+								Name:          aiobservabilitytypes.GenAIProviderName,
 								FieldContext:  telemetrytypes.FieldContextSpan,
 								FieldDataType: telemetrytypes.FieldDataTypeString,
 							}},
@@ -254,9 +255,9 @@ func (module *module) discoverModels(ctx context.Context, orgID valuer.UUID) ([]
 		switch c.Type {
 		case qbtypes.ColumnTypeGroup:
 			switch c.Name {
-			case telemetrytypes.GenAIRequestModel:
+			case aiobservabilitytypes.GenAIRequestModel:
 				modelIdx = i
-			case telemetrytypes.GenAIProviderName:
+			case aiobservabilitytypes.GenAIProviderName:
 				providerIdx = i
 			}
 		case qbtypes.ColumnTypeAggregation:

pkg/querier/collect.go

@@ -23,11 +23,7 @@ func (q *querier) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, e
 		traces           uint64
 		tracesLastSeenAt time.Time
 	)
-	tracesLastSeenExpr := "max(timestamp)"
-	if q.hasColumn(ctx, tracestelemetryschema.DBName, tracestelemetryschema.SpanIndexV3TableName, "inserted_at") {
-		tracesLastSeenExpr = "max(inserted_at)"
-	}
-	if err := q.telemetryStore.ClickhouseDB().QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*), %s FROM %s", tracesLastSeenExpr, tracesTable)).Scan(&traces, &tracesLastSeenAt); err == nil {
+	if err := q.telemetryStore.ClickhouseDB().QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*), max(timestamp) FROM %s", tracesTable)).Scan(&traces, &tracesLastSeenAt); err == nil {
 		stats["telemetry.traces.count"] = traces
 		if tracesLastSeenAt.Unix() != 0 {
 			stats["telemetry.traces.last_observed.time"] = tracesLastSeenAt.UTC()
@@ -41,11 +37,7 @@ func (q *querier) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, e
 		logs           uint64
 		logsLastSeenAt time.Time
 	)
-	logsLastSeenExpr := "fromUnixTimestamp64Nano(max(timestamp))"
-	if q.hasColumn(ctx, logstelemetryschema.DBName, logstelemetryschema.LogsV2TableName, "inserted_at") {
-		logsLastSeenExpr = "max(inserted_at)"
-	}
-	if err := q.telemetryStore.ClickhouseDB().QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*), %s FROM %s", logsLastSeenExpr, logsTable)).Scan(&logs, &logsLastSeenAt); err == nil {
+	if err := q.telemetryStore.ClickhouseDB().QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*), fromUnixTimestamp64Nano(max(timestamp)) FROM %s", logsTable)).Scan(&logs, &logsLastSeenAt); err == nil {
 		stats["telemetry.logs.count"] = logs
 		if logsLastSeenAt.Unix() != 0 {
 			stats["telemetry.logs.last_observed.time"] = logsLastSeenAt.UTC()
@@ -59,11 +51,7 @@ func (q *querier) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, e
 		metrics           uint64
 		metricsLastSeenAt time.Time
 	)
-	metricsLastSeenExpr := "toDateTime(max(unix_milli) / 1000)"
-	if q.hasColumn(ctx, metricstelemetryschema.DBName, metricstelemetryschema.SamplesV4TableName, "inserted_at_unix_milli") {
-		metricsLastSeenExpr = "fromUnixTimestamp64Milli(max(inserted_at_unix_milli))"
-	}
-	if err := q.telemetryStore.ClickhouseDB().QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*), %s FROM %s", metricsLastSeenExpr, metricsTable)).Scan(&metrics, &metricsLastSeenAt); err == nil {
+	if err := q.telemetryStore.ClickhouseDB().QueryRow(ctx, fmt.Sprintf("SELECT COUNT(*), toDateTime(max(unix_milli) / 1000) FROM %s", metricsTable)).Scan(&metrics, &metricsLastSeenAt); err == nil {
 		stats["telemetry.metrics.count"] = metrics
 		if metricsLastSeenAt.Unix() != 0 {
 			stats["telemetry.metrics.last_observed.time"] = metricsLastSeenAt.UTC()
@@ -75,12 +63,3 @@ func (q *querier) Collect(ctx context.Context, _ valuer.UUID) (map[string]any, e
 
 	return stats, nil
 }
-
-func (q *querier) hasColumn(ctx context.Context, database, table, column string) bool {
-	var exists bool
-	if err := q.telemetryStore.ClickhouseDB().QueryRow(ctx, "SELECT hasColumnInTable(?, ?, ?)", database, table, column).Scan(&exists); err != nil {
-		q.logger.DebugContext(ctx, "failed to check column existence", errors.Attr(err))
-		return false
-	}
-	return exists
-}

pkg/signoz/handler.go

@@ -12,6 +12,8 @@ import (
 	"github.com/SigNoz/signoz/pkg/global"
 	"github.com/SigNoz/signoz/pkg/global/signozglobal"
 	"github.com/SigNoz/signoz/pkg/licensing"
+	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
+	"github.com/SigNoz/signoz/pkg/modules/aiobservability/implaiobservability"
 	"github.com/SigNoz/signoz/pkg/modules/apdex"
 	"github.com/SigNoz/signoz/pkg/modules/apdex/implapdex"
 	"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
@@ -72,6 +74,7 @@ type Handlers struct {
 	FlaggerHandler          flagger.Handler
 	GatewayHandler          gateway.Handler
 	Fields                  fields.Handler
+	AIObservability         aiobservability.Handler
 	AuthzHandler            authz.Handler
 	ZeusHandler             zeus.Handler
 	QuerierHandler          querier.Handler
@@ -120,6 +123,7 @@ func NewHandlers(
 		FlaggerHandler:          flagger.NewHandler(flaggerService),
 		GatewayHandler:          gateway.NewHandler(gatewayService),
 		Fields:                  implfields.NewHandler(providerSettings, telemetryMetadataStore),
+		AIObservability:         implaiobservability.NewHandler(telemetryMetadataStore),
 		AuthzHandler:            signozauthzapi.NewHandler(authz),
 		ZeusHandler:             zeus.NewHandler(zeusService, licensing),
 		QuerierHandler:          querierHandler,

pkg/signoz/openapi.go

@@ -17,6 +17,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/global"
 	"github.com/SigNoz/signoz/pkg/http/handler"
 	"github.com/SigNoz/signoz/pkg/instrumentation"
+	"github.com/SigNoz/signoz/pkg/modules/aiobservability"
 	"github.com/SigNoz/signoz/pkg/modules/authdomain"
 	"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
 	"github.com/SigNoz/signoz/pkg/modules/dashboard"
@@ -74,6 +75,7 @@ func NewOpenAPI(ctx context.Context, instrumentation instrumentation.Instrumenta
 		struct{ inframonitoring.Handler }{},
 		struct{ gateway.Handler }{},
 		struct{ fields.Handler }{},
+		struct{ aiobservability.Handler }{},
 		struct{ authz.Handler }{},
 		struct{ rawdataexport.Handler }{},
 		struct{ zeus.Handler }{},

pkg/signoz/provider.go

@@ -240,6 +240,7 @@ func NewSQLMigrationProviderFactories(
 		sqlmigration.NewFixSavedViewSelectedFieldsFactory(sqlstore),
 		sqlmigration.NewBackfillSavedViewRequestTypeFactory(sqlstore),
 		sqlmigration.NewRestructureAuthDomainConfigFactory(sqlstore),
+		sqlmigration.NewFixSavedViewSelectFieldsFactory(sqlstore),
 	)
 }
 
@@ -325,6 +326,7 @@ func NewAPIServerProviderFactories(orgGetter organization.Getter, authz authz.Au
 			handlers.InfraMonitoring,
 			handlers.GatewayHandler,
 			handlers.Fields,
+			handlers.AIObservability,
 			handlers.AuthzHandler,
 			handlers.RawDataExport,
 			handlers.ZeusHandler,

pkg/sqlmigration/114_fix_saved_view_select_fields.go

@@ -0,0 +1,224 @@
+package sqlmigration
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"log/slog"
+
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/migrate"
+
+	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/sqlstore"
+)
+
+// storableSavedViewSelectFieldsRow is the shape of the `saved_view` table this migration repairs.
+type storableSavedViewSelectFieldsRow struct {
+	bun.BaseModel `bun:"table:saved_view"`
+
+	ID   string `bun:"id,pk,type:text"`
+	Data string `bun:"data,type:text"`
+}
+
+// selectedField is a superset of the current shape (name/signal/fieldContext/
+// fieldDataType) and the legacy v1 shape (key/dataType/type) it replaced.
+type selectedField struct {
+	Name          string `json:"name"`
+	Signal        string `json:"signal"`
+	FieldContext  string `json:"fieldContext"`
+	FieldDataType string `json:"fieldDataType"`
+
+	Key      string `json:"key"`
+	DataType string `json:"dataType"`
+	Type     string `json:"type"`
+}
+
+// telemetryFieldKeyOutput is the current shape only.
+type telemetryFieldKeyOutput struct {
+	Name          string `json:"name"`
+	Signal        string `json:"signal"`
+	FieldContext  string `json:"fieldContext"`
+	FieldDataType string `json:"fieldDataType"`
+}
+
+// legacyTypeToFieldContext holds the legacy AttributeKeyType values whose current
+// spelling differs. "tag" resolves to attribute through a telemetrytypes alias kept
+// only for old DB entries, so store the current spelling rather than rely on it.
+var legacyTypeToFieldContext = map[string]string{
+	"tag":             "attribute",
+	"spanSearchScope": "span",
+}
+
+// legacyDataTypeToFieldDataType holds the legacy AttributeKeyDataType values with no
+// matching telemetrytypes.FieldDataType alias.
+var legacyDataTypeToFieldDataType = map[string]string{
+	"array(string)":  "[]string",
+	"array(int64)":   "[]int64",
+	"array(float64)": "[]float64",
+	"array(bool)":    "[]bool",
+}
+
+type fixSavedViewSelectFields struct {
+	sqlstore sqlstore.SQLStore
+	settings factory.ProviderSettings
+}
+
+func NewFixSavedViewSelectFieldsFactory(sqlstore sqlstore.SQLStore) factory.ProviderFactory[SQLMigration, Config] {
+	return factory.NewProviderFactory(factory.MustNewName("fix_saved_view_select_fields"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
+		return &fixSavedViewSelectFields{sqlstore: sqlstore, settings: ps}, nil
+	})
+}
+
+func (migration *fixSavedViewSelectFields) Register(migrations *migrate.Migrations) error {
+	return migrations.Register(migration.Up, migration.Down)
+}
+
+func (migration *fixSavedViewSelectFields) Up(ctx context.Context, db *bun.DB) error {
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	var rows []*storableSavedViewSelectFieldsRow
+	if err := tx.NewSelect().Model(&rows).Scan(ctx); err != nil {
+		return err
+	}
+
+	var fixed, skipped int
+	for _, row := range rows {
+		fixedData, changed, ok := fixSelectFields(row.Data)
+		if !ok {
+			migration.settings.Logger.WarnContext(ctx, "saved view data could not be parsed, leaving it untouched", slog.String("saved_view_id", row.ID), slog.String("raw_data", row.Data))
+			skipped++
+			continue
+		}
+		if !changed {
+			continue
+		}
+
+		fixed++
+		if _, err := tx.NewUpdate().Model((*storableSavedViewSelectFieldsRow)(nil)).Set("data = ?", fixedData).Where("id = ?", row.ID).Exec(ctx); err != nil {
+			return err
+		}
+	}
+
+	migration.settings.Logger.InfoContext(ctx, "fixed invalid saved view selectedFields entries", slog.Int("total", len(rows)), slog.Int("fixed", fixed), slog.Int("skipped", skipped))
+
+	return tx.Commit()
+}
+
+func (migration *fixSavedViewSelectFields) Down(context.Context, *bun.DB) error {
+	return nil
+}
+
+// marshalUnescaped encodes without json.Marshal's HTML escaping, which would
+// otherwise rewrite <, > and & as \u003c, \u003e and \u0026 throughout the row --
+// json.RawMessage included, so it reaches query expressions this migration only
+// carries through.
+func marshalUnescaped(v any) ([]byte, error) {
+	var buf bytes.Buffer
+	encoder := json.NewEncoder(&buf)
+	encoder.SetEscapeHTML(false)
+	if err := encoder.Encode(v); err != nil {
+		return nil, err
+	}
+
+	return bytes.TrimRight(buf.Bytes(), "\n"), nil
+}
+
+func fieldContextFromLegacyType(legacyType string) string {
+	if mapped, ok := legacyTypeToFieldContext[legacyType]; ok {
+		return mapped
+	}
+	return legacyType
+}
+
+func fieldDataTypeFromLegacyDataType(legacyDataType string) string {
+	if mapped, ok := legacyDataTypeToFieldDataType[legacyDataType]; ok {
+		return mapped
+	}
+	return legacyDataType
+}
+
+// fixSelectFields recovers or drops entries in spec.selectedFields that never got
+// mapped from the legacy key/dataType/type shape to the current
+// name/fieldContext/fieldDataType shape. Entries that still carry a legacy key are
+// recovered by renaming the fields; entries with neither a name nor a key are dropped
+// as unrecoverable. Returns ok=false if data can't be parsed at all, and changed=false
+// if there was nothing to fix.
+func fixSelectFields(data string) (fixed string, changed bool, ok bool) {
+	var raw map[string]json.RawMessage
+	if err := json.Unmarshal([]byte(data), &raw); err != nil {
+		return "", false, false
+	}
+
+	var spec map[string]json.RawMessage
+	if err := json.Unmarshal(raw["spec"], &spec); err != nil {
+		return "", false, false
+	}
+
+	selectedFieldsRaw, ok := spec["selectedFields"]
+	if !ok {
+		return "", false, true
+	}
+
+	var fieldsRaw []json.RawMessage
+	if err := json.Unmarshal(selectedFieldsRaw, &fieldsRaw); err != nil {
+		return "", false, false
+	}
+
+	fixedFields := make([]json.RawMessage, 0, len(fieldsRaw))
+	for _, rawField := range fieldsRaw {
+		var field selectedField
+		if err := json.Unmarshal(rawField, &field); err != nil {
+			return "", false, false
+		}
+
+		switch {
+		case field.Name != "":
+			// already valid -- keep the original bytes untouched, e.g. to preserve
+			// description/unit rather than dropping them by re-deriving the entry.
+			fixedFields = append(fixedFields, rawField)
+		case field.Key != "":
+			// legacy shape -- recover by renaming the fields.
+			recoveredJSON, err := marshalUnescaped(telemetryFieldKeyOutput{
+				Name:          field.Key,
+				FieldContext:  fieldContextFromLegacyType(field.Type),
+				FieldDataType: fieldDataTypeFromLegacyDataType(field.DataType),
+			})
+			if err != nil {
+				return "", false, false
+			}
+			fixedFields = append(fixedFields, recoveredJSON)
+			changed = true
+		default:
+			// neither name nor key -- unrecoverable, drop it.
+			changed = true
+		}
+	}
+
+	if !changed {
+		return "", false, true
+	}
+
+	fixedFieldsJSON, err := marshalUnescaped(fixedFields)
+	if err != nil {
+		return "", false, false
+	}
+	spec["selectedFields"] = fixedFieldsJSON
+
+	fixedSpec, err := marshalUnescaped(spec)
+	if err != nil {
+		return "", false, false
+	}
+	raw["spec"] = fixedSpec
+
+	fixedData, err := marshalUnescaped(raw)
+	if err != nil {
+		return "", false, false
+	}
+
+	return string(fixedData), true, true
+}

pkg/statementbuilder/aistatementbuilder/statement_builder.go

@@ -7,7 +7,9 @@ import (
 	"github.com/SigNoz/signoz/pkg/flagger"
 	"github.com/SigNoz/signoz/pkg/statementbuilder"
 	scopedtraces "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
@@ -25,7 +27,7 @@ func NewFactory(
 // Scope describes gen_ai for the scoped trace builder: an AI trace has >=1 gen_ai
 // LLM, tool, or agent span, and its list adds AI/LLM per-trace metrics.
 func Scope() scopedtraces.TraceScope {
-	gateKeyNames := []string{telemetrytypes.GenAIRequestModel, telemetrytypes.GenAIToolName, telemetrytypes.GenAIAgentName}
+	gateKeyNames := []string{aiobservabilitytypes.GenAIRequestModel, aiobservabilitytypes.GenAIToolName, aiobservabilitytypes.GenAIAgentName}
 	gateExprs := make([]string, 0, len(gateKeyNames))
 	gateKeys := make([]*telemetrytypes.TelemetryFieldKey, 0, len(gateKeyNames))
 	for _, name := range gateKeyNames {
@@ -37,14 +39,14 @@ func Scope() scopedtraces.TraceScope {
 		})
 	}
 
-	defs := telemetrytypes.GenAIFieldDefinitions
-	reqModel := defs[telemetrytypes.GenAIRequestModel]
-	toolName := defs[telemetrytypes.GenAIToolName]
-	inTok := defs[telemetrytypes.GenAIUsageInputTokens]
-	outTok := defs[telemetrytypes.GenAIUsageOutputTokens]
-	cost := defs[telemetrytypes.SignozGenAITotalCost]
-	inMsg := defs[telemetrytypes.GenAIInputMessages]
-	outMsg := defs[telemetrytypes.GenAIOutputMessages]
+	defs := aitelemetryschema.GenAIFields
+	reqModel := defs[aiobservabilitytypes.GenAIRequestModel]
+	toolName := defs[aiobservabilitytypes.GenAIToolName]
+	inTok := defs[aiobservabilitytypes.GenAIUsageInputTokens]
+	outTok := defs[aiobservabilitytypes.GenAIUsageOutputTokens]
+	cost := defs[aiobservabilitytypes.SignozGenAITotalCost]
+	inMsg := defs[aiobservabilitytypes.GenAIInputMessages]
+	outMsg := defs[aiobservabilitytypes.GenAIOutputMessages]
 
 	str := telemetrytypes.FieldDataTypeString
 	columns := append(scopedtraces.CommonTraceColumns(),
@@ -62,13 +64,20 @@ func Scope() scopedtraces.TraceScope {
 		scopedtraces.TraceColumn{Alias: "max_llm_duration_nano", Orderable: true, Expr: scopedtraces.ScopedToKeyColumn(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("duration_nano"), &reqModel)},
 		// errors across the whole trace (any span), so display-only.
 		scopedtraces.TraceColumn{Alias: "error_count", Expr: scopedtraces.CondCount(scopedtraces.IntrinsicSpanKey("has_error"), qbtypes.FilterOperatorEqual, true)},
-		// timestamp of the last gen_ai span (LLM/tool/agent), hence gate-scoped.
+		// timestamp of the last gen_ai span (LLM/tool/agent), hence gate-scoped;
+		// order-only: a raw-nanos threshold makes no sense in the filter bar.
 		scopedtraces.TraceColumn{Alias: "last_activity_time", Orderable: true, Expr: scopedtraces.ScopedReduce(scopedtraces.AggMax, scopedtraces.IntrinsicSpanKey("timestamp"))},
 		// previews: first call's input (the prompt), last call's output (the answer).
 		scopedtraces.TraceColumn{Alias: "input", SpanLevel: true, Expr: scopedtraces.PickBy(&inMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickEarliest)},
 		scopedtraces.TraceColumn{Alias: "output", SpanLevel: true, Expr: scopedtraces.PickBy(&outMsg, str, scopedtraces.IntrinsicSpanKey("timestamp"), scopedtraces.PickLatest)},
 	)
 
+	for i, c := range columns {
+		if _, ok := aitelemetryschema.TraceAggregateFields[c.Alias]; ok {
+			columns[i].Filterable = true
+		}
+	}
+
 	return scopedtraces.TraceScope{
 		FilterExpression:  strings.Join(gateExprs, " OR "),
 		FieldKeys:         gateKeys,

pkg/statementbuilder/aistatementbuilder/statement_builder_test.go

@@ -11,6 +11,8 @@ import (
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/statementbuilder"
 	scopedtraces "github.com/SigNoz/signoz/pkg/statementbuilder/scopedtracesstatementbuilder"
+	"github.com/SigNoz/signoz/pkg/telemetryschema/aitelemetryschema"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
@@ -40,8 +42,7 @@ func otelKeysMap() map[string][]*telemetrytypes.TelemetryFieldKey {
 
 	m := make(map[string][]*telemetrytypes.TelemetryFieldKey)
 
-	// mirrors what enrichWithGenAIKeys surfaces in production
-	for name, def := range telemetrytypes.GenAIFieldDefinitions {
+	for name, def := range aitelemetryschema.GenAIFields {
 		keyCopy := def
 		m[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
 	}
@@ -976,8 +977,8 @@ func TestBuild_UnsupportedRequestType(t *testing.T) {
 // mask, OR-combined.
 func TestBuild_TraceList_MultiVariantGateKey(t *testing.T) {
 	keys := otelKeysMap()
-	keys[telemetrytypes.GenAIToolName] = append(keys[telemetrytypes.GenAIToolName], &telemetrytypes.TelemetryFieldKey{
-		Name:          telemetrytypes.GenAIToolName,
+	keys[aiobservabilitytypes.GenAIToolName] = append(keys[aiobservabilitytypes.GenAIToolName], &telemetrytypes.TelemetryFieldKey{
+		Name:          aiobservabilitytypes.GenAIToolName,
 		Signal:        telemetrytypes.SignalTraces,
 		FieldContext:  telemetrytypes.FieldContextAttribute,
 		FieldDataType: telemetrytypes.FieldDataTypeFloat64,

pkg/statementbuilder/scopedtracesstatementbuilder/scope.go

@@ -24,9 +24,10 @@ type TraceColumn struct {
 	// ClickHouse resolves bare identifiers to same-SELECT aliases first, so any
 	// expression referencing that column would silently bind to the alias.
 	Alias string
-	// Orderable columns can be used in ORDER BY and the aggregate filter; all-span
-	// aggregates are display-only and set false.
-	Orderable bool
+	// Orderable columns can be used in ORDER BY, Filterable ones in the aggregate
+	// filter; all-span aggregates are display-only and set neither.
+	Orderable  bool
+	Filterable bool
 	// SpanLevel columns surface a real span/resource attribute; a filter on them is
 	// applied span-level, so they are excluded from the trace-level aliases.
 	SpanLevel bool

pkg/statementbuilder/scopedtracesstatementbuilder/statement_builder.go

@@ -185,18 +185,19 @@ func (b *scopedTraceStatementBuilder) buildTraceListQuery(
 		return nil, err
 	}
 	orderableSet := orderableAliasSet(resolved)
+	filterableSet := filterableAliasSet(resolved)
 
 	resourceFrag, resourceArgs, resourcePred, err := b.maybeAttachResourceFilter(ctx, orgID, query, start, end, variables)
 	if err != nil {
 		return nil, err
 	}
 
-	fp, err := b.splitFilter(ctx, orgID, query, b.aggregateAliasSet(), orderableSet, start, end, variables, matchedSB)
+	fp, err := b.splitFilter(ctx, orgID, query, b.aggregateAliasSet(), filterableSet, start, end, variables, matchedSB)
 	if err != nil {
 		return nil, err
 	}
 
-	matchedFrag, matchedArgs, err := b.buildMatchedCTE(matchedSB, start, end, startBucket, endBucket, resolved, orders, orderableSet, maskExpr, fp, resourcePred, limit, query.Offset)
+	matchedFrag, matchedArgs, err := b.buildMatchedCTE(matchedSB, start, end, startBucket, endBucket, resolved, orders, orderableSet, filterableSet, maskExpr, fp, resourcePred, limit, query.Offset)
 	if err != nil {
 		return nil, err
 	}
@@ -326,9 +327,10 @@ func (b *scopedTraceStatementBuilder) resolveMask(ctx context.Context, orgID val
 }
 
 type resolvedColumn struct {
-	alias     string
-	expr      string
-	orderable bool
+	alias      string
+	expr       string
+	orderable  bool
+	filterable bool
 }
 
 func (b *scopedTraceStatementBuilder) resolveColumns(ctx context.Context, orgID valuer.UUID, start, end uint64, cols *columnResolver, preds *predicateResolver) ([]resolvedColumn, error) {
@@ -338,7 +340,7 @@ func (b *scopedTraceStatementBuilder) resolveColumns(ctx context.Context, orgID
 		if err != nil {
 			return nil, err
 		}
-		out = append(out, resolvedColumn{alias: c.Alias, expr: expr, orderable: c.Orderable})
+		out = append(out, resolvedColumn{alias: c.Alias, expr: expr, orderable: c.Orderable, filterable: c.Filterable})
 	}
 	return out, nil
 }
@@ -393,7 +395,7 @@ type filterParts struct {
 // splitFilter splits query.Filter into a span-level predicate (args bound into sb)
 // and a trace-level HAVING (explicit query.Having ANDed on), then validates the
 // trace-level part against the matched-pass aggregates.
-func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID valuer.UUID, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], classifySet, orderableSet map[string]struct{}, start, end uint64, variables map[string]qbtypes.VariableItem, sb *sqlbuilder.SelectBuilder) (filterParts, error) {
+func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID valuer.UUID, query qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], classifySet, filterableSet map[string]struct{}, start, end uint64, variables map[string]qbtypes.VariableItem, sb *sqlbuilder.SelectBuilder) (filterParts, error) {
 	var fp filterParts
 	if query.Filter != nil && strings.TrimSpace(query.Filter.Expression) != "" {
 		spanExpr, traceExpr, err := querybuilder.SplitFilterForAggregates(query.Filter.Expression, classifySet)
@@ -429,7 +431,7 @@ func (b *scopedTraceStatementBuilder) splitFilter(ctx context.Context, orgID val
 		}
 		fp.havingExpr = replaced
 	}
-	if err := validateAggregateFilter(fp.havingExpr, orderableSet); err != nil {
+	if err := validateAggregateFilter(fp.havingExpr, filterableSet); err != nil {
 		return fp, err
 	}
 	return fp, nil
@@ -473,7 +475,7 @@ func (b *scopedTraceStatementBuilder) resolveSpanPredicate(ctx context.Context,
 // span filter + HAVING + ORDER BY + LIMIT/OFFSET, selecting only the aliases ORDER BY
 // / HAVING reference. Expressions carry $n markers bound to sb, so each can appear
 // several times and every occurrence resolves to the same arg.
-func (b *scopedTraceStatementBuilder) buildMatchedCTE(sb *sqlbuilder.SelectBuilder, start, end, startBucket, endBucket uint64, resolved []resolvedColumn, orders []listOrder, orderableSet map[string]struct{}, maskExpr string, fp filterParts, resourcePred string, limit, offset int) (string, []any, error) {
+func (b *scopedTraceStatementBuilder) buildMatchedCTE(sb *sqlbuilder.SelectBuilder, start, end, startBucket, endBucket uint64, resolved []resolvedColumn, orders []listOrder, orderableSet, filterableSet map[string]struct{}, maskExpr string, fp filterParts, resourcePred string, limit, offset int) (string, []any, error) {
 	needed := neededMatchedAliases(orders, fp.havingExpr, orderableSet)
 	selects := []string{"trace_id"}
 	for _, rc := range resolved {
@@ -513,8 +515,8 @@ func (b *scopedTraceStatementBuilder) buildMatchedCTE(sb *sqlbuilder.SelectBuild
 	}
 	if strings.TrimSpace(fp.havingExpr) != "" {
 		// the rewriter matches raw key text, so map the trace. form alongside the bare name
-		columnMap := make(map[string]string, len(orderableSet)*2)
-		for a := range orderableSet {
+		columnMap := make(map[string]string, len(filterableSet)*2)
+		for a := range filterableSet {
 			columnMap[a] = quoteAlias(a)
 			columnMap[telemetrytypes.FieldContextTrace.StringValue()+"."+a] = quoteAlias(a)
 		}
@@ -603,6 +605,17 @@ func orderableAliasSet(resolved []resolvedColumn) map[string]struct{} {
 	return set
 }
 
+// filterableAliasSet is the subset of aliases usable in the trace-level filter.
+func filterableAliasSet(resolved []resolvedColumn) map[string]struct{} {
+	set := make(map[string]struct{})
+	for _, rc := range resolved {
+		if rc.filterable {
+			set[rc.alias] = struct{}{}
+		}
+	}
+	return set
+}
+
 // neededMatchedAliases is the minimal alias set the matched pass must select: those
 // in ORDER BY plus those in the aggregate HAVING.
 func neededMatchedAliases(orders []listOrder, havingExpr string, orderableSet map[string]struct{}) map[string]struct{} {
@@ -630,19 +643,19 @@ func traceAggregateNames(havingExpr string) []string {
 	return names
 }
 
-// validateAggregateFilter rejects a trace-level filter referencing an aggregate not
-// computable in the matched pass.
-func validateAggregateFilter(havingExpr string, orderableSet map[string]struct{}) error {
+// validateAggregateFilter rejects a trace-level filter referencing an aggregate that
+// is not filterable.
+func validateAggregateFilter(havingExpr string, filterableSet map[string]struct{}) error {
 	if strings.TrimSpace(havingExpr) == "" {
 		return nil
 	}
-	allowed := make([]string, 0, len(orderableSet))
-	for a := range orderableSet {
+	allowed := make([]string, 0, len(filterableSet))
+	for a := range filterableSet {
 		allowed = append(allowed, a)
 	}
 	sort.Strings(allowed)
 	for _, name := range traceAggregateNames(havingExpr) {
-		if _, ok := orderableSet[name]; !ok {
+		if _, ok := filterableSet[name]; !ok {
 			return errors.NewInvalidInputf(errors.CodeInvalidInput,
 				"aggregate %q cannot be used in the trace-list filter; filterable aggregates: %s", name, strings.Join(allowed, ", "))
 		}

pkg/telemetrymetadata/metadata.go

@@ -1168,27 +1168,6 @@ func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFi
 	return keys
 }
 
-// enrichWithGenAIKeys adds keys that can be queried for GenAI signals, even though they have not been ingested yet.
-func enrichWithGenAIKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
-	for _, selector := range selectors {
-		if selector.Signal != telemetrytypes.SignalTraces && selector.Signal != telemetrytypes.SignalUnspecified {
-			continue
-		}
-		for name, def := range telemetrytypes.GenAIFieldDefinitions {
-			if len(keys[name]) > 0 {
-				continue // already resolved from ingested data
-			}
-			if !selectorMatchesIntrinsicField(selector, def) {
-				continue
-			}
-			keyCopy := def
-			keys[name] = []*telemetrytypes.TelemetryFieldKey{&keyCopy}
-		}
-	}
-
-	return keys
-}
-
 func selectorMatchesIntrinsicField(selector *telemetrytypes.FieldKeySelector, definition telemetrytypes.TelemetryFieldKey) bool {
 	if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != definition.FieldContext {
 		return false
@@ -1274,9 +1253,6 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, orgID valuer.UUID, fie
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
-	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
-		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, selectors)
-	}
 
 	return mapOfKeys, complete, nil
 }
@@ -1355,9 +1331,6 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, orgID valuer.UUID
 
 	applyBackwardCompatibleKeys(mapOfKeys)
 	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
-	if t.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableAIObservability, featuretypes.NewFlaggerEvaluationContext(orgID)) {
-		mapOfKeys = enrichWithGenAIKeys(mapOfKeys, fieldKeySelectors)
-	}
 
 	return mapOfKeys, complete, nil
 }

pkg/telemetryschema/aitelemetryschema/const.go

@@ -0,0 +1,58 @@
+package aitelemetryschema
+
+import (
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+var (
+	// GenAIFields are the gen_ai span attributes the AI query builder relies on,
+	// suggested before ingestion so the filter bar works on a fresh install.
+	GenAIFields = map[string]telemetrytypes.TelemetryFieldKey{
+		aiobservabilitytypes.GenAIRequestModel:  genAIAttribute(aiobservabilitytypes.GenAIRequestModel, telemetrytypes.FieldDataTypeString),
+		aiobservabilitytypes.GenAIOperationName: genAIAttribute(aiobservabilitytypes.GenAIOperationName, telemetrytypes.FieldDataTypeString),
+		aiobservabilitytypes.GenAIToolName:      genAIAttribute(aiobservabilitytypes.GenAIToolName, telemetrytypes.FieldDataTypeString),
+		aiobservabilitytypes.GenAIAgentName:     genAIAttribute(aiobservabilitytypes.GenAIAgentName, telemetrytypes.FieldDataTypeString),
+		aiobservabilitytypes.GenAIProviderName:  genAIAttribute(aiobservabilitytypes.GenAIProviderName, telemetrytypes.FieldDataTypeString),
+
+		aiobservabilitytypes.GenAIUsageInputTokens:              genAIAttribute(aiobservabilitytypes.GenAIUsageInputTokens, telemetrytypes.FieldDataTypeFloat64),
+		aiobservabilitytypes.GenAIUsageOutputTokens:             genAIAttribute(aiobservabilitytypes.GenAIUsageOutputTokens, telemetrytypes.FieldDataTypeFloat64),
+		aiobservabilitytypes.GenAIUsageCacheReadInputTokens:     genAIAttribute(aiobservabilitytypes.GenAIUsageCacheReadInputTokens, telemetrytypes.FieldDataTypeFloat64),
+		aiobservabilitytypes.GenAIUsageCacheCreationInputTokens: genAIAttribute(aiobservabilitytypes.GenAIUsageCacheCreationInputTokens, telemetrytypes.FieldDataTypeFloat64),
+		aiobservabilitytypes.SignozGenAITotalCost:               genAIAttribute(aiobservabilitytypes.SignozGenAITotalCost, telemetrytypes.FieldDataTypeFloat64),
+
+		aiobservabilitytypes.GenAIInputMessages:  genAIAttribute(aiobservabilitytypes.GenAIInputMessages, telemetrytypes.FieldDataTypeString),
+		aiobservabilitytypes.GenAIOutputMessages: genAIAttribute(aiobservabilitytypes.GenAIOutputMessages, telemetrytypes.FieldDataTypeString),
+	}
+
+	// TraceAggregateFields are the per-trace aggregates the AI trace list computes;
+	// they are never ingested, so only this definition can surface them.
+	TraceAggregateFields = map[string]telemetrytypes.TelemetryFieldKey{
+		"llm_call_count":        traceAggregate("llm_call_count"),
+		"tool_call_count":       traceAggregate("tool_call_count"),
+		"distinct_tool_count":   traceAggregate("distinct_tool_count"),
+		"input_tokens":          traceAggregate("input_tokens"),
+		"output_tokens":         traceAggregate("output_tokens"),
+		"total_tokens":          traceAggregate("total_tokens"),
+		"estimated_total_cost":  traceAggregate("estimated_total_cost"),
+		"max_llm_duration_nano": traceAggregate("max_llm_duration_nano"),
+	}
+)
+
+func genAIAttribute(name string, dataType telemetrytypes.FieldDataType) telemetrytypes.TelemetryFieldKey {
+	return telemetrytypes.TelemetryFieldKey{
+		Name:          name,
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextAttribute,
+		FieldDataType: dataType,
+	}
+}
+
+func traceAggregate(name string) telemetrytypes.TelemetryFieldKey {
+	return telemetrytypes.TelemetryFieldKey{
+		Name:          name,
+		Signal:        telemetrytypes.SignalTraces,
+		FieldContext:  telemetrytypes.FieldContextTrace,
+		FieldDataType: telemetrytypes.FieldDataTypeFloat64,
+	}
+}

pkg/telemetryschema/aitelemetryschema/field_keys.go

@@ -0,0 +1,26 @@
+package aitelemetryschema
+
+import (
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// FieldKeys merges in the keys the metadata store cannot serve: the computed
+// aggregates always, the gen_ai attributes only where ingestion has not.
+func FieldKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selector *telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
+	for name, def := range TraceAggregateFields {
+		if selector.MatchesKey(&def) {
+			keys[name] = append(keys[name], &def)
+		}
+	}
+
+	for name, def := range GenAIFields {
+		if len(keys[name]) > 0 {
+			continue
+		}
+		if selector.MatchesKey(&def) {
+			keys[name] = []*telemetrytypes.TelemetryFieldKey{&def}
+		}
+	}
+
+	return keys
+}

pkg/types/aiobservabilitytypes/field.go

@@ -0,0 +1,46 @@
+package aiobservabilitytypes
+
+import (
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+// the explorer lists AI traces, so the signal is always traces and the metric
+// selectors do not apply.
+type PostableFieldKeysParams struct {
+	SearchText     string                       `query:"searchText"`
+	FieldContext   telemetrytypes.FieldContext  `query:"fieldContext"`
+	FieldDataType  telemetrytypes.FieldDataType `query:"fieldDataType"`
+	StartUnixMilli int64                        `query:"startUnixMilli"`
+	EndUnixMilli   int64                        `query:"endUnixMilli"`
+	Limit          int                          `query:"limit"`
+}
+
+// existingQuery is unsupported until the computed per-trace aggregates it may
+// reference can be narrowed on.
+type PostableFieldValueParams struct {
+	PostableFieldKeysParams
+	Name string `query:"name"`
+}
+
+func NewFieldKeySelectorFromPostableFieldKeysParams(params PostableFieldKeysParams) *telemetrytypes.FieldKeySelector {
+	return telemetrytypes.NewFieldKeySelectorFromPostableFieldKeysParams(params.telemetryParams())
+}
+
+func NewFieldValueSelectorFromPostableFieldValueParams(params PostableFieldValueParams) *telemetrytypes.FieldValueSelector {
+	return telemetrytypes.NewFieldValueSelectorFromPostableFieldValueParams(telemetrytypes.PostableFieldValueParams{
+		PostableFieldKeysParams: params.telemetryParams(),
+		Name:                    params.Name,
+	})
+}
+
+func (params PostableFieldKeysParams) telemetryParams() telemetrytypes.PostableFieldKeysParams {
+	return telemetrytypes.PostableFieldKeysParams{
+		Signal:         telemetrytypes.SignalTraces,
+		SearchText:     params.SearchText,
+		FieldContext:   params.FieldContext,
+		FieldDataType:  params.FieldDataType,
+		StartUnixMilli: params.StartUnixMilli,
+		EndUnixMilli:   params.EndUnixMilli,
+		Limit:          params.Limit,
+	}
+}

pkg/types/aiobservabilitytypes/keys.go

@@ -0,0 +1,28 @@
+package aiobservabilitytypes
+
+// OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
+// shared by the AI query builder and the LLM pricing pipeline.
+const (
+	GenAIRequestModel  = "gen_ai.request.model"
+	GenAIOperationName = "gen_ai.operation.name"
+	GenAIToolName      = "gen_ai.tool.name"
+	GenAIAgentName     = "gen_ai.agent.name"
+	GenAIProviderName  = "gen_ai.provider.name"
+
+	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
+	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
+	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
+	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
+
+	GenAIInputMessages  = "gen_ai.input.messages"
+	GenAIOutputMessages = "gen_ai.output.messages"
+)
+
+// Per-span costs the SigNoz LLM pricing processor attaches; not OTel semconv.
+const (
+	SignozGenAICostInput      = "_signoz.gen_ai.cost_input"
+	SignozGenAICostOutput     = "_signoz.gen_ai.cost_output"
+	SignozGenAICostCacheRead  = "_signoz.gen_ai.cost_cache_read"
+	SignozGenAICostCacheWrite = "_signoz.gen_ai.cost_cache_write"
+	SignozGenAITotalCost      = "_signoz.gen_ai.total_cost"
+)

pkg/types/llmpricingruletypes/pricing.go

@@ -15,11 +15,6 @@ import (
 
 const (
 	LLMCostFeatureType agentConf.AgentFeatureType = "llm_pricing"
-
-	SignozGenAICostInput      = "_signoz.gen_ai.cost_input"
-	SignozGenAICostOutput     = "_signoz.gen_ai.cost_output"
-	SignozGenAICostCacheRead  = "_signoz.gen_ai.cost_cache_read"
-	SignozGenAICostCacheWrite = "_signoz.gen_ai.cost_cache_write"
 )
 
 var (

pkg/types/llmpricingruletypes/processorconfig.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 
 	"github.com/SigNoz/signoz/pkg/errors"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
 	"gopkg.in/yaml.v3"
 )
 
@@ -84,21 +84,21 @@ func buildProcessorConfig(rules []*LLMPricingRule) *LLMPricingRuleProcessorConfi
 
 	return &LLMPricingRuleProcessorConfig{
 		Attrs: LLMPricingRuleProcessorAttrs{
-			Model:      telemetrytypes.GenAIRequestModel,
-			In:         telemetrytypes.GenAIUsageInputTokens,
-			Out:        telemetrytypes.GenAIUsageOutputTokens,
-			CacheRead:  telemetrytypes.GenAIUsageCacheReadInputTokens,
-			CacheWrite: telemetrytypes.GenAIUsageCacheCreationInputTokens,
+			Model:      aiobservabilitytypes.GenAIRequestModel,
+			In:         aiobservabilitytypes.GenAIUsageInputTokens,
+			Out:        aiobservabilitytypes.GenAIUsageOutputTokens,
+			CacheRead:  aiobservabilitytypes.GenAIUsageCacheReadInputTokens,
+			CacheWrite: aiobservabilitytypes.GenAIUsageCacheCreationInputTokens,
 		},
 		DefaultPricing: LLMPricingRuleProcessorDefaultPricing{
 			Rules: pricingRules,
 		},
 		OutputAttrs: LLMPricingRuleProcessorOutputAttrs{
-			In:         SignozGenAICostInput,
-			Out:        SignozGenAICostOutput,
-			CacheRead:  SignozGenAICostCacheRead,
-			CacheWrite: SignozGenAICostCacheWrite,
-			Total:      telemetrytypes.SignozGenAITotalCost,
+			In:         aiobservabilitytypes.SignozGenAICostInput,
+			Out:        aiobservabilitytypes.SignozGenAICostOutput,
+			CacheRead:  aiobservabilitytypes.SignozGenAICostCacheRead,
+			CacheWrite: aiobservabilitytypes.SignozGenAICostCacheWrite,
+			Total:      aiobservabilitytypes.SignozGenAITotalCost,
 		},
 	}
 }

pkg/types/quickfiltertypes/filter.go

@@ -7,7 +7,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/errors"
 	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
 	"github.com/SigNoz/signoz/pkg/types"
-	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/types/aiobservabilitytypes"
 	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/uptrace/bun"
 )
@@ -195,12 +195,12 @@ func NewDefaultQuickFilter(orgID valuer.UUID) ([]*StorableQuickFilter, error) {
 	// usage: env scoping, the LLM identity keys, then service and the rest.
 	aiObservabilityFilters := []map[string]interface{}{
 		{"key": "deployment.environment", "dataType": "string", "type": "resource"},
-		{"key": telemetrytypes.GenAIOperationName, "dataType": "string", "type": "tag"},
-		{"key": telemetrytypes.GenAIProviderName, "dataType": "string", "type": "tag"},
-		{"key": telemetrytypes.GenAIRequestModel, "dataType": "string", "type": "tag"},
+		{"key": aiobservabilitytypes.GenAIOperationName, "dataType": "string", "type": "tag"},
+		{"key": aiobservabilitytypes.GenAIProviderName, "dataType": "string", "type": "tag"},
+		{"key": aiobservabilitytypes.GenAIRequestModel, "dataType": "string", "type": "tag"},
 		{"key": "service.name", "dataType": "string", "type": "resource"},
-		{"key": telemetrytypes.GenAIToolName, "dataType": "string", "type": "tag"},
-		{"key": telemetrytypes.GenAIAgentName, "dataType": "string", "type": "tag"},
+		{"key": aiobservabilitytypes.GenAIToolName, "dataType": "string", "type": "tag"},
+		{"key": aiobservabilitytypes.GenAIAgentName, "dataType": "string", "type": "tag"},
 	}
 
 	tracesJSON, err := json.Marshal(tracesFilters)

pkg/types/savedviewtypes/spec.go

@@ -92,6 +92,11 @@ func (s *SavedViewSpec) Validate() error {
 	if s.RequestType.IsZero() {
 		return errors.NewInvalidInputf(ErrCodeSavedViewInvalidInput, "requestType is required")
 	}
+	for i, field := range s.SelectedFields {
+		if field.Name == "" {
+			return errors.NewInvalidInputf(ErrCodeSavedViewInvalidInput, "selectedFields[%d].name is required", i)
+		}
+	}
 
 	return (&qbtypes.CompositeQuery{Queries: s.Queries}).Validate(qbtypes.GetValidationOptions(s.RequestType)...)
 }

pkg/types/savedviewtypes/spec_test.go

@@ -99,6 +99,17 @@ func TestSavedViewSpecValidate(t *testing.T) {
 			},
 			expectError: false,
 		},
+		{
+			name: "selectedFields entry with no name is rejected",
+			spec: SavedViewSpec{
+				DisplayName:    "My View",
+				PanelType:      PanelTypeTable,
+				RequestType:    qbtypes.RequestTypeScalar,
+				Queries:        validQueries(),
+				SelectedFields: []telemetrytypes.TelemetryFieldKey{{Name: "service.name"}, {}},
+			},
+			expectError: true,
+		},
 		{
 			name: "nil selectedFields is valid -- selectedFields itself is not required",
 			spec: SavedViewSpec{

pkg/types/telemetrytypes/field.go

@@ -246,6 +246,27 @@ type FieldKeySelector struct {
 	MetricContext     *MetricContext         `json:"metricContext,omitempty"`
 }
 
+// MatchesKey reports whether a statically defined key satisfies the selector, so
+// callers can suggest keys that were never ingested.
+func (s *FieldKeySelector) MatchesKey(key *TelemetryFieldKey) bool {
+	if s.FieldContext != FieldContextUnspecified && s.FieldContext != key.FieldContext {
+		return false
+	}
+
+	if s.FieldDataType != FieldDataTypeUnspecified && s.FieldDataType != key.FieldDataType {
+		return false
+	}
+
+	if s.Name == "" {
+		return true
+	}
+
+	if s.SelectorMatchType == FieldSelectorMatchTypeExact {
+		return strings.EqualFold(s.Name, key.Name)
+	}
+	return strings.Contains(strings.ToLower(key.Name), strings.ToLower(s.Name))
+}
+
 type FieldValueSelector struct {
 	*FieldKeySelector
 	ExistingQuery string `json:"existingQuery"`

pkg/types/telemetrytypes/field_context.go

@@ -188,7 +188,7 @@ func (FieldContext) Enum() []any {
 		FieldContextMetric,
 		FieldContextLog,
 		FieldContextSpan,
-		// FieldContextTrace,
+		FieldContextTrace,
 		FieldContextResource,
 		// FieldContextScope,
 		FieldContextAttribute,

pkg/types/telemetrytypes/genai_semconv.go

@@ -1,43 +0,0 @@
-package telemetrytypes
-
-// OpenTelemetry gen_ai semantic-convention attribute keys. Single source of truth
-// shared by the AI query builder and the LLM pricing pipeline.
-const (
-	GenAIRequestModel  = "gen_ai.request.model"
-	GenAIOperationName = "gen_ai.operation.name"
-	GenAIToolName      = "gen_ai.tool.name"
-	GenAIAgentName     = "gen_ai.agent.name"
-	GenAIProviderName  = "gen_ai.provider.name"
-
-	GenAIUsageInputTokens              = "gen_ai.usage.input_tokens"
-	GenAIUsageOutputTokens             = "gen_ai.usage.output_tokens"
-	GenAIUsageCacheReadInputTokens     = "gen_ai.usage.cache_read.input_tokens"
-	GenAIUsageCacheCreationInputTokens = "gen_ai.usage.cache_creation.input_tokens"
-
-	GenAIInputMessages  = "gen_ai.input.messages"
-	GenAIOutputMessages = "gen_ai.output.messages"
-
-	// SignozGenAITotalCost is not OTel semconv: it is the per-span cost the SigNoz
-	// LLM pricing processor attaches.
-	SignozGenAITotalCost = "_signoz.gen_ai.total_cost"
-)
-
-// GenAIFieldDefinitions are the gen_ai span attributes the AI query builder relies
-// on, surfaced by the metadata store even before ingestion so the AI gate/columns
-// resolve on a fresh install.
-var GenAIFieldDefinitions = map[string]TelemetryFieldKey{
-	GenAIRequestModel:  {Name: GenAIRequestModel, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIOperationName: {Name: GenAIOperationName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIToolName:      {Name: GenAIToolName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIAgentName:     {Name: GenAIAgentName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIProviderName:  {Name: GenAIProviderName, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-
-	GenAIUsageInputTokens:              {Name: GenAIUsageInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageOutputTokens:             {Name: GenAIUsageOutputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageCacheReadInputTokens:     {Name: GenAIUsageCacheReadInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	GenAIUsageCacheCreationInputTokens: {Name: GenAIUsageCacheCreationInputTokens, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-	SignozGenAITotalCost:               {Name: SignozGenAITotalCost, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeFloat64},
-
-	GenAIInputMessages:  {Name: GenAIInputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-	GenAIOutputMessages: {Name: GenAIOutputMessages, Signal: SignalTraces, FieldContext: FieldContextAttribute, FieldDataType: FieldDataTypeString},
-}

tests/fixtures/metadata.py

@@ -6,6 +6,7 @@ from typing import Any
 
 import numpy as np
 import pytest
+import requests
 
 from fixtures import types
 from fixtures.fingerprint import LogsOrTracesFingerprint
@@ -106,6 +107,24 @@ def truncate_attributes_metadata_table(conn, cluster: str) -> None:
     conn.query(f"TRUNCATE TABLE signoz_metadata.attributes_metadata ON CLUSTER '{cluster}' SYNC")
 
 
+def get_field_keys(signoz: types.SigNoz, token: str, params: dict, path: str = "/api/v1/fields/keys") -> requests.Response:
+    return requests.get(
+        signoz.self.host_configs["8080"].get(path),
+        timeout=5,
+        headers={"authorization": f"Bearer {token}"},
+        params=params,
+    )
+
+
+def get_field_values(signoz: types.SigNoz, token: str, params: dict, path: str = "/api/v1/fields/values") -> requests.Response:
+    return requests.get(
+        signoz.self.host_configs["8080"].get(path),
+        timeout=5,
+        headers={"authorization": f"Bearer {token}"},
+        params=params,
+    )
+
+
 @pytest.fixture(name="insert_attributes_metadata", scope="function")
 def insert_attributes_metadata(
     clickhouse: types.TestContainerClickhouse,

tests/integration/tests/querierai/03_ai_metadata.py

@@ -0,0 +1,137 @@
+from collections.abc import Callable
+from datetime import UTC, datetime
+from http import HTTPStatus
+
+from fixtures import types
+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
+from fixtures.metadata import get_field_keys, get_field_values
+from fixtures.querierai import ai_trace
+from fixtures.traces import Traces
+
+AI_KEYS_PATH = "/api/v1/ai_observability/fields/keys"
+AI_VALUES_PATH = "/api/v1/ai_observability/fields/values"
+
+# The filterable per-trace aggregates; the display-only columns (error_count,
+# last_activity_time, span_count, input, output) must not be suggested.
+AI_TRACE_AGGREGATES = {
+    "llm_call_count",
+    "tool_call_count",
+    "distinct_tool_count",
+    "input_tokens",
+    "output_tokens",
+    "total_tokens",
+    "estimated_total_cost",
+    "max_llm_duration_nano",
+}
+
+
+def test_ai_fields_lists_filterable_aggregates(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_field_keys(signoz, token, {}, AI_KEYS_PATH)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert response.json()["status"] == "success"
+
+    keys = response.json()["data"]["keys"]
+    trace_context_names = {name for name, variants in keys.items() if any(key["fieldContext"] == "trace" for key in variants)}
+    assert trace_context_names == AI_TRACE_AGGREGATES, keys
+
+
+def test_ai_fields_trace_prefix_search(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    # `trace.output` in the filter bar parses into the trace context
+    response = get_field_keys(signoz, token, {"searchText": "trace.output"}, AI_KEYS_PATH)
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    keys = response.json()["data"]["keys"]
+    trace_context_names = {name for name, variants in keys.items() if any(key["fieldContext"] == "trace" for key in variants)}
+    assert trace_context_names == {"output_tokens"}, keys
+
+
+def test_ai_fields_bare_prefix_suggests_aggregate_and_attribute(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_field_keys(signoz, token, {"searchText": "output_tok"}, AI_KEYS_PATH)
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    keys = response.json()["data"]["keys"]
+    assert any(key["fieldContext"] == "trace" for key in keys["output_tokens"]), keys
+    assert any(key["fieldContext"] == "attribute" for key in keys["gen_ai.usage.output_tokens"]), keys
+
+
+def test_ai_fields_are_not_served_by_the_generic_endpoint(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_field_keys(signoz, token, {"signal": "traces", "searchText": "output_tok"})
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    keys = response.json()["data"]["keys"]
+    assert "output_tokens" not in keys, keys
+    assert "gen_ai.usage.output_tokens" not in keys, keys
+
+
+def test_ai_field_values_suggests_ingested_attribute_values(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+    insert_traces: Callable[[list[Traces]], None],
+) -> None:
+    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
+    insert_traces(ai_trace(now=now, service="ai-it-values", user="alice", in_tokens=100, out_tokens=20, cost=0.5, model="gpt-it-values"))
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_field_values(signoz, token, {"name": "gen_ai.request.model", "searchText": "gpt-it-values"}, AI_VALUES_PATH)
+    assert response.status_code == HTTPStatus.OK, response.text
+    assert response.json()["status"] == "success"
+
+    values = response.json()["data"]["values"]
+    assert values["stringValues"] == ["gpt-it-values"], values
+
+
+def test_ai_field_values_reject_existing_query(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_field_values(
+        signoz,
+        token,
+        {"name": "gen_ai.request.model", "existingQuery": "service.name = 'ai-it-values'"},
+        AI_VALUES_PATH,
+    )
+    assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
+
+
+def test_ai_field_values_of_computed_aggregate_are_empty(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+
+    response = get_field_values(signoz, token, {"name": "llm_call_count", "fieldContext": "trace"}, AI_VALUES_PATH)
+    assert response.status_code == HTTPStatus.OK, response.text
+
+    values = response.json()["data"]["values"]
+    assert values.get("stringValues", []) == [], values
+    assert values.get("numberValues", []) == [], values