chore: add further filtering for required attribute columns

pkg/querier/querier.go

@@ -426,17 +426,9 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 	}
 	nonExistentMetrics := []string{}
 	var dormantMetricsWarningMsg string
-	// internal metrics aren't user-controlled — skip errors/warnings for them since users can't act on them
-	isInternalMetric := func(n string) bool { return strings.HasPrefix(n, "signoz.") || strings.HasPrefix(n, "signoz_") }
-	externalMissingMetrics := make([]string, 0, len(missingMetrics))
-	for _, m := range missingMetrics {
-		if !isInternalMetric(m) {
-			externalMissingMetrics = append(externalMissingMetrics, m)
-		}
-	}
-	if len(externalMissingMetrics) > 0 {
-		lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, externalMissingMetrics...)
-		for _, missingMetricName := range externalMissingMetrics {
+	if len(missingMetrics) > 0 {
+		lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, missingMetrics...)
+		for _, missingMetricName := range missingMetrics {
 			if ts, ok := lastSeenInfo[missingMetricName]; ok && ts > 0 {
 				continue
 			}
@@ -448,22 +440,24 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 			return nil, errors.NewNotFoundf(errors.CodeNotFound, "the following metrics were not found: %s", strings.Join(nonExistentMetrics, ", "))
 		}
 		lastSeenStr := func(name string) string {
-			ts := lastSeenInfo[name]
-			ago := humanize.RelTime(time.UnixMilli(ts), time.Now(), "ago", "from now")
-			return fmt.Sprintf("%s (last seen %s)", name, ago)
+			if ts, ok := lastSeenInfo[name]; ok && ts > 0 {
+				ago := humanize.RelTime(time.UnixMilli(ts), time.Now(), "ago", "from now")
+				return fmt.Sprintf("%s (last seen %s)", name, ago)
+			}
+			return name // this case won't come cuz lastSeenStr is never called for metrics in nonExistentMetrics
 		}
-		if len(externalMissingMetrics) == 1 {
-			dormantMetricsWarningMsg = fmt.Sprintf("no data found for the metric %s in the query time range", lastSeenStr(externalMissingMetrics[0]))
+		if len(missingMetrics) == 1 {
+			dormantMetricsWarningMsg = fmt.Sprintf("no data found for the metric %s in the query time range", lastSeenStr(missingMetrics[0]))
 		} else {
-			parts := make([]string, len(externalMissingMetrics))
-			for i, m := range externalMissingMetrics {
+			parts := make([]string, len(missingMetrics))
+			for i, m := range missingMetrics {
 				parts[i] = lastSeenStr(m)
 			}
 			dormantMetricsWarningMsg = fmt.Sprintf("no data found for the following metrics in the query time range: %s", strings.Join(parts, ", "))
 		}
 	}
 	preseededResults := make(map[string]any)
-	for _, name := range missingMetricQueries {
+	for _, name := range missingMetricQueries { // at this point missing metrics will not have any non existent metrics, only normal ones
 		switch req.RequestType {
 		case qbtypes.RequestTypeTimeSeries:
 			preseededResults[name] = &qbtypes.TimeSeriesData{QueryName: name}

pkg/query-service/app/clickhouseReader/reader.go

@@ -1156,9 +1156,11 @@ func (r *ClickHouseReader) GetFlamegraphSpansForTrace(ctx context.Context, orgID
 		r.logger.Info("cache miss for getFlamegraphSpansForTrace", "traceID", traceID)
 
 		selectCols := "timestamp, duration_nano, span_id, trace_id, has_error, links as references, resource_string_service$$name, name, events"
-		if len(req.SelectFields) > 0 {
-			selectCols += ", attributes_string, attributes_number, attributes_bool, resources_string"
+		selectFieldCols := req.GetSelectedFieldsSourceColumns()
+		if len(selectFieldCols) > 0 {
+			selectCols = fmt.Sprintf("%s, %s", selectCols, strings.Join(selectFieldCols, ", "))
 		}
+
 		flamegraphQuery := fmt.Sprintf("SELECT %s FROM %s.%s WHERE trace_id=$1 and ts_bucket_start>=$2 and ts_bucket_start<=$3 ORDER BY timestamp ASC, name ASC", selectCols, r.TraceDB, r.traceTableName)
 
 		searchScanResponses, err := r.GetSpansForTrace(ctx, traceID, flamegraphQuery)

pkg/query-service/model/queryParams.go

@@ -346,6 +346,45 @@ type GetFlamegraphSpansForTraceParams struct {
 	SelectFields    []telemetrytypes.TelemetryFieldKey `json:"selectFields"`
 }
 
+func (r *GetFlamegraphSpansForTraceParams) GetSelectedFieldsSourceColumns() []string {
+	var needsAttrString, needsAttrNumber, needsAttrBool, needsResourceMap bool
+	for _, f := range r.SelectFields {
+		switch f.FieldContext {
+		case telemetrytypes.FieldContextAttribute:
+			switch f.FieldDataType {
+			case telemetrytypes.FieldDataTypeString:
+				needsAttrString = true
+			case telemetrytypes.FieldDataTypeFloat64, telemetrytypes.FieldDataTypeNumber, telemetrytypes.FieldDataTypeInt64:
+				needsAttrNumber = true
+			case telemetrytypes.FieldDataTypeBool:
+				needsAttrBool = true
+			default:
+				// Unknown type: AttributeValue searches all three maps, so we need all.
+				needsAttrString = true
+				needsAttrNumber = true
+				needsAttrBool = true
+			}
+		case telemetrytypes.FieldContextResource:
+			needsResourceMap = true
+		}
+	}
+
+	var cols []string
+	if needsAttrString {
+		cols = append(cols, "attributes_string")
+	}
+	if needsAttrNumber {
+		cols = append(cols, "attributes_number")
+	}
+	if needsAttrBool {
+		cols = append(cols, "attributes_bool")
+	}
+	if needsResourceMap {
+		cols = append(cols, "resources_string")
+	}
+	return cols
+}
+
 type SpanFilterParams struct {
 	TraceID            []string `json:"traceID"`
 	Status             []string `json:"status"`

tests/integration/tests/querier/03_metrics.py

@@ -640,32 +640,6 @@ def test_non_existent_metrics_returns_404(
     assert get_error_message(response.json()) == "could not find the metric whatevergoennnsgoeshere"
 
 
-def test_non_existent_internal_metrics_returns_no_warning(
-    signoz: types.SigNoz,
-    create_user_admin: None,  # pylint: disable=unused-argument
-    get_token: Callable[[str, str], str],
-) -> None:
-
-    now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
-    metric_name = "signoz_calls_total"
-
-    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
-    query = build_builder_query(
-        "A",
-        metric_name,
-        "doesnotreallymatter",
-        "sum",
-    )
-
-    end_ms = int(now.timestamp() * 1000)
-
-    start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
-    response = make_query_request(signoz, token, start_2h, end_ms, [query])
-    assert response.status_code == HTTPStatus.OK
-    data = response.json()
-    assert get_all_warnings(data) == []
-
-
 # Verify /api/v1/fields/values filters label values by metricNamespace prefix.
 # Inserts metrics under ns.a and ns.b, then asserts a specific prefix returns
 # only matching values while a common prefix returns both.