Merge branch 'main' into nv/4325

Co-authored-by: primus-bot[bot] <171087277+primus-bot[bot]@users.noreply.github.com>
Co-authored-by: Priyanshu Shrivastava <priyanshu@signoz.io>

deploy/docker-swarm/docker-compose.ha.yaml

@@ -190,7 +190,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:v0.117.1
+    image: signoz/signoz:v0.118.0
     ports:
       - "8080:8080" # signoz port
     #   - "6060:6060"     # pprof port

deploy/docker-swarm/docker-compose.yaml

@@ -117,7 +117,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:v0.117.1
+    image: signoz/signoz:v0.118.0
     ports:
       - "8080:8080" # signoz port
     volumes:

deploy/docker/docker-compose.ha.yaml

@@ -181,7 +181,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:${VERSION:-v0.117.1}
+    image: signoz/signoz:${VERSION:-v0.118.0}
     container_name: signoz
     ports:
       - "8080:8080" # signoz port

deploy/docker/docker-compose.yaml

@@ -109,7 +109,7 @@ services:
       # - ../common/clickhouse/storage.xml:/etc/clickhouse-server/config.d/storage.xml
   signoz:
     !!merge <<: *db-depend
-    image: signoz/signoz:${VERSION:-v0.117.1}
+    image: signoz/signoz:${VERSION:-v0.118.0}
     container_name: signoz
     ports:
       - "8080:8080" # signoz port

pkg/modules/serviceaccount/implserviceaccount/module.go

@@ -361,7 +361,7 @@ func (module *module) getOrGetSetIdentity(ctx context.Context, serviceAccountID
 		return identity, nil
 	}
 
-	storableServiceAccount, err := module.store.GetActiveByID(ctx, serviceAccountID)
+	storableServiceAccount, err := module.store.GetByID(ctx, serviceAccountID)
 	if err != nil {
 		return nil, err
 	}

pkg/modules/serviceaccount/implserviceaccount/store.go

@@ -105,24 +105,6 @@ func (store *store) GetByID(ctx context.Context, id valuer.UUID) (*serviceaccoun
 	return storable, nil
 }
 
-func (store *store) GetActiveByID(ctx context.Context, id valuer.UUID) (*serviceaccounttypes.ServiceAccount, error) {
-	storable := new(serviceaccounttypes.ServiceAccount)
-
-	err := store.
-		sqlstore.
-		BunDBCtx(ctx).
-		NewSelect().
-		Model(storable).
-		Where("id = ?", id).
-		Where("status = ?", serviceaccounttypes.ServiceAccountStatusActive).
-		Scan(ctx)
-	if err != nil {
-		return nil, store.sqlstore.WrapNotFoundErrf(err, serviceaccounttypes.ErrCodeServiceAccountNotFound, "service account with id: %s doesn't exist", id)
-	}
-
-	return storable, nil
-}
-
 func (store *store) CountByOrgID(ctx context.Context, orgID valuer.UUID) (int64, error) {
 	storable := new(serviceaccounttypes.ServiceAccount)
 

pkg/querier/querier.go

@@ -419,10 +419,12 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 	}
 	nonExistentMetrics := []string{}
 	var dormantMetricsWarningMsg string
+	// question: should we maintain a list of all internal metrics, cuz what if a user defines a metric with these prefixes?
+	isInternalMetric := func(n string) bool { return strings.HasPrefix(n, "signoz.") || strings.HasPrefix(n, "signoz_") }
 	if len(missingMetrics) > 0 {
 		lastSeenInfo, _ := q.metadataStore.FetchLastSeenInfoMulti(ctx, missingMetrics...)
 		for _, missingMetricName := range missingMetrics {
-			if ts, ok := lastSeenInfo[missingMetricName]; ok && ts > 0 {
+			if ts, ok := lastSeenInfo[missingMetricName]; (ok && ts > 0) || isInternalMetric(missingMetricName) {
 				continue
 			}
 			nonExistentMetrics = append(nonExistentMetrics, missingMetricName)
@@ -437,7 +439,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 				ago := humanize.RelTime(time.UnixMilli(ts), time.Now(), "ago", "from now")
 				return fmt.Sprintf("%s (last seen %s)", name, ago)
 			}
-			return name // this case won't come cuz lastSeenStr is never called for metrics in nonExistentMetrics
+			return name // this case will come only for internal metrics
 		}
 		if len(missingMetrics) == 1 {
 			dormantMetricsWarningMsg = fmt.Sprintf("no data found for the metric %s in the query time range", lastSeenStr(missingMetrics[0]))
@@ -450,7 +452,7 @@ func (q *querier) QueryRange(ctx context.Context, orgID valuer.UUID, req *qbtype
 		}
 	}
 	preseededResults := make(map[string]any)
-	for _, name := range missingMetricQueries { // at this point missing metrics will not have any non existent metrics, only normal ones
+	for _, name := range missingMetricQueries {
 		switch req.RequestType {
 		case qbtypes.RequestTypeTimeSeries:
 			preseededResults[name] = &qbtypes.TimeSeriesData{QueryName: name}

pkg/querybuilder/where_clause_visitor.go

@@ -818,9 +818,9 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
 			case "has":
 				cond = fmt.Sprintf("has(%s, %s)", fieldName, v.builder.Var(value[0]))
 			case "hasAny":
-				cond = fmt.Sprintf("hasAny(%s, %s)", fieldName, v.builder.Var(value))
+				cond = fmt.Sprintf("hasAny(%s, %s)", fieldName, v.builder.Var(value[0]))
 			case "hasAll":
-				cond = fmt.Sprintf("hasAll(%s, %s)", fieldName, v.builder.Var(value))
+				cond = fmt.Sprintf("hasAll(%s, %s)", fieldName, v.builder.Var(value[0]))
 			}
 			conds = append(conds, cond)
 		}

pkg/telemetrylogs/json_stmt_builder_test.go

@@ -631,7 +631,7 @@ func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
 			filter: "hasAll(body.user.permissions, ['read', 'write'])",
 			expected: TestExpected{
 				WhereClause: "hasAll(dynamicElement(body_v2.`user.permissions`, 'Array(Nullable(String))'), ?)",
-				Args:        []any{uint64(1747945619), uint64(1747983448), []any{[]any{"read", "write"}}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
+				Args:        []any{uint64(1747945619), uint64(1747983448), []any{"read", "write"}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 		},
 
@@ -757,7 +757,7 @@ func TestJSONStmtBuilder_ArrayPaths(t *testing.T) {
 			filter: "hasAny(education[].awards[].participated[].members, ['Piyush', 'Tushar'])",
 			expected: TestExpected{
 				WhereClause: "hasAny(arrayFlatten(arrayConcat(arrayMap(`body_v2.education`->arrayConcat(arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=4, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), dynamicElement(`body_v2.education`.`awards`, 'Array(JSON(max_dynamic_types=8, max_dynamic_paths=0))')), arrayMap(`body_v2.education[].awards`->arrayConcat(arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')), arrayMap(`body_v2.education[].awards[].participated`->dynamicElement(`body_v2.education[].awards[].participated`.`members`, 'Array(Nullable(String))'), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education[].awards`.`participated`, 'Array(Dynamic)'))))), arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), dynamicElement(`body_v2.education`.`awards`, 'Array(Dynamic)'))))), dynamicElement(body_v2.`education`, 'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))')))), ?)",
-				Args:        []any{uint64(1747945619), uint64(1747983448), []any{[]any{"Piyush", "Tushar"}}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
+				Args:        []any{uint64(1747945619), uint64(1747983448), []any{"Piyush", "Tushar"}, "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 		},
 		{

pkg/types/serviceaccounttypes/service_acccount.go

@@ -238,7 +238,6 @@ type Store interface {
 	Get(context.Context, valuer.UUID, valuer.UUID) (*ServiceAccount, error)
 	GetActiveByOrgIDAndName(context.Context, valuer.UUID, string) (*ServiceAccount, error)
 	GetByID(context.Context, valuer.UUID) (*ServiceAccount, error)
-	GetActiveByID(context.Context, valuer.UUID) (*ServiceAccount, error)
 	CountByOrgID(context.Context, valuer.UUID) (int64, error)
 	List(context.Context, valuer.UUID) ([]*ServiceAccount, error)
 	Update(context.Context, valuer.UUID, *ServiceAccount) error

tests/integration/src/querier/03_metrics.py

@@ -658,3 +658,33 @@ def test_non_existent_metrics_returns_404(
         get_error_message(response.json())
         == "could not find the metric whatevergoennnsgoeshere"
     )
+
+
+def test_non_existent__internal_metrics_returns_warning(
+    signoz: types.SigNoz,
+    create_user_admin: None,  # pylint: disable=unused-argument
+    get_token: Callable[[str, str], str],
+) -> None:
+
+    now = datetime.now(tz=timezone.utc).replace(second=0, microsecond=0)
+    metric_name = "signoz_calls_total"
+
+    token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
+    query = build_builder_query(
+        "A",
+        metric_name,
+        "doesnotreallymatter",
+        "sum",
+    )
+
+    end_ms = int(now.timestamp() * 1000)
+
+    start_2h = int((now - timedelta(hours=2)).timestamp() * 1000)
+    response = make_query_request(signoz, token, start_2h, end_ms, [query])
+    assert response.status_code == HTTPStatus.OK
+    data = response.json()
+    warnings = get_all_warnings(data)
+    assert len(warnings) == 1
+    assert warnings[0]["message"].startswith(
+        f"no data found for the metric {metric_name}"
+    )

tests/integration/src/serviceaccount/03_auth.py

@@ -245,9 +245,10 @@ def test_service_account_key_deleted_account_rejected(
         timeout=5,
     )
 
-    assert (
-        response.status_code == HTTPStatus.UNAUTHORIZED
-    ), f"Expected 401 for disabled service account, got {response.status_code}: {response.text}"
+    assert response.status_code in (
+        HTTPStatus.UNAUTHORIZED,
+        HTTPStatus.FORBIDDEN,
+    ), f"Expected 401/403 for disabled service account, got {response.status_code}: {response.text}"
 
 
 def test_service_account_key_revoked_key_rejected(

tests/integration/src/serviceaccount/05_me.py

@@ -54,8 +54,11 @@ def test_get_me_requires_sa_identity(
     )
 
     # user JWT has no service account ID in claims, should fail
-    assert (
-        response.status_code == HTTPStatus.NOT_FOUND
+    assert response.status_code in (
+        HTTPStatus.BAD_REQUEST,
+        HTTPStatus.FORBIDDEN,
+        HTTPStatus.NOT_FOUND,
+        HTTPStatus.UNAUTHORIZED,
     ), f"Expected error for user JWT on service account /me, got {response.status_code}: {response.text}"