mirror of
https://github.com/SigNoz/signoz.git
synced 2026-08-06 21:20:42 +01:00
Compare commits
2 Commits
test/semco
...
feat/semco
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
303d6a0363 | ||
|
|
6b4bb82efb |
8
Makefile
8
Makefile
@@ -224,6 +224,14 @@ py-test: ## Runs integration tests
|
||||
py-test-semconv-phase1: py-test-setup ## Rebuild the shared stack and run the semantic-convention Phase 1 matrix
|
||||
@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/tests/queriertraces/13_semconv_evolution.py
|
||||
|
||||
.PHONY: py-test-semconv-phase2
|
||||
py-test-semconv-phase2: py-test-setup ## Rebuild the shared stack and run the Phase 1-2 cross-signal matrices
|
||||
@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/tests/queriertraces/13_semconv_evolution.py integration/tests/queriersemconv/02_cross_signal.py
|
||||
|
||||
.PHONY: py-test-semconv-phase3
|
||||
py-test-semconv-phase3: py-test-setup ## Rebuild the shared stack and run the Phase 1-3 compatibility and migration-report matrices
|
||||
@cd tests && uv run pytest --basetemp=./tmp/ -vv --reuse --capture=no integration/tests/queriertraces/13_semconv_evolution.py integration/tests/queriersemconv/02_cross_signal.py
|
||||
|
||||
.PHONY: py-clean
|
||||
py-clean: ## Clear all pycache and pytest cache from tests directory recursively
|
||||
@echo ">> cleaning python cache files from tests directory"
|
||||
|
||||
9
frontend/src/api/semconv/getMigrationReport.ts
Normal file
9
frontend/src/api/semconv/getMigrationReport.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import axios from 'api';
|
||||
import { SemconvMigrationReport } from 'types/api/semconvMigration';
|
||||
|
||||
async function getSemconvMigrationReport(): Promise<SemconvMigrationReport> {
|
||||
const response = await axios.get('/fields/semconv-migration');
|
||||
return response.data.data;
|
||||
}
|
||||
|
||||
export default getSemconvMigrationReport;
|
||||
@@ -5,7 +5,7 @@ import { ArrowUpRight } from '@signozhq/icons';
|
||||
|
||||
const QUICK_FILTER_DOC_PATHS: Record<string, string> = {
|
||||
severity_text: 'severity-text',
|
||||
'deployment.environment': 'environment',
|
||||
'deployment.environment.name': 'environment',
|
||||
'service.name': 'service-name',
|
||||
'host.name': 'hostname',
|
||||
'k8s.cluster.name': 'k8s-cluster-name',
|
||||
|
||||
32
frontend/src/components/Semconv/SemconvEditorWarning.tsx
Normal file
32
frontend/src/components/Semconv/SemconvEditorWarning.tsx
Normal file
@@ -0,0 +1,32 @@
|
||||
import { Alert } from 'antd';
|
||||
import { findOldSemconvNames } from 'utils/semconv';
|
||||
|
||||
interface SemconvEditorWarningProps {
|
||||
value: unknown;
|
||||
editor: string;
|
||||
}
|
||||
|
||||
function SemconvEditorWarning({
|
||||
value,
|
||||
editor,
|
||||
}: SemconvEditorWarningProps): JSX.Element | null {
|
||||
const text = typeof value === 'string' ? value : JSON.stringify(value ?? '');
|
||||
const renames = findOldSemconvNames(text);
|
||||
if (renames.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Alert
|
||||
type="warning"
|
||||
showIcon
|
||||
data-testid="semconv-editor-warning"
|
||||
message={`${editor} contains renamed OpenTelemetry fields`}
|
||||
description={renames
|
||||
.map(({ old, current }) => `${old} → ${current}`)
|
||||
.join(', ')}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
export default SemconvEditorWarning;
|
||||
23
frontend/src/components/Semconv/SemconvOldNameBadge.tsx
Normal file
23
frontend/src/components/Semconv/SemconvOldNameBadge.tsx
Normal file
@@ -0,0 +1,23 @@
|
||||
import { Badge } from '@signozhq/ui/badge';
|
||||
import { getSemconvRename } from 'utils/semconv';
|
||||
|
||||
interface SemconvOldNameBadgeProps {
|
||||
name: string;
|
||||
}
|
||||
|
||||
function SemconvOldNameBadge({
|
||||
name,
|
||||
}: SemconvOldNameBadgeProps): JSX.Element | null {
|
||||
const rename = getSemconvRename(name);
|
||||
if (!rename || rename.family.kind !== 'attribute') {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<Badge color="amber" variant="outline" data-testid="semconv-old-name-badge">
|
||||
old name, renamed to {rename.current}
|
||||
</Badge>
|
||||
);
|
||||
}
|
||||
|
||||
export default SemconvOldNameBadge;
|
||||
39
frontend/src/components/Semconv/__tests__/Semconv.test.tsx
Normal file
39
frontend/src/components/Semconv/__tests__/Semconv.test.tsx
Normal file
@@ -0,0 +1,39 @@
|
||||
import { render, screen } from '@testing-library/react';
|
||||
|
||||
import SemconvEditorWarning from '../SemconvEditorWarning';
|
||||
import SemconvOldNameBadge from '../SemconvOldNameBadge';
|
||||
|
||||
describe('semantic convention product hints', () => {
|
||||
it('badges an old raw attribute with its current name', () => {
|
||||
render(<SemconvOldNameBadge name="deployment.environment" />);
|
||||
|
||||
expect(screen.getByTestId('semconv-old-name-badge')).toHaveTextContent(
|
||||
'old name, renamed to deployment.environment.name',
|
||||
);
|
||||
});
|
||||
|
||||
it('does not badge a current raw attribute', () => {
|
||||
render(<SemconvOldNameBadge name="deployment.environment.name" />);
|
||||
|
||||
expect(
|
||||
screen.queryByTestId('semconv-old-name-badge'),
|
||||
).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('shows an informational editor warning without disabling the editor', () => {
|
||||
render(
|
||||
<>
|
||||
<input aria-label="query" defaultValue="db.system = 'postgresql'" />
|
||||
<SemconvEditorWarning
|
||||
value="db.system = 'postgresql'"
|
||||
editor="ClickHouse SQL"
|
||||
/>
|
||||
</>,
|
||||
);
|
||||
|
||||
expect(screen.getByLabelText('query')).not.toBeDisabled();
|
||||
expect(screen.getByTestId('semconv-editor-warning')).toHaveTextContent(
|
||||
'db.system → db.system.name',
|
||||
);
|
||||
});
|
||||
});
|
||||
2
frontend/src/components/Semconv/index.ts
Normal file
2
frontend/src/components/Semconv/index.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export { default as SemconvEditorWarning } from './SemconvEditorWarning';
|
||||
export { default as SemconvOldNameBadge } from './SemconvOldNameBadge';
|
||||
@@ -11,12 +11,21 @@ export type SemconvFamily = {
|
||||
};
|
||||
|
||||
export const SEMCONV_FAMILIES: readonly SemconvFamily[] = [
|
||||
{
|
||||
current: 'container.cpu.usage',
|
||||
old: ['container.cpu.utilization'],
|
||||
kind: 'metric',
|
||||
contexts: ['metric'],
|
||||
signals: ['metrics'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
{
|
||||
current: 'db.system.name',
|
||||
old: ['db.system'],
|
||||
kind: 'attribute',
|
||||
contexts: [],
|
||||
signals: [],
|
||||
contexts: ['attribute', 'resource'],
|
||||
signals: ['logs', 'metrics', 'traces'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
@@ -24,8 +33,26 @@ export const SEMCONV_FAMILIES: readonly SemconvFamily[] = [
|
||||
current: 'deployment.environment.name',
|
||||
old: ['deployment.environment'],
|
||||
kind: 'attribute',
|
||||
contexts: [],
|
||||
signals: [],
|
||||
contexts: ['attribute', 'resource'],
|
||||
signals: ['logs', 'metrics', 'traces'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
{
|
||||
current: 'k8s.node.cpu.usage',
|
||||
old: ['k8s.node.cpu.utilization'],
|
||||
kind: 'metric',
|
||||
contexts: ['metric'],
|
||||
signals: ['metrics'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
{
|
||||
current: 'k8s.pod.cpu.usage',
|
||||
old: ['k8s.pod.cpu.utilization'],
|
||||
kind: 'metric',
|
||||
contexts: ['metric'],
|
||||
signals: ['metrics'],
|
||||
applyToMetrics: [],
|
||||
valueMap: {},
|
||||
},
|
||||
|
||||
@@ -155,7 +155,7 @@ function DomainList(): JSX.Element {
|
||||
dataSource={DataSource.TRACES}
|
||||
queryData={query}
|
||||
onChange={handleSearchChange}
|
||||
placeholder="Enter your filter query (e.g., deployment.environment = 'otel-demo' AND service.name = 'frontend')"
|
||||
placeholder="Enter your filter query (e.g., deployment.environment.name = 'otel-demo' AND service.name = 'frontend')"
|
||||
hardcodedAttributeKeys={ApiMonitoringHardcodedAttributeKeys}
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -6,9 +6,9 @@ import { SPAN_ATTRIBUTES } from './Explorer/Domains/DomainDetails/constants';
|
||||
export const ApiMonitoringHardcodedAttributeKeys: QueryKeyDataSuggestionsProps[] =
|
||||
[
|
||||
{
|
||||
label: 'deployment.environment',
|
||||
label: 'deployment.environment.name',
|
||||
type: 'resource',
|
||||
name: 'deployment.environment',
|
||||
name: 'deployment.environment.name',
|
||||
signal: 'traces',
|
||||
fieldDataType: QUERY_BUILDER_KEY_TYPES.STRING,
|
||||
},
|
||||
|
||||
@@ -87,7 +87,7 @@ export const ApiMonitoringQuickFiltersConfig: IQuickFiltersConfig[] = [
|
||||
title: 'Environment',
|
||||
|
||||
attributeKey: {
|
||||
key: 'deployment.environment',
|
||||
key: 'deployment.environment.name',
|
||||
dataType: DataTypes.String,
|
||||
type: 'resource',
|
||||
},
|
||||
|
||||
@@ -112,7 +112,7 @@ export const INFRA_MONITORING_ATTR_KEYS = {
|
||||
K8S_OBJECT_NAME: 'k8s.object.name',
|
||||
|
||||
// Environment
|
||||
DEPLOYMENT_ENVIRONMENT: 'deployment.environment',
|
||||
DEPLOYMENT_ENVIRONMENT: 'deployment.environment.name',
|
||||
|
||||
// Host System
|
||||
OS_TYPE: 'os.type',
|
||||
@@ -733,7 +733,7 @@ export const ENTITY_FILTER_PLACEHOLDERS: Record<InfraMonitoringEntity, string> =
|
||||
[InfraMonitoringEntity.NAMESPACES]:
|
||||
"Enter your filter query (e.g., k8s.namespace.name = 'production' AND k8s.cluster.name = 'prod-cluster')",
|
||||
[InfraMonitoringEntity.CLUSTERS]:
|
||||
"Enter your filter query (e.g., k8s.cluster.name = 'prod-cluster' AND deployment.environment = 'production')",
|
||||
"Enter your filter query (e.g., k8s.cluster.name = 'prod-cluster' AND deployment.environment.name = 'production')",
|
||||
[InfraMonitoringEntity.DEPLOYMENTS]:
|
||||
"Enter your filter query (e.g., k8s.deployment.name = 'api-server' AND k8s.namespace.name = 'production')",
|
||||
[InfraMonitoringEntity.STATEFULSETS]:
|
||||
|
||||
@@ -2,6 +2,17 @@
|
||||
color: white;
|
||||
}
|
||||
|
||||
.semconv-migration-report {
|
||||
margin-top: 32px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
|
||||
.ant-table-wrapper {
|
||||
margin-top: 4px;
|
||||
}
|
||||
}
|
||||
|
||||
.ingestion-key-container {
|
||||
margin-top: 24px;
|
||||
display: flex;
|
||||
|
||||
@@ -5,6 +5,8 @@ import getIngestionData from 'api/settings/getIngestionData';
|
||||
import { useAppContext } from 'providers/App/App';
|
||||
import { IngestionDataType } from 'types/api/settings/ingestion';
|
||||
|
||||
import SemconvMigrationReport from './SemconvMigrationReport';
|
||||
|
||||
import './IngestionSettings.styles.scss';
|
||||
|
||||
export default function IngestionSettings(): JSX.Element {
|
||||
@@ -84,6 +86,7 @@ export default function IngestionSettings(): JSX.Element {
|
||||
dataSource={data}
|
||||
bordered
|
||||
/>
|
||||
<SemconvMigrationReport />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -83,6 +83,8 @@ import { MeterAggregateOperator } from 'types/common/queryBuilder';
|
||||
import { USER_ROLES } from 'types/roles';
|
||||
import { getDaysUntilExpiry } from 'utils/timeUtils';
|
||||
|
||||
import SemconvMigrationReport from './SemconvMigrationReport';
|
||||
|
||||
import './IngestionSettings.styles.scss';
|
||||
|
||||
const { Option } = Select;
|
||||
@@ -1705,6 +1707,7 @@ function MultiIngestionSettings(): JSX.Element {
|
||||
}}
|
||||
className="ingestion-keys-table"
|
||||
/>
|
||||
<SemconvMigrationReport />
|
||||
</div>
|
||||
|
||||
{/* Delete Key Modal */}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import { useQuery } from 'react-query';
|
||||
import { Alert, Table, TableColumnsType } from 'antd';
|
||||
import { Typography } from '@signozhq/ui/typography';
|
||||
import getSemconvMigrationReport from 'api/semconv/getMigrationReport';
|
||||
import dayjs from 'dayjs';
|
||||
import { SemconvMigrationReportEntry } from 'types/api/semconvMigration';
|
||||
|
||||
function SemconvMigrationReport(): JSX.Element {
|
||||
const { data, isLoading, isError } = useQuery({
|
||||
queryKey: ['semconv-migration-report'],
|
||||
queryFn: getSemconvMigrationReport,
|
||||
});
|
||||
|
||||
const columns: TableColumnsType<SemconvMigrationReportEntry> = [
|
||||
{
|
||||
title: 'Old name',
|
||||
dataIndex: 'old',
|
||||
key: 'old',
|
||||
},
|
||||
{
|
||||
title: 'Current name',
|
||||
dataIndex: 'current',
|
||||
key: 'current',
|
||||
},
|
||||
{
|
||||
title: 'Signal',
|
||||
dataIndex: 'signal',
|
||||
key: 'signal',
|
||||
},
|
||||
{
|
||||
title: 'Services still sending only the old name',
|
||||
dataIndex: 'services',
|
||||
key: 'services',
|
||||
render: (services: string[]): string => services.join(', '),
|
||||
},
|
||||
{
|
||||
title: 'Last seen',
|
||||
dataIndex: 'lastSeenUnixMilli',
|
||||
key: 'lastSeenUnixMilli',
|
||||
render: (value: number): string =>
|
||||
dayjs(value).format('YYYY-MM-DD HH:mm:ss'),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<section className="semconv-migration-report">
|
||||
<Typography.Title level={4}>Semantic convention migration</Typography.Title>
|
||||
<Typography.Text>
|
||||
Services in this report sent an old OpenTelemetry field during the last 24
|
||||
hours without sending its current replacement. Update their SDK or
|
||||
instrumentation when practical; SigNoz queries remain backward compatible.
|
||||
</Typography.Text>
|
||||
{isError && (
|
||||
<Alert
|
||||
type="error"
|
||||
showIcon
|
||||
message="Could not load the semantic convention migration report"
|
||||
/>
|
||||
)}
|
||||
<Table
|
||||
loading={isLoading}
|
||||
columns={columns}
|
||||
dataSource={data?.entries ?? []}
|
||||
rowKey={(entry): string => `${entry.current}-${entry.old}-${entry.signal}`}
|
||||
pagination={false}
|
||||
locale={{ emptyText: 'No old-only services found in the last 24 hours' }}
|
||||
/>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export default SemconvMigrationReport;
|
||||
@@ -15,7 +15,7 @@ export const SAMPLE_SPAN_JSON = `{
|
||||
},
|
||||
"resource": {
|
||||
"service.name": "llm-gateway",
|
||||
"deployment.environment": "production"
|
||||
"deployment.environment.name": "production"
|
||||
}
|
||||
}`;
|
||||
|
||||
|
||||
@@ -1120,7 +1120,7 @@
|
||||
"plugin": {
|
||||
"kind": "signoz/QueryVariable",
|
||||
"spec": {
|
||||
"queryValue": "SELECT DISTINCT resources_string['deployment.environment'] AS environment FROM signoz_traces.distributed_signoz_index_v3 WHERE mapContains(resources_string, 'deployment.environment') AND timestamp >= now() - INTERVAL 1 DAY"
|
||||
"queryValue": "SELECT DISTINCT resources_string['deployment.environment.name'] AS environment FROM signoz_traces.distributed_signoz_index_v3 WHERE mapContains(resources_string, 'deployment.environment.name') AND timestamp >= now() - INTERVAL 1 DAY"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { Divider } from '@signozhq/ui/divider';
|
||||
import { TooltipSimple } from '@signozhq/ui/tooltip';
|
||||
import { Typography } from '@signozhq/ui/typography';
|
||||
import { SemconvOldNameBadge } from 'components/Semconv';
|
||||
|
||||
import { TagContainer, TagLabel, TagValue } from './FieldRenderer.styles';
|
||||
import { FieldRendererProps } from './LogDetailedView.types';
|
||||
@@ -28,6 +29,7 @@ function FieldRenderer({ field }: FieldRendererProps): JSX.Element {
|
||||
<Typography.Text truncate={1} className="label">
|
||||
{newField}{' '}
|
||||
</Typography.Text>
|
||||
<SemconvOldNameBadge name={newField} />
|
||||
</TooltipSimple>
|
||||
|
||||
<div className="tags">
|
||||
@@ -47,7 +49,10 @@ function FieldRenderer({ field }: FieldRendererProps): JSX.Element {
|
||||
</div>
|
||||
</>
|
||||
) : (
|
||||
<span className="label">{field}</span>
|
||||
<>
|
||||
<span className="label">{field}</span>
|
||||
<SemconvOldNameBadge name={field} />
|
||||
</>
|
||||
)}
|
||||
</span>
|
||||
);
|
||||
|
||||
@@ -164,7 +164,9 @@ describe('useInitialQuery - Priority-Based Resource Filtering', () => {
|
||||
value: 'frontend-service',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
key: expect.objectContaining({ key: 'deployment.environment' }),
|
||||
key: expect.objectContaining({
|
||||
key: 'deployment.environment.name',
|
||||
}),
|
||||
value: 'production',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
@@ -286,7 +288,9 @@ describe('useInitialQuery - Priority-Based Resource Filtering', () => {
|
||||
value: 'legacy-app',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
key: expect.objectContaining({ key: 'deployment.environment' }),
|
||||
key: expect.objectContaining({
|
||||
key: 'deployment.environment.name',
|
||||
}),
|
||||
value: 'production',
|
||||
}),
|
||||
expect.objectContaining({
|
||||
|
||||
@@ -6,13 +6,14 @@ import {
|
||||
TagFilterItem,
|
||||
} from 'types/api/queryBuilder/queryBuilderData';
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import { getSemconvRename } from 'utils/semconv';
|
||||
|
||||
const FALLBACK_STARTS_WITH_REGEX = /^(k8s|cloud|host|deployment)/; // regex to filter out resources that start with the specified keywords
|
||||
const FALLBACK_CONTAINS_REGEX = /(env|service|file|container|tenant)/; // regex to filter out resources that contains the specified keywords
|
||||
|
||||
// Priority categories for filter selection
|
||||
// Strategy:
|
||||
// - Always include: service.name, deployment.environment, env, environment
|
||||
// - Always include: service.name, deployment.environment.name, env, environment
|
||||
// - Select ONE category only: stops at the first category with a matching attribute
|
||||
// - Within category: picks the first available attribute by order
|
||||
// - Order (highest to lowest priority): Kubernetes > Cloud > Host > Container
|
||||
@@ -26,27 +27,36 @@ const PRIORITY_CATEGORIES = [
|
||||
|
||||
const SERVICE_AND_ENVIRONMENT_KEYS = [
|
||||
'service.name',
|
||||
'deployment.environment',
|
||||
'deployment.environment.name',
|
||||
'env',
|
||||
'environment',
|
||||
];
|
||||
|
||||
export const getFiltersFromResources = (
|
||||
resources: ILog['resources_string'],
|
||||
): TagFilterItem[] =>
|
||||
Object.keys(resources).map((key: string) => {
|
||||
): TagFilterItem[] => {
|
||||
const items = new Map<string, TagFilterItem>();
|
||||
Object.keys(resources).forEach((key: string) => {
|
||||
const currentKey = getSemconvRename(key)?.current ?? key;
|
||||
const resourceValue = resources[key] as string;
|
||||
return {
|
||||
const item = {
|
||||
id: uuid(),
|
||||
key: {
|
||||
key,
|
||||
key: currentKey,
|
||||
dataType: DataTypes.String,
|
||||
type: 'resource',
|
||||
},
|
||||
op: OPERATORS['='],
|
||||
value: resourceValue,
|
||||
};
|
||||
// If raw data contains both names, retain the current value just like the
|
||||
// backend's current-first resolver.
|
||||
if (!items.has(currentKey) || key === currentKey) {
|
||||
items.set(currentKey, item);
|
||||
}
|
||||
});
|
||||
return Array.from(items.values());
|
||||
};
|
||||
|
||||
export const isServiceOrEnvironmentAttribute = (key: string): boolean =>
|
||||
SERVICE_AND_ENVIRONMENT_KEYS.includes(key);
|
||||
|
||||
@@ -94,7 +94,7 @@ function DBCall(): JSX.Element {
|
||||
featureFlags?.find((flag) => flag.name === FeatureKeys.DOT_METRICS_ENABLED)
|
||||
?.active || false;
|
||||
|
||||
const legend = dotMetricsEnabled ? '{{db.system}}' : '{{db_system}}';
|
||||
const legend = dotMetricsEnabled ? '{{db.system.name}}' : '{{db_system_name}}';
|
||||
|
||||
const databaseCallsRPSWidget = useMemo(
|
||||
() =>
|
||||
|
||||
@@ -28,7 +28,7 @@ import { v4 as uuid } from 'uuid';
|
||||
|
||||
export const dbSystemTags: Tags[] = [
|
||||
{
|
||||
Key: 'db.system.(string)',
|
||||
Key: 'db.system.name.(string)',
|
||||
StringValues: [''],
|
||||
NumberValues: [],
|
||||
BoolValues: [],
|
||||
|
||||
@@ -103,7 +103,7 @@ export enum WidgetKeys {
|
||||
SignozExternalCallLatencySum = 'signoz_external_call_latency_sum',
|
||||
Signoz_latency_bucket_norm = 'signoz_latency_bucket',
|
||||
Signoz_latency_bucket = 'signoz_latency.bucket',
|
||||
Db_system = 'db.system',
|
||||
Db_system = 'db.system.name',
|
||||
Db_system_norm = 'db_system',
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { ChangeEvent, useCallback } from 'react';
|
||||
import MEditor, { Monaco } from '@monaco-editor/react';
|
||||
import { Color } from '@signozhq/design-tokens';
|
||||
import { Input } from 'antd';
|
||||
import { SemconvEditorWarning } from 'components/Semconv';
|
||||
import { LEGEND } from 'constants/global';
|
||||
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
|
||||
import { useIsDarkMode } from 'hooks/useDarkMode';
|
||||
@@ -118,6 +119,7 @@ function ClickHouseQueryBuilder({
|
||||
theme={isDarkMode ? 'my-theme' : 'light'}
|
||||
beforeMount={setEditorTheme}
|
||||
/>
|
||||
<SemconvEditorWarning value={queryData?.query} editor="ClickHouse SQL" />
|
||||
<Input
|
||||
onChange={handleUpdateInput}
|
||||
name="legend"
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { ChangeEvent, useCallback } from 'react';
|
||||
import { Input } from 'antd';
|
||||
import { LEGEND } from 'constants/global';
|
||||
import { SemconvEditorWarning } from 'components/Semconv';
|
||||
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
|
||||
import { IPromQLQuery } from 'types/api/queryBuilder/queryBuilderData';
|
||||
import { EQueryType } from 'types/common/dashboard';
|
||||
@@ -66,6 +67,7 @@ function PromQLQueryBuilder({
|
||||
style={{ marginBottom: '0.5rem' }}
|
||||
data-testid="promql-query-input"
|
||||
/>
|
||||
<SemconvEditorWarning value={queryData?.query} editor="PromQL" />
|
||||
|
||||
<Input
|
||||
onChange={handleUpdateQuery}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Form } from 'antd';
|
||||
import { initialQueryBuilderFormValuesMap } from 'constants/queryBuilder';
|
||||
import { SemconvEditorWarning } from 'components/Semconv';
|
||||
import QueryBuilderSearchV2 from 'container/QueryBuilder/filters/QueryBuilderSearchV2/QueryBuilderSearchV2';
|
||||
import isEqual from 'lodash-es/isEqual';
|
||||
import { TagFilter } from 'types/api/queryBuilder/queryBuilderData';
|
||||
@@ -55,6 +56,7 @@ function TagFilterInputWithLogsResultPreview({
|
||||
value={value}
|
||||
onChange={onChange}
|
||||
/>
|
||||
<SemconvEditorWarning value={value} editor="Pipeline filter" />
|
||||
<div className="pipeline-filter-input-preview-container">
|
||||
<LogsFilterPreview filter={value} />
|
||||
</div>
|
||||
|
||||
@@ -424,7 +424,7 @@ describe('ResourceProvider', () => {
|
||||
await waitFor(() => {
|
||||
expect(result.current.queries).toHaveLength(1);
|
||||
expect(result.current.queries[0]).toMatchObject({
|
||||
tagKey: 'resource_deployment_environment',
|
||||
tagKey: 'resource_deployment_environment_name',
|
||||
operator: 'IN',
|
||||
tagValue: ['production'],
|
||||
});
|
||||
@@ -435,7 +435,7 @@ describe('ResourceProvider', () => {
|
||||
const seeded = [
|
||||
{
|
||||
id: 'env',
|
||||
tagKey: 'resource_deployment_environment',
|
||||
tagKey: 'resource_deployment_environment_name',
|
||||
operator: 'IN',
|
||||
tagValue: ['production'],
|
||||
},
|
||||
@@ -459,7 +459,7 @@ describe('ResourceProvider', () => {
|
||||
|
||||
await waitFor(() => {
|
||||
const tagKeys = result.current.queries.map((q) => q.tagKey);
|
||||
expect(tagKeys).not.toContain('resource_deployment_environment');
|
||||
expect(tagKeys).not.toContain('resource_deployment_environment_name');
|
||||
expect(tagKeys).toContain('resource_service_name');
|
||||
});
|
||||
});
|
||||
@@ -468,7 +468,7 @@ describe('ResourceProvider', () => {
|
||||
const seeded = [
|
||||
{
|
||||
id: 'env',
|
||||
tagKey: 'resource_deployment_environment',
|
||||
tagKey: 'resource_deployment_environment_name',
|
||||
operator: 'IN',
|
||||
tagValue: ['production'],
|
||||
},
|
||||
@@ -486,7 +486,7 @@ describe('ResourceProvider', () => {
|
||||
|
||||
await waitFor(() => {
|
||||
const envQueries = result.current.queries.filter(
|
||||
(q) => q.tagKey === 'resource_deployment_environment',
|
||||
(q) => q.tagKey === 'resource_deployment_environment_name',
|
||||
);
|
||||
expect(envQueries).toHaveLength(1);
|
||||
expect(envQueries[0].tagValue).toStrictEqual(['staging']);
|
||||
@@ -518,7 +518,7 @@ describe('ResourceProvider', () => {
|
||||
|
||||
await waitFor(() => {
|
||||
expect(result.current.queries[0].tagKey).toBe(
|
||||
'resource_deployment.environment',
|
||||
'resource_deployment.environment.name',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,13 +6,13 @@ import { mappingWithRoutesAndKeys } from '../utils';
|
||||
describe('useResourceAttribute config', () => {
|
||||
describe('whilelistedKeys', () => {
|
||||
it('should include underscore-notation keys (DOT_METRICS_ENABLED=false)', () => {
|
||||
expect(whilelistedKeys).toContain('resource_deployment_environment');
|
||||
expect(whilelistedKeys).toContain('resource_deployment_environment_name');
|
||||
expect(whilelistedKeys).toContain('resource_k8s_cluster_name');
|
||||
expect(whilelistedKeys).toContain('resource_k8s_cluster_namespace');
|
||||
});
|
||||
|
||||
it('should include dot-notation keys (DOT_METRICS_ENABLED=true)', () => {
|
||||
expect(whilelistedKeys).toContain('resource_deployment.environment');
|
||||
expect(whilelistedKeys).toContain('resource_deployment.environment.name');
|
||||
expect(whilelistedKeys).toContain('resource_k8s.cluster.name');
|
||||
expect(whilelistedKeys).toContain('resource_k8s.cluster.namespace');
|
||||
});
|
||||
@@ -21,8 +21,8 @@ describe('useResourceAttribute config', () => {
|
||||
describe('mappingWithRoutesAndKeys', () => {
|
||||
const dotNotationFilters = [
|
||||
{
|
||||
label: 'deployment.environment',
|
||||
value: 'resource_deployment.environment',
|
||||
label: 'deployment.environment.name',
|
||||
value: 'resource_deployment.environment.name',
|
||||
},
|
||||
{ label: 'k8s.cluster.name', value: 'resource_k8s.cluster.name' },
|
||||
{ label: 'k8s.cluster.namespace', value: 'resource_k8s.cluster.namespace' },
|
||||
@@ -30,8 +30,8 @@ describe('useResourceAttribute config', () => {
|
||||
|
||||
const underscoreNotationFilters = [
|
||||
{
|
||||
label: 'deployment.environment',
|
||||
value: 'resource_deployment_environment',
|
||||
label: 'deployment.environment.name',
|
||||
value: 'resource_deployment_environment_name',
|
||||
},
|
||||
{ label: 'k8s.cluster.name', value: 'resource_k8s_cluster_name' },
|
||||
{ label: 'k8s.cluster.namespace', value: 'resource_k8s_cluster_namespace' },
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
export const whilelistedKeys = [
|
||||
'resource_deployment_environment',
|
||||
'resource_deployment.environment',
|
||||
'resource_deployment_environment_name',
|
||||
'resource_deployment.environment.name',
|
||||
'resource_k8s_cluster_name',
|
||||
'resource_k8s.cluster.name',
|
||||
'resource_k8s_cluster_namespace',
|
||||
|
||||
@@ -148,9 +148,9 @@ export const getResourceDeploymentKeys = (
|
||||
dotMetricsEnabled: boolean,
|
||||
): string => {
|
||||
if (dotMetricsEnabled) {
|
||||
return 'resource_deployment.environment';
|
||||
return 'resource_deployment.environment.name';
|
||||
}
|
||||
return 'resource_deployment_environment';
|
||||
return 'resource_deployment_environment_name';
|
||||
};
|
||||
|
||||
export const GetTagKeys = async (
|
||||
|
||||
@@ -40,7 +40,7 @@ export const LogsQuickFiltersConfig: IQuickFiltersConfig[] = [
|
||||
type: FiltersType.CHECKBOX,
|
||||
title: 'Environment',
|
||||
attributeKey: {
|
||||
key: 'deployment.environment',
|
||||
key: 'deployment.environment.name',
|
||||
dataType: DataTypes.String,
|
||||
type: 'resource',
|
||||
},
|
||||
|
||||
@@ -11,6 +11,7 @@ import { Skeleton } from 'antd';
|
||||
import { DetailsHeader, DetailsPanelDrawer } from 'components/DetailsPanel';
|
||||
import { HeaderAction } from 'components/DetailsPanel/DetailsHeader/DetailsHeader';
|
||||
import { DetailsPanelState } from 'components/DetailsPanel/types';
|
||||
import { SemconvOldNameBadge } from 'components/Semconv';
|
||||
import { QueryParams } from 'constants/query';
|
||||
import {
|
||||
initialQueryBuilderFormValuesMap,
|
||||
@@ -108,6 +109,12 @@ function SpanDetailsContent({
|
||||
() => getSpanDisplayData(selectedSpan),
|
||||
[selectedSpan],
|
||||
);
|
||||
const semconvLabelSuffix = useCallback(
|
||||
(fieldKey: string): React.ReactNode => (
|
||||
<SemconvOldNameBadge name={fieldKey} />
|
||||
),
|
||||
[],
|
||||
);
|
||||
|
||||
// Map span attribute actions to PrettyView actions format.
|
||||
// Use the last key in fieldKeyPath (the actual attribute key), not the full display path.
|
||||
@@ -329,6 +336,7 @@ function SpanDetailsContent({
|
||||
visibleActions: VISIBLE_ACTIONS,
|
||||
pinnedFieldsValue,
|
||||
onPinnedFieldsChange,
|
||||
labelSuffixRenderer: semconvLabelSuffix,
|
||||
}}
|
||||
/>
|
||||
</TabsContent>
|
||||
|
||||
@@ -29,7 +29,7 @@ export const KEY_ATTRIBUTE_KEYS: Record<string, string[]> = {
|
||||
traces: [
|
||||
'service.name',
|
||||
'service.namespace',
|
||||
'deployment.environment',
|
||||
'deployment.environment.name',
|
||||
'timestamp',
|
||||
'duration_nano',
|
||||
'kind_string',
|
||||
|
||||
@@ -22,7 +22,7 @@ export const SPAN_CATEGORIES: readonly SpanCategory[] = [
|
||||
|
||||
// Map each category to the attribute key it filters on
|
||||
const CATEGORY_KEYS: Record<Exclude<SpanCategory, 'All'>, string> = {
|
||||
Database: 'db.system',
|
||||
Database: 'db.system.name',
|
||||
HTTP: 'http.method',
|
||||
Functions: 'kind_string',
|
||||
Jobs: 'messaging.system',
|
||||
@@ -34,7 +34,7 @@ const ALL_CATEGORY_KEYS = Object.values(CATEGORY_KEYS);
|
||||
|
||||
// The expression clause to add for each category
|
||||
const CATEGORY_EXPRESSIONS: Record<Exclude<SpanCategory, 'All'>, string> = {
|
||||
Database: 'db.system exists',
|
||||
Database: 'db.system.name exists',
|
||||
HTTP: 'http.method exists',
|
||||
Functions: "kind_string = 'Internal'",
|
||||
Jobs: 'messaging.system exists',
|
||||
|
||||
@@ -38,7 +38,7 @@ export function Section(props: SectionProps): JSX.Element {
|
||||
'hasError',
|
||||
'durationNano',
|
||||
'serviceName',
|
||||
'deployment.environment',
|
||||
'deployment.environment.name',
|
||||
]),
|
||||
),
|
||||
[selectedFilters],
|
||||
|
||||
@@ -14,7 +14,7 @@ export const AllTraceFilterKeyValue: Record<string, string> = {
|
||||
durationNano: 'Duration',
|
||||
duration_nano: 'Duration',
|
||||
durationNanoMax: 'Duration',
|
||||
'deployment.environment': 'Environment',
|
||||
'deployment.environment.name': 'Environment',
|
||||
hasError: 'Status',
|
||||
has_error: 'Status',
|
||||
serviceName: 'Service Name',
|
||||
@@ -208,11 +208,11 @@ export const traceFilterKeys: Record<AllTraceFilterKeys, BaseAutocompleteData> =
|
||||
id: 'serviceName--string--tag--true',
|
||||
},
|
||||
|
||||
'deployment.environment': {
|
||||
key: 'deployment.environment',
|
||||
'deployment.environment.name': {
|
||||
key: 'deployment.environment.name',
|
||||
dataType: DataTypes.String,
|
||||
type: 'resource',
|
||||
id: 'deployment.environment--string--resource--false',
|
||||
id: 'deployment.environment.name--string--resource--false',
|
||||
},
|
||||
name: {
|
||||
key: 'name',
|
||||
|
||||
@@ -223,6 +223,12 @@
|
||||
padding-left: 6px !important;
|
||||
}
|
||||
|
||||
&__label {
|
||||
display: inline-flex;
|
||||
align-items: baseline;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
&__pinned-icon {
|
||||
flex-shrink: 0;
|
||||
color: var(--text-robin-400);
|
||||
|
||||
@@ -67,6 +67,7 @@ export interface PrettyViewProps {
|
||||
*/
|
||||
pinnedFieldsValue?: string[];
|
||||
onPinnedFieldsChange?: (next: string[]) => void;
|
||||
labelSuffixRenderer?: (fieldKey: string) => React.ReactNode;
|
||||
}
|
||||
|
||||
function PrettyView({
|
||||
@@ -78,6 +79,7 @@ function PrettyView({
|
||||
drawerKey = 'default',
|
||||
pinnedFieldsValue,
|
||||
onPinnedFieldsChange,
|
||||
labelSuffixRenderer,
|
||||
}: PrettyViewProps): JSX.Element {
|
||||
const isDarkMode = useIsDarkMode();
|
||||
const [, setCopy] = useCopyToClipboard();
|
||||
@@ -305,10 +307,24 @@ function PrettyView({
|
||||
}}
|
||||
/>
|
||||
<span>{displayKey}</span>
|
||||
{labelSuffixRenderer?.(displayKey)}
|
||||
</span>
|
||||
);
|
||||
},
|
||||
[togglePin, pinnedEntries],
|
||||
[togglePin, pinnedEntries, labelSuffixRenderer],
|
||||
);
|
||||
|
||||
const labelRenderer = useCallback(
|
||||
(keyPath: KeyPath): React.ReactNode => {
|
||||
const displayKey = String(keyPath[0]);
|
||||
return (
|
||||
<span className="pretty-view__label">
|
||||
<span>{displayKey}</span>
|
||||
{labelSuffixRenderer?.(displayKey)}
|
||||
</span>
|
||||
);
|
||||
},
|
||||
[labelSuffixRenderer],
|
||||
);
|
||||
|
||||
return (
|
||||
@@ -351,6 +367,7 @@ function PrettyView({
|
||||
shouldExpandNodeInitially={shouldExpandNodeInitially}
|
||||
valueRenderer={valueRenderer}
|
||||
getItemString={getItemString}
|
||||
labelRenderer={labelRenderer}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
|
||||
14
frontend/src/types/api/semconvMigration.ts
Normal file
14
frontend/src/types/api/semconvMigration.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
export interface SemconvMigrationReportEntry {
|
||||
current: string;
|
||||
old: string;
|
||||
signal: string;
|
||||
services: string[];
|
||||
resourceSets: number;
|
||||
lastSeenUnixMilli: number;
|
||||
}
|
||||
|
||||
export interface SemconvMigrationReport {
|
||||
startUnixMilli: number;
|
||||
endUnixMilli: number;
|
||||
entries: SemconvMigrationReportEntry[];
|
||||
}
|
||||
29
frontend/src/utils/__tests__/semconv.test.ts
Normal file
29
frontend/src/utils/__tests__/semconv.test.ts
Normal file
@@ -0,0 +1,29 @@
|
||||
import { findOldSemconvNames, getSemconvRename } from 'utils/semconv';
|
||||
|
||||
describe('semantic convention helpers', () => {
|
||||
it('returns the current name for an old attribute', () => {
|
||||
expect(getSemconvRename('deployment.environment')).toMatchObject({
|
||||
old: 'deployment.environment',
|
||||
current: 'deployment.environment.name',
|
||||
});
|
||||
});
|
||||
|
||||
it('finds old names in editor text without matching larger custom names', () => {
|
||||
expect(
|
||||
findOldSemconvNames(
|
||||
"deployment.environment = 'prod' AND custom.db.system.value = 'x'",
|
||||
),
|
||||
).toStrictEqual([
|
||||
expect.objectContaining({
|
||||
old: 'deployment.environment',
|
||||
current: 'deployment.environment.name',
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not warn for current names', () => {
|
||||
expect(
|
||||
findOldSemconvNames('deployment.environment.name = prod'),
|
||||
).toStrictEqual([]);
|
||||
});
|
||||
});
|
||||
50
frontend/src/utils/semconv.ts
Normal file
50
frontend/src/utils/semconv.ts
Normal file
@@ -0,0 +1,50 @@
|
||||
import {
|
||||
SEMCONV_FAMILIES,
|
||||
SemconvFamily,
|
||||
} from 'constants/generated/semconvFamilies.gen';
|
||||
|
||||
export type SemconvRename = {
|
||||
old: string;
|
||||
current: string;
|
||||
family: SemconvFamily;
|
||||
};
|
||||
|
||||
const OLD_NAMES = SEMCONV_FAMILIES.flatMap((family) =>
|
||||
family.old.map((old) => ({ old, current: family.current, family })),
|
||||
);
|
||||
|
||||
const OLD_NAME_INDEX = new Map(OLD_NAMES.map((rename) => [rename.old, rename]));
|
||||
|
||||
export function getSemconvRename(name: string): SemconvRename | undefined {
|
||||
return OLD_NAME_INDEX.get(name);
|
||||
}
|
||||
|
||||
export function findOldSemconvNames(text: string): SemconvRename[] {
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return OLD_NAMES.filter(({ old }) => containsSemconvName(text, old));
|
||||
}
|
||||
|
||||
function containsSemconvName(text: string, name: string): boolean {
|
||||
let offset = 0;
|
||||
while (offset < text.length) {
|
||||
const index = text.indexOf(name, offset);
|
||||
if (index === -1) {
|
||||
return false;
|
||||
}
|
||||
const before = index === 0 ? '' : text[index - 1];
|
||||
const afterIndex = index + name.length;
|
||||
const after = afterIndex === text.length ? '' : text[afterIndex];
|
||||
if (!isSemconvNameCharacter(before) && !isSemconvNameCharacter(after)) {
|
||||
return true;
|
||||
}
|
||||
offset = index + 1;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isSemconvNameCharacter(value: string): boolean {
|
||||
return /[A-Za-z0-9_.-]/.test(value);
|
||||
}
|
||||
@@ -46,5 +46,23 @@ func (provider *provider) addFieldsRoutes(router *mux.Router) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/fields/semconv-migration", handler.New(provider.authzMiddleware.ViewAccess(provider.fieldsHandler.GetSemconvMigrationReport), handler.OpenAPIDef{
|
||||
ID: "GetSemconvMigrationReport",
|
||||
Tags: []string{"fields"},
|
||||
Summary: "Get semantic-convention migration report",
|
||||
Description: "Returns services that still emit old semantic-convention names without the current family name",
|
||||
Request: nil,
|
||||
RequestQuery: new(telemetrytypes.PostableSemconvMigrationReportParams),
|
||||
RequestContentType: "",
|
||||
Response: new(telemetrytypes.GettableSemconvMigrationReport),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{},
|
||||
Deprecated: false,
|
||||
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
|
||||
})).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -8,4 +8,7 @@ type Handler interface {
|
||||
|
||||
// Gets the fields values for the given field value selector
|
||||
GetFieldsValues(http.ResponseWriter, *http.Request)
|
||||
|
||||
// Gets services that still emit only historical semantic-convention names.
|
||||
GetSemconvMigrationReport(http.ResponseWriter, *http.Request)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package implfields
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/http/binding"
|
||||
@@ -16,6 +17,43 @@ type handler struct {
|
||||
telemetryMetadataStore telemetrytypes.MetadataStore
|
||||
}
|
||||
|
||||
func (handler *handler) GetSemconvMigrationReport(rw http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
|
||||
var params telemetrytypes.PostableSemconvMigrationReportParams
|
||||
if err := binding.Query.BindQuery(req.URL.Query(), ¶ms); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
if params.EndUnixMilli == 0 {
|
||||
params.EndUnixMilli = now.UnixMilli()
|
||||
}
|
||||
if params.StartUnixMilli == 0 {
|
||||
params.StartUnixMilli = now.Add(-24 * time.Hour).UnixMilli()
|
||||
}
|
||||
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
report, err := handler.telemetryMetadataStore.GetSemconvMigrationReport(
|
||||
ctx,
|
||||
valuer.MustNewUUID(claims.OrgID),
|
||||
params.StartUnixMilli,
|
||||
params.EndUnixMilli,
|
||||
)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, report)
|
||||
}
|
||||
|
||||
func NewHandler(settings factory.ProviderSettings, telemetryMetadataStore telemetrytypes.MetadataStore) fields.Handler {
|
||||
return &handler{
|
||||
telemetryMetadataStore: telemetryMetadataStore,
|
||||
|
||||
@@ -786,10 +786,11 @@ func (q *querier) run(
|
||||
Results: maps.Values(processedResults),
|
||||
},
|
||||
Meta: qbtypes.ExecStats{
|
||||
RowsScanned: stats.RowsScanned,
|
||||
BytesScanned: stats.BytesScanned,
|
||||
DurationMS: stats.DurationMS,
|
||||
StepIntervals: stepIntervals,
|
||||
RowsScanned: stats.RowsScanned,
|
||||
BytesScanned: stats.BytesScanned,
|
||||
DurationMS: stats.DurationMS,
|
||||
StepIntervals: stepIntervals,
|
||||
SemconvResolutions: semconvResolutionsForRequest(req),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
128
pkg/querier/semconv_resolutions.go
Normal file
128
pkg/querier/semconv_resolutions.go
Normal file
@@ -0,0 +1,128 @@
|
||||
package querier
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
// semconvResolutionsForRequest reports only query-builder resolutions. Raw
|
||||
// ClickHouse SQL and PromQL are deliberately excluded: SigNoz does not rewrite
|
||||
// those languages and their editors surface non-blocking warnings instead.
|
||||
func semconvResolutionsForRequest(req *qbtypes.QueryRangeRequest) []qbtypes.SemconvResolution {
|
||||
if req == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
resolutions := make([]qbtypes.SemconvResolution, 0)
|
||||
seen := make(map[string]struct{})
|
||||
for _, envelope := range req.CompositeQuery.Queries {
|
||||
signal, applies := semconvResolutionSignal(envelope.Spec)
|
||||
if !applies {
|
||||
continue
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(envelope.Spec)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
text := string(payload)
|
||||
|
||||
for _, family := range semconv.All() {
|
||||
if _, ok := semconv.Lookup(family.Kind, telemetrytypes.FieldKeySelector{
|
||||
Name: family.Current,
|
||||
Signal: signal,
|
||||
}); !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, requested := range semconvRequestSpellings(family, signal) {
|
||||
if !containsSemconvName(text, requested) {
|
||||
continue
|
||||
}
|
||||
identity := family.Kind.StringValue() + "\x00" + requested + "\x00" + family.Current
|
||||
if _, ok := seen[identity]; ok {
|
||||
continue
|
||||
}
|
||||
seen[identity] = struct{}{}
|
||||
resolutions = append(resolutions, qbtypes.SemconvResolution{
|
||||
Requested: requested,
|
||||
Current: family.Current,
|
||||
Members: append([]string{family.Current}, family.Old...),
|
||||
Kind: family.Kind.StringValue(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(resolutions) == 0 {
|
||||
return nil
|
||||
}
|
||||
return resolutions
|
||||
}
|
||||
|
||||
func semconvResolutionSignal(spec any) (telemetrytypes.Signal, bool) {
|
||||
switch spec.(type) {
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation], qbtypes.QueryBuilderTraceOperator:
|
||||
return telemetrytypes.SignalTraces, true
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]:
|
||||
return telemetrytypes.SignalLogs, true
|
||||
case qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]:
|
||||
return telemetrytypes.SignalMetrics, true
|
||||
case qbtypes.QueryBuilderJoin:
|
||||
// Joins can contain more than one signal. An unspecified signal keeps
|
||||
// family signal scopes in force while allowing every applicable family.
|
||||
return telemetrytypes.SignalUnspecified, true
|
||||
default:
|
||||
return telemetrytypes.SignalUnspecified, false
|
||||
}
|
||||
}
|
||||
|
||||
func semconvRequestSpellings(family semconv.Family, signal telemetrytypes.Signal) []string {
|
||||
spellings := append([]string{family.Current}, family.Old...)
|
||||
if signal != telemetrytypes.SignalMetrics {
|
||||
return spellings
|
||||
}
|
||||
|
||||
logical := slices.Clone(spellings)
|
||||
for _, name := range logical {
|
||||
normalized := strings.ReplaceAll(name, ".", "_")
|
||||
if !slices.Contains(spellings, normalized) {
|
||||
spellings = append(spellings, normalized)
|
||||
}
|
||||
if family.Kind == semconv.KindAttribute {
|
||||
for _, resourceName := range []string{"resource_" + name, "resource_" + normalized} {
|
||||
if !slices.Contains(spellings, resourceName) {
|
||||
spellings = append(spellings, resourceName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return spellings
|
||||
}
|
||||
|
||||
func containsSemconvName(text, name string) bool {
|
||||
for offset := 0; offset < len(text); {
|
||||
index := strings.Index(text[offset:], name)
|
||||
if index < 0 {
|
||||
return false
|
||||
}
|
||||
start := offset + index
|
||||
end := start + len(name)
|
||||
if (start == 0 || !isSemconvNameRune(rune(text[start-1]))) &&
|
||||
(end == len(text) || !isSemconvNameRune(rune(text[end]))) {
|
||||
return true
|
||||
}
|
||||
offset = start + 1
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isSemconvNameRune(r rune) bool {
|
||||
return unicode.IsLetter(r) || unicode.IsDigit(r) || r == '_' || r == '.' || r == '-'
|
||||
}
|
||||
70
pkg/querier/semconv_resolutions_test.go
Normal file
70
pkg/querier/semconv_resolutions_test.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package querier
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestSemconvResolutionsReportsOldTraceAttribute(t *testing.T) {
|
||||
req := &qbtypes.QueryRangeRequest{
|
||||
CompositeQuery: qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{{
|
||||
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Filter: &qbtypes.Filter{Expression: "deployment.environment = 'prod'"},
|
||||
},
|
||||
}}},
|
||||
}
|
||||
|
||||
assert.Equal(t, []qbtypes.SemconvResolution{{
|
||||
Requested: "deployment.environment",
|
||||
Current: "deployment.environment.name",
|
||||
Members: []string{"deployment.environment.name", "deployment.environment"},
|
||||
Kind: "attribute",
|
||||
}}, semconvResolutionsForRequest(req), "old trace attribute should be reported as a family resolution")
|
||||
}
|
||||
|
||||
func TestSemconvResolutionsReportsCurrentLogAttribute(t *testing.T) {
|
||||
req := &qbtypes.QueryRangeRequest{
|
||||
CompositeQuery: qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{{
|
||||
Spec: qbtypes.QueryBuilderQuery[qbtypes.LogAggregation]{
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
SelectFields: []telemetrytypes.TelemetryFieldKey{{
|
||||
Name: "db.system.name",
|
||||
}},
|
||||
},
|
||||
}}},
|
||||
}
|
||||
|
||||
assert.Equal(t, []qbtypes.SemconvResolution{{
|
||||
Requested: "db.system.name",
|
||||
Current: "db.system.name",
|
||||
Members: []string{"db.system.name", "db.system"},
|
||||
Kind: "attribute",
|
||||
}}, semconvResolutionsForRequest(req), "current log attribute should identify its complete family")
|
||||
}
|
||||
|
||||
func TestSemconvResolutionsIgnoresRawSQL(t *testing.T) {
|
||||
req := &qbtypes.QueryRangeRequest{
|
||||
CompositeQuery: qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{{
|
||||
Spec: qbtypes.ClickHouseQuery{Query: "SELECT attributes_string['deployment.environment']"},
|
||||
}}},
|
||||
}
|
||||
|
||||
assert.Empty(t, semconvResolutionsForRequest(req), "raw SQL is not rewritten and should not report a resolution")
|
||||
}
|
||||
|
||||
func TestSemconvResolutionsRequiresNameBoundary(t *testing.T) {
|
||||
req := &qbtypes.QueryRangeRequest{
|
||||
CompositeQuery: qbtypes.CompositeQuery{Queries: []qbtypes.QueryEnvelope{{
|
||||
Spec: qbtypes.QueryBuilderQuery[qbtypes.TraceAggregation]{
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Filter: &qbtypes.Filter{Expression: "custom.deployment.environment = 'prod'"},
|
||||
},
|
||||
}}},
|
||||
}
|
||||
|
||||
assert.Empty(t, semconvResolutionsForRequest(req), "a family name embedded in a larger custom key must not match")
|
||||
}
|
||||
@@ -52,10 +52,10 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/constants"
|
||||
|
||||
chErrors "github.com/SigNoz/signoz/pkg/query-service/errors"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/metrics"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/model"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -3190,11 +3190,6 @@ func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, orgID v
|
||||
var rows driver.Rows
|
||||
var attributeValues v3.FilterAttributeValueResponse
|
||||
|
||||
normalized := true
|
||||
if constants.IsDotMetricsEnabled {
|
||||
normalized = false
|
||||
}
|
||||
|
||||
reductionEnabled := r.fl.BooleanOrEmpty(ctx, flagger.FeatureEnableMetricsReduction, featuretypes.NewFlaggerEvaluationContext(orgID))
|
||||
|
||||
if reductionEnabled {
|
||||
@@ -3205,8 +3200,7 @@ func (r *ClickHouseReader) GetMetricAttributeValues(ctx context.Context, orgID v
|
||||
if req.Limit != 0 {
|
||||
query = query + fmt.Sprintf(" LIMIT %d;", req.Limit)
|
||||
}
|
||||
names := []string{req.AggregateAttribute}
|
||||
names = append(names, metrics.GetTransitionedMetric(req.AggregateAttribute, normalized))
|
||||
names := semconv.MetricNames(req.AggregateAttribute)
|
||||
|
||||
rows, err = r.db.Query(ctx, query, req.FilterAttributeKey, names, req.FilterAttributeKey, fmt.Sprintf("%%%s%%", req.SearchText), common.PastDayRoundOff())
|
||||
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
package metrics
|
||||
|
||||
var MetricsUnderTransition = map[string]string{
|
||||
"k8s_pod_cpu_utilization": "k8s_pod_cpu_usage",
|
||||
"k8s_node_cpu_utilization": "k8s_node_cpu_usage",
|
||||
"container_cpu_utilization": "container_cpu_usage",
|
||||
}
|
||||
|
||||
var DotMetricsUnderTransition = map[string]string{
|
||||
"k8s.pod.cpu.utilization": "k8s.pod.cpu.usage",
|
||||
"k8s.node.cpu.utilization": "k8s.node.cpu.usage",
|
||||
"container.cpu.utilization": "container.cpu.usage",
|
||||
}
|
||||
|
||||
func GetTransitionedMetric(metric string, normalized bool) string {
|
||||
if normalized {
|
||||
if _, ok := MetricsUnderTransition[metric]; ok {
|
||||
return MetricsUnderTransition[metric]
|
||||
}
|
||||
return metric
|
||||
} else {
|
||||
if _, ok := DotMetricsUnderTransition[metric]; ok {
|
||||
return DotMetricsUnderTransition[metric]
|
||||
}
|
||||
return metric
|
||||
}
|
||||
}
|
||||
@@ -10,8 +10,8 @@ import (
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/constants"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/metrics"
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
)
|
||||
|
||||
// ValidateAndCastValue validates and casts the value of a key to the corresponding data type of the key
|
||||
@@ -234,12 +234,12 @@ func ClickHouseFormattedValue(v interface{}) string {
|
||||
|
||||
func ClickHouseFormattedMetricNames(v interface{}) string {
|
||||
if name, ok := v.(string); ok {
|
||||
transitionedMetrics := metrics.GetTransitionedMetric(name, !constants.IsDotMetricsEnabled)
|
||||
if transitionedMetrics != name {
|
||||
return ClickHouseFormattedValue([]interface{}{transitionedMetrics})
|
||||
} else {
|
||||
return ClickHouseFormattedValue([]interface{}{name})
|
||||
members := semconv.MetricNames(name)
|
||||
values := make([]interface{}, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, member)
|
||||
}
|
||||
return ClickHouseFormattedValue(values)
|
||||
}
|
||||
|
||||
return ClickHouseFormattedValue(v)
|
||||
|
||||
@@ -2,13 +2,29 @@ package querybuilder
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
|
||||
func physicalSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
|
||||
return []string{key.Name}
|
||||
}
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: key.Signal,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
// ExistsExpression renders the existence predicate for a key resolved to the given
|
||||
// columns (negated when exists is false). Comparisons are against constants rendered
|
||||
// as literals, so the expression carries no bind args and can guard column expressions
|
||||
@@ -43,11 +59,26 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
|
||||
if len(evolutionsEntries) > 0 && evolutionsEntries[0] != nil {
|
||||
columnName = evolutionsEntries[0].ColumnName
|
||||
}
|
||||
rawPath := fmt.Sprintf("%s.`%s`", columnName, key.Name)
|
||||
if exists {
|
||||
return rawPath + " IS NOT NULL", nil
|
||||
members := physicalSemconvMembers(key)
|
||||
paths := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
paths = append(paths, fmt.Sprintf("%s.`%s`", columnName, member))
|
||||
}
|
||||
return rawPath + " IS NULL", nil
|
||||
if len(paths) == 1 {
|
||||
if exists {
|
||||
return paths[0] + " IS NOT NULL", nil
|
||||
}
|
||||
return paths[0] + " IS NULL", nil
|
||||
}
|
||||
guards := make([]string, 0, len(paths))
|
||||
for _, path := range paths {
|
||||
guards = append(guards, path+" IS NOT NULL")
|
||||
}
|
||||
rawPath := "(" + strings.Join(guards, " OR ") + ")"
|
||||
if exists {
|
||||
return rawPath, nil
|
||||
}
|
||||
return "NOT " + rawPath, nil
|
||||
case schema.ColumnTypeEnumString,
|
||||
schema.ColumnTypeEnumFixedString:
|
||||
if exists {
|
||||
@@ -88,8 +119,16 @@ func ExistsExpression(columns []*schema.Column, key *telemetrytypes.TelemetryFie
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
leftOperand := fmt.Sprintf("mapContains(%s, '%s')", column.Name, key.Name)
|
||||
if key.Materialized {
|
||||
members := physicalSemconvMembers(key)
|
||||
operands := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
operands = append(operands, fmt.Sprintf("mapContains(%s, '%s')", column.Name, member))
|
||||
}
|
||||
leftOperand := strings.Join(operands, " OR ")
|
||||
if len(operands) > 1 {
|
||||
leftOperand = "(" + leftOperand + ")"
|
||||
}
|
||||
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
|
||||
leftOperand = telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
|
||||
}
|
||||
if exists {
|
||||
|
||||
@@ -39,6 +39,7 @@ type filterExpressionVisitor struct {
|
||||
fullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
skipResourceFilter bool
|
||||
skipFullTextFilter bool
|
||||
exactSemconv bool
|
||||
variables map[string]qbtypes.VariableItem
|
||||
|
||||
keysWithWarnings map[string]bool
|
||||
@@ -59,6 +60,7 @@ type FilterExprVisitorOpts struct {
|
||||
FullTextColumn *telemetrytypes.TelemetryFieldKey
|
||||
SkipResourceFilter bool
|
||||
SkipFullTextFilter bool
|
||||
ExactSemconv bool
|
||||
Variables map[string]qbtypes.VariableItem
|
||||
StartNs uint64
|
||||
EndNs uint64
|
||||
@@ -76,6 +78,7 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
|
||||
fullTextColumn: opts.FullTextColumn,
|
||||
skipResourceFilter: opts.SkipResourceFilter,
|
||||
skipFullTextFilter: opts.SkipFullTextFilter,
|
||||
exactSemconv: opts.ExactSemconv,
|
||||
variables: opts.Variables,
|
||||
keysWithWarnings: make(map[string]bool),
|
||||
startNs: opts.StartNs,
|
||||
@@ -380,7 +383,7 @@ func (v *filterExpressionVisitor) VisitPrimary(ctx *grammar.PrimaryContext) any
|
||||
// VisitComparison handles all comparison operators.
|
||||
func (v *filterExpressionVisitor) VisitComparison(ctx *grammar.ComparisonContext) any {
|
||||
key := v.Visit(ctx.Key()).(*telemetrytypes.TelemetryFieldKey)
|
||||
matching := MatchingFieldKeys(key, v.fieldKeys)
|
||||
matching := v.matchingFieldKeys(key)
|
||||
|
||||
// Handle EXISTS specially
|
||||
if ctx.EXISTS() != nil {
|
||||
@@ -731,7 +734,7 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
|
||||
conds, ok := v.buildConditions(key, MatchingFieldKeys(key, v.fieldKeys), operator, value)
|
||||
conds, ok := v.buildConditions(key, v.matchingFieldKeys(key), operator, value)
|
||||
if !ok {
|
||||
return ErrorConditionLiteral
|
||||
}
|
||||
@@ -924,7 +927,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
|
||||
// buildConditions invokes the condition builder for a filter term, folding its
|
||||
// warnings/errors into visitor state; returns false if an error was recorded.
|
||||
func (v *filterExpressionVisitor) buildConditions(key *telemetrytypes.TelemetryFieldKey, matching []*telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator, value any) ([]string, bool) {
|
||||
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter}, op, value, v.builder)
|
||||
conds, warns, err := v.conditionBuilder.ConditionFor(v.context, v.orgID, v.startNs, v.endNs, key, v.fieldKeys, qbtypes.ConditionBuilderOptions{SkipResourceFilter: v.skipResourceFilter, ExactSemconv: v.exactSemconv}, op, value, v.builder)
|
||||
if err != nil {
|
||||
_, _, _, _, errURL, _ := errors.Unwrapb(err)
|
||||
assignIfEmpty(&v.mainErrorURL, errURL)
|
||||
@@ -983,12 +986,24 @@ func assignIfEmpty(s *string, value string) {
|
||||
// MatchingFieldKeys returns the field keys from the map that match the given key,
|
||||
// honoring any context/data type the user specified.
|
||||
func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
return matchingFieldKeys(field, fieldKeys, true)
|
||||
}
|
||||
|
||||
// MatchingFieldKeysExact matches only the requested physical spelling.
|
||||
func MatchingFieldKeysExact(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
return matchingFieldKeys(field, fieldKeys, false)
|
||||
}
|
||||
|
||||
func matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey, resolveSemconv bool) []*telemetrytypes.TelemetryFieldKey {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: field.Signal,
|
||||
FieldContext: field.FieldContext,
|
||||
}
|
||||
members := semconv.Members(semconv.KindAttribute, selector)
|
||||
members := []string{field.Name}
|
||||
if resolveSemconv {
|
||||
members = semconv.AttributeMembers(selector)
|
||||
}
|
||||
isFamily := len(members) > 1
|
||||
fieldKeysForName := make([]*telemetrytypes.TelemetryFieldKey, 0)
|
||||
indexByIdentity := make(map[string]int)
|
||||
@@ -1005,13 +1020,13 @@ func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[st
|
||||
// A wildcard lookup may have found a same-named field in a scope where
|
||||
// this family does not apply. Keep exact names, but reject cross-member
|
||||
// matches outside the generated family scope.
|
||||
if memberName != field.Name {
|
||||
if resolveSemconv && memberName != field.Name {
|
||||
itemSelector := telemetrytypes.FieldKeySelector{
|
||||
Name: field.Name,
|
||||
Signal: item.Signal,
|
||||
FieldContext: item.FieldContext,
|
||||
}
|
||||
if !slices.Contains(semconv.Members(semconv.KindAttribute, itemSelector), memberName) {
|
||||
if !slices.Contains(semconv.AttributeMembers(itemSelector), memberName) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
@@ -1038,6 +1053,8 @@ func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[st
|
||||
if isFamily {
|
||||
resolved.Name = field.Name
|
||||
resolved.SemconvMembers = slices.Clone(physicalMembers)
|
||||
} else if !resolveSemconv {
|
||||
resolved.SemconvMembers = []string{field.Name}
|
||||
}
|
||||
fieldKeysForName = append(fieldKeysForName, &resolved)
|
||||
}
|
||||
@@ -1060,3 +1077,25 @@ func MatchingFieldKeys(field *telemetrytypes.TelemetryFieldKey, fieldKeys map[st
|
||||
|
||||
return fieldKeysForName
|
||||
}
|
||||
|
||||
func (v *filterExpressionVisitor) matchingFieldKeys(field *telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
if v.exactSemconv {
|
||||
return MatchingFieldKeysExact(field, v.fieldKeys)
|
||||
}
|
||||
return MatchingFieldKeys(field, v.fieldKeys)
|
||||
}
|
||||
|
||||
// ExactSemconvKeys returns copies pinned to their physical names, preventing a
|
||||
// field mapper from expanding a synthesized or metadata-free key into a family.
|
||||
func ExactSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
result := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
if key == nil {
|
||||
continue
|
||||
}
|
||||
resolved := *key
|
||||
resolved.SemconvMembers = []string{key.Name}
|
||||
result = append(result, &resolved)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -731,6 +731,11 @@ func TestMatchingFieldKeysResolvesSemconvFamily(t *testing.T) {
|
||||
assert.Equal(t, current.Name, matches[0].Name)
|
||||
assert.Equal(t, "old metadata", matches[0].Description)
|
||||
assert.Equal(t, []string{old.Name}, matches[0].SemconvMembers)
|
||||
|
||||
exactMatches := MatchingFieldKeysExact(requested, fieldKeys)
|
||||
require.Len(t, exactMatches, 1, "exact lookup must return one field before its metadata is inspected")
|
||||
assert.Equal(t, current.Name, exactMatches[0].Name)
|
||||
assert.Equal(t, []string{current.Name}, exactMatches[0].SemconvMembers)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -2,21 +2,47 @@
|
||||
|
||||
package semconv
|
||||
|
||||
import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
|
||||
var families = []Family{
|
||||
{
|
||||
Current: "container.cpu.usage",
|
||||
Old: []string{"container.cpu.utilization"},
|
||||
Kind: KindMetric,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "db.system.name",
|
||||
Old: []string{"db.system"},
|
||||
Kind: KindAttribute,
|
||||
Contexts: nil,
|
||||
Signals: nil,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalLogs, telemetrytypes.SignalMetrics, telemetrytypes.SignalTraces},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "deployment.environment.name",
|
||||
Old: []string{"deployment.environment"},
|
||||
Kind: KindAttribute,
|
||||
Contexts: nil,
|
||||
Signals: nil,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextAttribute, telemetrytypes.FieldContextResource},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalLogs, telemetrytypes.SignalMetrics, telemetrytypes.SignalTraces},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "k8s.node.cpu.usage",
|
||||
Old: []string{"k8s.node.cpu.utilization"},
|
||||
Kind: KindMetric,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
{
|
||||
Current: "k8s.pod.cpu.usage",
|
||||
Old: []string{"k8s.pod.cpu.utilization"},
|
||||
Kind: KindMetric,
|
||||
Contexts: []telemetrytypes.FieldContext{telemetrytypes.FieldContextMetric},
|
||||
Signals: []telemetrytypes.Signal{telemetrytypes.SignalMetrics},
|
||||
ApplyToMetrics: nil,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package semconv
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -26,6 +27,13 @@ type Family struct {
|
||||
ValueMap map[string]string
|
||||
}
|
||||
|
||||
type metricSpelling uint8
|
||||
|
||||
const (
|
||||
metricSpellingDotted metricSpelling = iota
|
||||
metricSpellingNormalized
|
||||
)
|
||||
|
||||
var (
|
||||
KindAttribute = Kind{String: valuer.NewString("attribute")}
|
||||
KindMetric = Kind{String: valuer.NewString("metric")}
|
||||
@@ -60,6 +68,88 @@ func Members(kind Kind, selector telemetrytypes.FieldKeySelector) []string {
|
||||
return familyMembers[idx]
|
||||
}
|
||||
|
||||
// AttributeMembers returns the physical attribute spellings that may represent
|
||||
// selector.Name. Metrics have used both dotted and normalized label layouts;
|
||||
// resource labels have additionally used a resource_ prefix. Keeping that
|
||||
// storage detail here prevents metrics readers from maintaining local
|
||||
// transition tables.
|
||||
func AttributeMembers(selector telemetrytypes.FieldKeySelector) []string {
|
||||
if selector.Signal != telemetrytypes.SignalMetrics {
|
||||
return Members(KindAttribute, selector)
|
||||
}
|
||||
|
||||
lookupSelector := selector
|
||||
lookupSelector.Name = strings.TrimPrefix(selector.Name, "resource_")
|
||||
family, style, ok := lookupMetricSpelling(KindAttribute, lookupSelector)
|
||||
if !ok {
|
||||
return []string{selector.Name}
|
||||
}
|
||||
|
||||
logicalMembers := familyMembers[family]
|
||||
result := make([]string, 0, len(logicalMembers)*4)
|
||||
for _, member := range logicalMembers {
|
||||
dotted := member
|
||||
normalized := normalizeMetricSpelling(member)
|
||||
variants := []string{dotted, normalized}
|
||||
if style == metricSpellingNormalized {
|
||||
variants[0], variants[1] = variants[1], variants[0]
|
||||
}
|
||||
|
||||
if selector.FieldContext == telemetrytypes.FieldContextResource ||
|
||||
selector.FieldContext == telemetrytypes.FieldContextUnspecified ||
|
||||
strings.HasPrefix(selector.Name, "resource_") {
|
||||
for _, variant := range variants {
|
||||
result = appendUniqueString(result, "resource_"+variant)
|
||||
}
|
||||
}
|
||||
for _, variant := range variants {
|
||||
result = appendUniqueString(result, variant)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// MetricNames returns the current and historical storage names for a metric.
|
||||
// The input's dotted or normalized style is preserved because both layouts are
|
||||
// valid metric identities and must not be mixed in one query.
|
||||
func MetricNames(name string) []string {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextMetric,
|
||||
}
|
||||
family, style, ok := lookupMetricSpelling(KindMetric, selector)
|
||||
if !ok {
|
||||
return []string{name}
|
||||
}
|
||||
|
||||
logicalMembers := familyMembers[family]
|
||||
result := make([]string, 0, len(logicalMembers))
|
||||
for _, member := range logicalMembers {
|
||||
if style == metricSpellingNormalized {
|
||||
member = normalizeMetricSpelling(member)
|
||||
}
|
||||
result = appendUniqueString(result, member)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// CurrentAttribute returns the canonical dotted name for an attribute
|
||||
// spelling, or selector.Name if no enabled family matches.
|
||||
func CurrentAttribute(selector telemetrytypes.FieldKeySelector) string {
|
||||
if selector.Signal != telemetrytypes.SignalMetrics {
|
||||
return Current(KindAttribute, selector)
|
||||
}
|
||||
|
||||
lookupSelector := selector
|
||||
lookupSelector.Name = strings.TrimPrefix(selector.Name, "resource_")
|
||||
family, _, ok := lookupMetricSpelling(KindAttribute, lookupSelector)
|
||||
if !ok {
|
||||
return selector.Name
|
||||
}
|
||||
return families[family].Current
|
||||
}
|
||||
|
||||
// Current returns the current name for selector.Name, or the input name when
|
||||
// it does not belong to an enabled family.
|
||||
func Current(kind Kind, selector telemetrytypes.FieldKeySelector) string {
|
||||
@@ -103,6 +193,35 @@ func lookupIndex(kind Kind, selector telemetrytypes.FieldKeySelector) (int, bool
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func lookupMetricSpelling(kind Kind, selector telemetrytypes.FieldKeySelector) (int, metricSpelling, bool) {
|
||||
if idx, ok := lookupIndex(kind, selector); ok {
|
||||
return idx, metricSpellingDotted, true
|
||||
}
|
||||
|
||||
for idx, family := range families {
|
||||
if !matchesSelector(family, kind, selector) {
|
||||
continue
|
||||
}
|
||||
for _, member := range familyMembers[idx] {
|
||||
if normalizeMetricSpelling(member) == selector.Name {
|
||||
return idx, metricSpellingNormalized, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, metricSpellingDotted, false
|
||||
}
|
||||
|
||||
func normalizeMetricSpelling(name string) string {
|
||||
return strings.ReplaceAll(name, ".", "_")
|
||||
}
|
||||
|
||||
func appendUniqueString(values []string, value string) []string {
|
||||
if value == "" || slices.Contains(values, value) {
|
||||
return values
|
||||
}
|
||||
return append(values, value)
|
||||
}
|
||||
|
||||
func matchesSelector(family Family, kind Kind, selector telemetrytypes.FieldKeySelector) bool {
|
||||
if family.Kind != kind {
|
||||
return false
|
||||
|
||||
@@ -57,3 +57,58 @@ func TestAllReturnsDefensiveCopies(t *testing.T) {
|
||||
second := All()
|
||||
assert.NotEqual(t, "mutated", second[0].Old[0], "All must not expose mutable generated data")
|
||||
}
|
||||
|
||||
func TestAttributeMembersIncludesMetricResourceStorageSpellings(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{
|
||||
"resource_db.system.name", "resource_db_system_name", "db.system.name", "db_system_name",
|
||||
"resource_db.system", "resource_db_system", "db.system", "db_system",
|
||||
}, AttributeMembers(selector), "resource metric attributes should cover every historical storage layout")
|
||||
}
|
||||
|
||||
func TestCurrentAttributeResolvesNormalizedMetricResourceSpelling(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "resource_db_system",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
}
|
||||
|
||||
assert.Equal(t, "db.system.name", CurrentAttribute(selector), "normalized resource spelling should resolve to the dotted current name")
|
||||
}
|
||||
|
||||
func TestAttributeMembersPreservesNormalizedMetricPointStyle(t *testing.T) {
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: "db_system",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{
|
||||
"db_system_name", "db.system.name", "db_system", "db.system",
|
||||
}, AttributeMembers(selector), "normalized point attribute should remain the preferred storage spelling")
|
||||
}
|
||||
|
||||
func TestMetricNamesPreservesDottedStyle(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
[]string{"k8s.pod.cpu.usage", "k8s.pod.cpu.utilization"},
|
||||
MetricNames("k8s.pod.cpu.usage"),
|
||||
"dotted metric input should produce dotted family names",
|
||||
)
|
||||
}
|
||||
|
||||
func TestMetricNamesPreservesNormalizedStyle(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
[]string{"k8s_pod_cpu_usage", "k8s_pod_cpu_utilization"},
|
||||
MetricNames("k8s_pod_cpu_utilization"),
|
||||
"normalized metric input should produce normalized family names",
|
||||
)
|
||||
}
|
||||
|
||||
func TestMetricNamesReturnsUnknownNameUnchanged(t *testing.T) {
|
||||
assert.Equal(t, []string{"custom_metric"}, MetricNames("custom_metric"), "unknown metrics should not be expanded")
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/statementbuilder"
|
||||
"github.com/SigNoz/signoz/pkg/telemetryschema/metricstelemetryschema"
|
||||
"github.com/SigNoz/signoz/pkg/types/metrictypes"
|
||||
@@ -31,6 +32,15 @@ const (
|
||||
OthersMultiTemporality = `IF(LOWER(temporality) LIKE LOWER('delta'), %s, %s) AS per_series_value`
|
||||
)
|
||||
|
||||
func metricNameValues(name string) []any {
|
||||
members := semconv.MetricNames(name)
|
||||
values := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, member)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
type StatementBuilder struct {
|
||||
logger *slog.Logger
|
||||
metadataStore telemetrytypes.MetadataStore
|
||||
@@ -341,7 +351,7 @@ func (b *StatementBuilder) buildReducedTimeSeriesCTE(
|
||||
sb.SelectMore(col)
|
||||
}
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LTE("unix_milli", end),
|
||||
)
|
||||
@@ -385,7 +395,7 @@ func (b *StatementBuilder) buildReducedSpatialAggFastPath(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
sb.In("metric_name", metricNameValues(agg.MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -427,7 +437,7 @@ func (b *StatementBuilder) buildReducedTemporalAggregationCTE(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points FINAL", metricstelemetryschema.DBName, metricstelemetryschema.WhichReducedSamplesTableToUse(agg.Type)))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.reduced_fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", agg.MetricName),
|
||||
sb.In("metric_name", metricNameValues(agg.MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -505,7 +515,7 @@ func (b *StatementBuilder) buildTemporalAggDeltaFastPath(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -560,7 +570,7 @@ func (b *StatementBuilder) buildTimeSeriesCTE(
|
||||
}
|
||||
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LTE("unix_milli", end),
|
||||
)
|
||||
@@ -638,7 +648,7 @@ func (b *StatementBuilder) buildTemporalAggDelta(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -679,7 +689,7 @@ func (b *StatementBuilder) buildTemporalAggCumulativeOrUnspecified(
|
||||
baseSb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
baseSb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
baseSb.Where(
|
||||
baseSb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
baseSb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
baseSb.GTE("unix_milli", start),
|
||||
baseSb.LT("unix_milli", end),
|
||||
)
|
||||
@@ -770,7 +780,7 @@ func (b *StatementBuilder) buildTemporalAggForMultipleTemporalities(
|
||||
sb.From(fmt.Sprintf("%s.%s AS points", metricstelemetryschema.DBName, samplesTable))
|
||||
sb.JoinWithOption(sqlbuilder.InnerJoin, timeSeriesCTE, "points.fingerprint = filtered_time_series.fingerprint")
|
||||
sb.Where(
|
||||
sb.In("metric_name", query.Aggregations[0].MetricName),
|
||||
sb.In("metric_name", metricNameValues(query.Aggregations[0].MetricName)...),
|
||||
sb.GTE("unix_milli", start),
|
||||
sb.LT("unix_milli", end),
|
||||
)
|
||||
|
||||
@@ -13,9 +13,21 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMetricNameValuesResolveFamily(t *testing.T) {
|
||||
assert.Equal(t,
|
||||
[]any{"k8s.pod.cpu.usage", "k8s.pod.cpu.utilization"},
|
||||
metricNameValues("k8s.pod.cpu.usage"),
|
||||
)
|
||||
assert.Equal(t,
|
||||
[]any{"container_cpu_usage", "container_cpu_utilization"},
|
||||
metricNameValues("container_cpu_utilization"),
|
||||
)
|
||||
}
|
||||
|
||||
func TestStatementBuilder(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
|
||||
@@ -153,15 +153,19 @@ func (t *telemetryMetaStore) tracesTblStatementToFieldKeys(ctx context.Context)
|
||||
return materialisedKeys, nil
|
||||
}
|
||||
|
||||
func traceSemconvMembers(name string, fieldContext telemetrytypes.FieldContext) []string {
|
||||
return semconv.Members(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
func attributeSemconvMembers(name string, signal telemetrytypes.Signal, fieldContext telemetrytypes.FieldContext) []string {
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Signal: signal,
|
||||
FieldContext: fieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
func traceSemconvDuplicateFactor() int {
|
||||
func traceSemconvMembers(name string, fieldContext telemetrytypes.FieldContext) []string {
|
||||
return attributeSemconvMembers(name, telemetrytypes.SignalTraces, fieldContext)
|
||||
}
|
||||
|
||||
func semconvDuplicateFactor(signal telemetrytypes.Signal) int {
|
||||
factor := 1
|
||||
for _, family := range semconv.All() {
|
||||
if family.Kind != semconv.KindAttribute {
|
||||
@@ -169,43 +173,66 @@ func traceSemconvDuplicateFactor() int {
|
||||
}
|
||||
if _, ok := semconv.Lookup(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
Name: family.Current,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Signal: signal,
|
||||
}); ok {
|
||||
factor = max(factor, len(family.Old)+1)
|
||||
factor = max(factor, len(attributeSemconvMembers(family.Current, signal, telemetrytypes.FieldContextUnspecified)))
|
||||
}
|
||||
}
|
||||
return factor
|
||||
}
|
||||
|
||||
func traceSemconvDuplicateFactor() int {
|
||||
return semconvDuplicateFactor(telemetrytypes.SignalTraces)
|
||||
}
|
||||
|
||||
func inStrings(sb *sqlbuilder.SelectBuilder, column string, values []string) string {
|
||||
args := make([]any, 0, len(values))
|
||||
for _, value := range values {
|
||||
args = append(args, value)
|
||||
}
|
||||
return sb.In(column, args...)
|
||||
}
|
||||
|
||||
// canonicalizeTraceSemconvKeys presents one current-name key for each family.
|
||||
// If metadata contains both spellings, metadata attached to the current name
|
||||
// wins; otherwise the old entry is copied under the current response name.
|
||||
func canonicalizeTraceSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
return canonicalizeSemconvKeys(keys, telemetrytypes.SignalTraces)
|
||||
}
|
||||
|
||||
// canonicalizeSemconvKeys presents one current-name key for each enabled
|
||||
// attribute family and records only the physical spellings actually present in
|
||||
// metadata. That lets field mappers retain their optimized single-key SQL for
|
||||
// homogeneous ranges while mixed ranges coalesce current-first.
|
||||
func canonicalizeSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey, signal telemetrytypes.Signal) []*telemetrytypes.TelemetryFieldKey {
|
||||
result := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
|
||||
indexByIdentity := make(map[string]int)
|
||||
currentSourceByIdentity := make(map[string]bool)
|
||||
|
||||
for _, key := range keys {
|
||||
if key.Signal != telemetrytypes.SignalTraces {
|
||||
if key.Signal != signal {
|
||||
result = append(result, key)
|
||||
continue
|
||||
}
|
||||
|
||||
family, ok := semconv.Lookup(semconv.KindAttribute, telemetrytypes.FieldKeySelector{
|
||||
selector := telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
Signal: signal,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
if !ok {
|
||||
}
|
||||
members := semconv.AttributeMembers(selector)
|
||||
current := semconv.CurrentAttribute(selector)
|
||||
if len(members) == 1 && current == key.Name {
|
||||
result = append(result, key)
|
||||
continue
|
||||
}
|
||||
|
||||
resolved := *key
|
||||
resolved.Name = family.Current
|
||||
resolved.Name = current
|
||||
resolved.SemconvMembers = []string{key.Name}
|
||||
identity := resolved.Name + ";" + resolved.Signal.StringValue() + ";" + resolved.FieldContext.StringValue() + ";" + resolved.FieldDataType.StringValue()
|
||||
fromCurrent := key.Name == family.Current
|
||||
physicalName := strings.TrimPrefix(key.Name, "resource_")
|
||||
fromCurrent := physicalName == current || physicalName == strings.ReplaceAll(current, ".", "_")
|
||||
|
||||
if index, found := indexByIdentity[identity]; found {
|
||||
physicalMembers := result[index].SemconvMembers
|
||||
@@ -238,7 +265,7 @@ func canonicalizeTraceSemconvKeys(keys []*telemetrytypes.TelemetryFieldKey) []*t
|
||||
present[member] = true
|
||||
}
|
||||
ordered := make([]string, 0, len(key.SemconvMembers))
|
||||
for _, member := range traceSemconvMembers(key.Name, key.FieldContext) {
|
||||
for _, member := range attributeSemconvMembers(key.Name, signal, key.FieldContext) {
|
||||
if present[member] {
|
||||
ordered = append(ordered, member)
|
||||
}
|
||||
@@ -569,10 +596,19 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
continue
|
||||
}
|
||||
fieldKeyConds := []string{}
|
||||
members := attributeSemconvMembers(sel.Name, telemetrytypes.SignalLogs, fieldContext)
|
||||
if sel.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("name", sel.Name))
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
fieldKeyConds = append(fieldKeyConds, sb.In("name", memberValues...))
|
||||
} else {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.ILike("name", "%"+escapeForLike(sel.Name)+"%"))
|
||||
memberConds := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberConds = append(memberConds, sb.ILike("name", "%"+escapeForLike(member)+"%"))
|
||||
}
|
||||
fieldKeyConds = append(fieldKeyConds, sb.Or(memberConds...))
|
||||
}
|
||||
if sel.FieldDataType != telemetrytypes.FieldDataTypeUnspecified {
|
||||
fieldKeyConds = append(fieldKeyConds, sb.E("datatype", sel.FieldDataType.TagDataType()))
|
||||
@@ -665,6 +701,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalLogs)
|
||||
mainQuery := fmt.Sprintf(`
|
||||
SELECT tag_key, tag_type, tag_data_type, max(priority) as priority
|
||||
FROM (
|
||||
@@ -673,7 +710,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
GROUP BY tag_key, tag_type, tag_data_type
|
||||
ORDER BY priority
|
||||
LIMIT %d
|
||||
`, strings.Join(queries, " UNION ALL "), limit+1)
|
||||
`, strings.Join(queries, " UNION ALL "), dbLimit+1)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, mainQuery, allArgs...)
|
||||
if err != nil {
|
||||
@@ -694,7 +731,7 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -742,7 +779,16 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, orgID valuer.UUID,
|
||||
}
|
||||
|
||||
// hit the limit? (only counting DB results)
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit
|
||||
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalLogs)
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
complete = false
|
||||
}
|
||||
mapOfKeys = make(map[string]*telemetrytypes.TelemetryFieldKey, len(keys))
|
||||
for _, key := range keys {
|
||||
mapOfKeys[key.Name+";"+key.FieldContext.StringValue()+";"+key.FieldDataType.StringValue()] = key
|
||||
}
|
||||
|
||||
staticKeys := []string{}
|
||||
staticKeys = append(staticKeys, maps.Keys(logstelemetryschema.IntrinsicFields)...)
|
||||
@@ -1050,10 +1096,19 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
conds := []string{}
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
fieldConds := []string{}
|
||||
members := attributeSemconvMembers(fieldKeySelector.Name, telemetrytypes.SignalMetrics, fieldKeySelector.FieldContext)
|
||||
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name))
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("attr_name", memberValues...))
|
||||
} else {
|
||||
fieldConds = append(fieldConds, sb.ILike("attr_name", "%"+escapeForLike(fieldKeySelector.Name)+"%"))
|
||||
memberConds := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberConds = append(memberConds, sb.ILike("attr_name", "%"+escapeForLike(member)+"%"))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.Or(memberConds...))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.NotLike("attr_name", "\\_\\_%"))
|
||||
|
||||
@@ -1069,7 +1124,12 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
|
||||
if fieldKeySelector.MetricContext != nil {
|
||||
if fieldKeySelector.MetricContext.MetricName != "" {
|
||||
fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName))
|
||||
metricNames := semconv.MetricNames(fieldKeySelector.MetricContext.MetricName)
|
||||
metricValues := make([]any, 0, len(metricNames))
|
||||
for _, metricName := range metricNames {
|
||||
metricValues = append(metricValues, metricName)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("metric_name", metricValues...))
|
||||
}
|
||||
if fieldKeySelector.MetricContext.MetricNamespace != "" {
|
||||
fieldConds = append(fieldConds, sb.Like("metric_name", escapeForLike(fieldKeySelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -1091,7 +1151,8 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
mainSb.GroupBy("name", "field_context", "field_data_type")
|
||||
mainSb.OrderBy("priority")
|
||||
// query one extra to check if we hit the limit
|
||||
mainSb.Limit(limit + 1)
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
|
||||
mainSb.Limit(dbLimit + 1)
|
||||
|
||||
query, args := mainSb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
@@ -1106,7 +1167,7 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1131,7 +1192,12 @@ func (t *telemetryMetaStore) getMetricsKeys(ctx context.Context, fieldKeySelecto
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit
|
||||
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalMetrics)
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
complete = false
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
}
|
||||
@@ -1153,16 +1219,30 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
var limit int
|
||||
for _, fieldKeySelector := range fieldKeySelectors {
|
||||
fieldConds := []string{}
|
||||
members := attributeSemconvMembers(fieldKeySelector.Name, telemetrytypes.SignalMetrics, fieldKeySelector.FieldContext)
|
||||
if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
|
||||
fieldConds = append(fieldConds, sb.E("attr_name", fieldKeySelector.Name))
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("attr_name", memberValues...))
|
||||
} else {
|
||||
fieldConds = append(fieldConds, sb.Like("attr_name", "%"+fieldKeySelector.Name+"%"))
|
||||
memberConds := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberConds = append(memberConds, sb.Like("attr_name", "%"+member+"%"))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.Or(memberConds...))
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.NotLike("attr_name", "\\_\\_%"))
|
||||
|
||||
if fieldKeySelector.MetricContext != nil {
|
||||
if fieldKeySelector.MetricContext.MetricName != "" {
|
||||
fieldConds = append(fieldConds, sb.E("metric_name", fieldKeySelector.MetricContext.MetricName))
|
||||
metricNames := semconv.MetricNames(fieldKeySelector.MetricContext.MetricName)
|
||||
metricValues := make([]any, 0, len(metricNames))
|
||||
for _, metricName := range metricNames {
|
||||
metricValues = append(metricValues, metricName)
|
||||
}
|
||||
fieldConds = append(fieldConds, sb.In("metric_name", metricValues...))
|
||||
}
|
||||
if fieldKeySelector.MetricContext.MetricNamespace != "" {
|
||||
fieldConds = append(fieldConds, sb.Like("metric_name", escapeForLike(fieldKeySelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -1177,7 +1257,8 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
limit = 1000
|
||||
}
|
||||
|
||||
sb.Limit(limit)
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
|
||||
sb.Limit(dbLimit + 1)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
@@ -1191,7 +1272,7 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -1200,9 +1281,14 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterKeys.Error())
|
||||
}
|
||||
fieldContext := telemetrytypes.FieldContextAttribute
|
||||
if strings.HasPrefix(name, "resource_") {
|
||||
fieldContext = telemetrytypes.FieldContextResource
|
||||
}
|
||||
keys = append(keys, &telemetrytypes.TelemetryFieldKey{
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
Name: name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: fieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1211,7 +1297,12 @@ func (t *telemetryMetaStore) getMeterSourceMetricKeys(ctx context.Context, field
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit
|
||||
keys = canonicalizeSemconvKeys(keys, telemetrytypes.SignalMetrics)
|
||||
if len(keys) > limit {
|
||||
keys = keys[:limit]
|
||||
complete = false
|
||||
}
|
||||
|
||||
return keys, complete, nil
|
||||
|
||||
@@ -1570,7 +1661,6 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, orgID valuer.
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
}
|
||||
// query one extra to check if we hit the limit
|
||||
sb.Limit(limit + 1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
@@ -1725,7 +1815,12 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
|
||||
sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("tag_key", fieldValueSelector.Name))
|
||||
members := attributeSemconvMembers(fieldValueSelector.Name, telemetrytypes.SignalLogs, fieldValueSelector.FieldContext)
|
||||
memberValues := make([]any, 0, len(members))
|
||||
for _, member := range members {
|
||||
memberValues = append(memberValues, member)
|
||||
}
|
||||
sb.Where(sb.In("tag_key", memberValues...))
|
||||
}
|
||||
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
@@ -1923,7 +2018,9 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, orgID val
|
||||
From(t.metricsDBName + "." + t.metricsFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("attr_name", fieldValueSelector.Name))
|
||||
sb.Where(inStrings(sb, "attr_name", attributeSemconvMembers(
|
||||
fieldValueSelector.Name, telemetrytypes.SignalMetrics, fieldValueSelector.FieldContext,
|
||||
)))
|
||||
}
|
||||
|
||||
if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
@@ -1935,7 +2032,7 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, orgID val
|
||||
}
|
||||
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
|
||||
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
|
||||
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
|
||||
}
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
|
||||
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -2069,7 +2166,7 @@ func (t *telemetryMetaStore) getIntrinsicMetricFieldValuesForTable(ctx context.C
|
||||
From(t.metricsDBName + "." + tableName)
|
||||
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
|
||||
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
|
||||
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
|
||||
}
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
|
||||
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -2132,12 +2229,14 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
From(t.meterDBName + "." + t.meterFieldsTblName)
|
||||
|
||||
if fieldValueSelector.Name != "" {
|
||||
sb.Where(sb.E("attr.1", fieldValueSelector.Name))
|
||||
sb.Where(inStrings(sb, "attr.1", attributeSemconvMembers(
|
||||
fieldValueSelector.Name, telemetrytypes.SignalMetrics, fieldValueSelector.FieldContext,
|
||||
)))
|
||||
}
|
||||
sb.Where(sb.NotLike("attr.1", "\\_\\_%"))
|
||||
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
|
||||
sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
|
||||
sb.Where(inStrings(sb, "metric_name", semconv.MetricNames(fieldValueSelector.MetricContext.MetricName)))
|
||||
}
|
||||
if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricNamespace != "" {
|
||||
sb.Where(sb.Like("metric_name", escapeForLike(fieldValueSelector.MetricContext.MetricNamespace)+"%"))
|
||||
@@ -2156,8 +2255,10 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
if limit == 0 {
|
||||
limit = 50
|
||||
}
|
||||
// query one extra to check if we hit the limit
|
||||
sb.Limit(limit + 1)
|
||||
// A value can be present under several physical spellings; over-fetch and
|
||||
// de-duplicate after scanning so one family cannot consume the result limit.
|
||||
dbLimit := limit * semconvDuplicateFactor(telemetrytypes.SignalMetrics)
|
||||
sb.Limit(dbLimit + 1)
|
||||
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
@@ -2167,11 +2268,13 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
defer rows.Close()
|
||||
|
||||
values := &telemetrytypes.TelemetryFieldValues{}
|
||||
seen := make(map[string]bool)
|
||||
rowCount := 0
|
||||
uniqueCount := 0
|
||||
for rows.Next() {
|
||||
rowCount++
|
||||
// reached the limit, we know there are more results
|
||||
if rowCount > limit {
|
||||
if rowCount > dbLimit {
|
||||
break
|
||||
}
|
||||
|
||||
@@ -2180,12 +2283,16 @@ func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context
|
||||
return nil, false, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMeterValues.Error())
|
||||
}
|
||||
if len(attribute) > 1 {
|
||||
values.StringValues = append(values.StringValues, attribute[1])
|
||||
if !seen[attribute[1]] && uniqueCount < limit {
|
||||
values.StringValues = append(values.StringValues, attribute[1])
|
||||
seen[attribute[1]] = true
|
||||
uniqueCount++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hit the limit?
|
||||
complete := rowCount <= limit
|
||||
complete := rowCount <= dbLimit && uniqueCount < limit
|
||||
return values, complete, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -130,6 +130,44 @@ func TestCanonicalizeTraceSemconvKeys(t *testing.T) {
|
||||
assert.Equal(t, "deployment.environment", result[2].Name, "phase 1 must not rewrite raw log metadata")
|
||||
}
|
||||
|
||||
func TestCanonicalizeLogAndMetricSemconvKeys(t *testing.T) {
|
||||
logKeys := canonicalizeSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
Name: "db.system",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
}, telemetrytypes.SignalLogs)
|
||||
require.Len(t, logKeys, 1)
|
||||
assert.Equal(t, "db.system.name", logKeys[0].Name)
|
||||
assert.Equal(t, []string{"db.system.name", "db.system"}, logKeys[0].SemconvMembers)
|
||||
|
||||
metricKeys := canonicalizeSemconvKeys([]*telemetrytypes.TelemetryFieldKey{
|
||||
{
|
||||
Name: "resource_db_system",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
},
|
||||
}, telemetrytypes.SignalMetrics)
|
||||
require.Len(t, metricKeys, 1)
|
||||
assert.Equal(t, "db.system.name", metricKeys[0].Name)
|
||||
assert.Equal(t, []string{"db.system.name", "resource_db_system"}, metricKeys[0].SemconvMembers)
|
||||
}
|
||||
|
||||
func TestGetSpanFieldValuesMergesSemconvFamily(t *testing.T) {
|
||||
mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, ®exMatcher{})
|
||||
mock := mockTelemetryStore.Mock()
|
||||
|
||||
156
pkg/telemetrymetadata/semconv_migration.go
Normal file
156
pkg/telemetrymetadata/semconv_migration.go
Normal file
@@ -0,0 +1,156 @@
|
||||
package telemetrymetadata
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
type semconvMigrationRow struct {
|
||||
current string
|
||||
old string
|
||||
signal string
|
||||
service string
|
||||
resourceSets uint64
|
||||
lastSeenUnixMilli int64
|
||||
}
|
||||
|
||||
// GetSemconvMigrationReport derives an old-only service report from the
|
||||
// generated family registry and attributes_metadata. The latter is already a
|
||||
// deduplicated set of resource/attribute fingerprints, so this audit avoids a
|
||||
// scan of the raw telemetry tables.
|
||||
func (t *telemetryMetaStore) GetSemconvMigrationReport(
|
||||
ctx context.Context,
|
||||
_ valuer.UUID,
|
||||
startUnixMilli, endUnixMilli int64,
|
||||
) (*telemetrytypes.GettableSemconvMigrationReport, error) {
|
||||
query, args := t.semconvMigrationReportQuery(startUnixMilli, endUnixMilli)
|
||||
report := &telemetrytypes.GettableSemconvMigrationReport{
|
||||
StartUnixMilli: startUnixMilli,
|
||||
EndUnixMilli: endUnixMilli,
|
||||
Entries: []*telemetrytypes.SemconvMigrationReportEntry{},
|
||||
}
|
||||
if query == "" {
|
||||
return report, nil
|
||||
}
|
||||
|
||||
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, "failed to build semantic-convention migration report")
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
grouped := make(map[string]*telemetrytypes.SemconvMigrationReportEntry)
|
||||
serviceSets := make(map[string]map[string]struct{})
|
||||
for rows.Next() {
|
||||
var row semconvMigrationRow
|
||||
if err := rows.Scan(
|
||||
&row.current,
|
||||
&row.old,
|
||||
&row.signal,
|
||||
&row.service,
|
||||
&row.resourceSets,
|
||||
&row.lastSeenUnixMilli,
|
||||
); err != nil {
|
||||
return nil, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, "failed to scan semantic-convention migration report")
|
||||
}
|
||||
|
||||
identity := row.current + "\x00" + row.old + "\x00" + row.signal
|
||||
entry, ok := grouped[identity]
|
||||
if !ok {
|
||||
entry = &telemetrytypes.SemconvMigrationReportEntry{
|
||||
Current: row.current,
|
||||
Old: row.old,
|
||||
Signal: row.signal,
|
||||
Services: []string{},
|
||||
}
|
||||
grouped[identity] = entry
|
||||
serviceSets[identity] = make(map[string]struct{})
|
||||
report.Entries = append(report.Entries, entry)
|
||||
}
|
||||
serviceSets[identity][row.service] = struct{}{}
|
||||
entry.ResourceSets += row.resourceSets
|
||||
entry.LastSeenUnixMilli = max(entry.LastSeenUnixMilli, row.lastSeenUnixMilli)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, "failed to read semantic-convention migration report")
|
||||
}
|
||||
|
||||
for identity, entry := range grouped {
|
||||
for service := range serviceSets[identity] {
|
||||
entry.Services = append(entry.Services, service)
|
||||
}
|
||||
slices.Sort(entry.Services)
|
||||
}
|
||||
slices.SortFunc(report.Entries, func(a, b *telemetrytypes.SemconvMigrationReportEntry) int {
|
||||
return strings.Compare(a.Current+"\x00"+a.Old+"\x00"+a.Signal, b.Current+"\x00"+b.Old+"\x00"+b.Signal)
|
||||
})
|
||||
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (t *telemetryMetaStore) semconvMigrationReportQuery(startUnixMilli, endUnixMilli int64) (string, []any) {
|
||||
builders := make([]sqlbuilder.Builder, 0)
|
||||
for _, family := range semconv.All() {
|
||||
if family.Kind != semconv.KindAttribute {
|
||||
continue
|
||||
}
|
||||
for _, old := range family.Old {
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(
|
||||
fmt.Sprintf("%s AS current_name", sb.Var(family.Current)),
|
||||
fmt.Sprintf("%s AS old_name", sb.Var(old)),
|
||||
"data_source",
|
||||
"if(empty(resource_attributes['service.name']), '<unknown>', resource_attributes['service.name']) AS service_name",
|
||||
"uniqExact(tuple(resource_fingerprint, attrs_fingerprint)) AS resource_sets",
|
||||
"max(unix_milli) AS last_seen_unix_milli",
|
||||
)
|
||||
sb.From(t.relatedMetadataDBName + "." + t.relatedMetadataTblName)
|
||||
sb.Where(sb.GE("unix_milli", startUnixMilli))
|
||||
sb.Where(sb.LE("unix_milli", endUnixMilli))
|
||||
|
||||
if len(family.Signals) > 0 {
|
||||
signals := make([]any, 0, len(family.Signals))
|
||||
for _, signal := range family.Signals {
|
||||
signals = append(signals, signal.StringValue())
|
||||
}
|
||||
sb.Where(sb.In("data_source", signals...))
|
||||
}
|
||||
|
||||
oldPresence := semconvMetadataPresenceConditions(sb, old)
|
||||
currentPresence := semconvMetadataPresenceConditions(sb, family.Current)
|
||||
sb.Where(sb.Or(oldPresence...))
|
||||
sb.Where(fmt.Sprintf("NOT (%s)", sb.Or(currentPresence...)))
|
||||
sb.GroupBy("data_source", "service_name")
|
||||
builders = append(builders, sb)
|
||||
}
|
||||
}
|
||||
|
||||
if len(builders) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
union := sqlbuilder.UnionAll(builders...)
|
||||
return union.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
}
|
||||
|
||||
func semconvMetadataPresenceConditions(sb *sqlbuilder.SelectBuilder, name string) []string {
|
||||
spellings := []string{name, strings.ReplaceAll(name, ".", "_")}
|
||||
spellings = append(spellings, "resource_"+name, "resource_"+strings.ReplaceAll(name, ".", "_"))
|
||||
spellings = slices.Compact(spellings)
|
||||
conditions := make([]string, 0, len(spellings)*2)
|
||||
for _, spelling := range spellings {
|
||||
conditions = append(conditions,
|
||||
fmt.Sprintf("mapContains(resource_attributes, %s)", sb.Var(spelling)),
|
||||
fmt.Sprintf("mapContains(attributes, %s)", sb.Var(spelling)),
|
||||
)
|
||||
}
|
||||
return conditions
|
||||
}
|
||||
@@ -89,10 +89,12 @@ func (v *TelemetryFieldVisitor) VisitColumnDef(expr *parser.ColumnDef) error {
|
||||
|
||||
// Create and store the TelemetryFieldKey
|
||||
field := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: fieldName,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
Materialized: true,
|
||||
Name: fieldName,
|
||||
FieldContext: fieldContext,
|
||||
FieldDataType: fieldDataType,
|
||||
Materialized: true,
|
||||
MaterializedColumnName: columnName,
|
||||
MaterializedSemconv: strings.Count(defaultExprStr, "['") > 1,
|
||||
}
|
||||
|
||||
v.Fields = append(v.Fields, field)
|
||||
|
||||
@@ -5,8 +5,25 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestExtractFieldKeyPreservesHistoricalMaterializedColumnName(t *testing.T) {
|
||||
statement := `CREATE TABLE signoz_traces.signoz_index_v3
|
||||
(
|
||||
attributes_string Map(LowCardinality(String), String),
|
||||
` + "`attribute_string_db$$system`" + ` LowCardinality(String)
|
||||
DEFAULT if(mapContains(attributes_string, 'db.system.name'), attributes_string['db.system.name'], attributes_string['db.system'])
|
||||
) ENGINE = MergeTree ORDER BY tuple()`
|
||||
keys, err := ExtractFieldKeysFromTblStatement(statement)
|
||||
require.NoError(t, err, "table statement should parse")
|
||||
require.Len(t, keys, 1, "table statement should contain one materialized key")
|
||||
assert.Equal(t, "db.system.name", keys[0].Name)
|
||||
assert.Equal(t, "attribute_string_db$$system", keys[0].MaterializedColumnName)
|
||||
assert.True(t, keys[0].MaterializedSemconv)
|
||||
}
|
||||
|
||||
func TestExtractFieldKeysFromTblStatement(t *testing.T) {
|
||||
|
||||
var statement = `CREATE TABLE signoz_logs.logs_v2
|
||||
|
||||
@@ -453,6 +453,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
) ([]string, []string, error) {
|
||||
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
if options.ExactSemconv {
|
||||
matches = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
|
||||
}
|
||||
skipResourceFilter := options.SkipResourceFilter
|
||||
|
||||
// search() resolves its own (optional) scope; handle it before key resolution.
|
||||
@@ -499,6 +502,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
|
||||
}
|
||||
}
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.ExactSemconvKeys(keys)
|
||||
}
|
||||
|
||||
if skipResourceFilter && !synthesized {
|
||||
filtered := make([]*telemetrytypes.TelemetryFieldKey, 0, len(keys))
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
@@ -67,6 +68,20 @@ type fieldMapper struct {
|
||||
fl flagger.Flagger
|
||||
}
|
||||
|
||||
func logSemconvMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource && key.FieldContext != telemetrytypes.FieldContextAttribute {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
func NewFieldMapper(fl flagger.Flagger) qbtypes.FieldMapper {
|
||||
return &fieldMapper{fl: fl}
|
||||
}
|
||||
@@ -141,8 +156,20 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
|
||||
case schema.ColumnTypeEnumJSON:
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource:
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, key.Name))
|
||||
members := logSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
values := make([]string, 0, len(members))
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s.`%s`::String, '')", columnName, member))
|
||||
guards = append(guards, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
|
||||
existExpr = append(existExpr, "("+strings.Join(guards, " OR ")+")")
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s.`%s`::String", columnName, members[0]))
|
||||
existExpr = append(existExpr, fmt.Sprintf("%s.`%s` IS NOT NULL", columnName, members[0]))
|
||||
}
|
||||
case telemetrytypes.FieldContextBody:
|
||||
if key.Name == messageSubField {
|
||||
exprs = append(exprs, messageSubColumn)
|
||||
@@ -181,13 +208,32 @@ func (m *fieldMapper) FieldFor(ctx context.Context, orgID valuer.UUID, tsStart,
|
||||
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumBool, schema.ColumnTypeEnumFloat64:
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
if key.Materialized {
|
||||
members := logSemconvMembers(key)
|
||||
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
|
||||
existExpr = append(existExpr, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
|
||||
} else if len(members) > 1 {
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
guards = append(guards, fmt.Sprintf("mapContains(%s, '%s')", columnName, member))
|
||||
}
|
||||
if valueType.GetType() == schema.ColumnTypeEnumString {
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(%s['%s'], '')", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "COALESCE("+strings.Join(values, ", ")+")")
|
||||
} else {
|
||||
branches := make([]string, 0, len(members)*2+1)
|
||||
for i, member := range members {
|
||||
branches = append(branches, guards[i], fmt.Sprintf("%s['%s']", columnName, member))
|
||||
}
|
||||
exprs = append(exprs, "multiIf("+strings.Join(branches, ", ")+", NULL)")
|
||||
}
|
||||
existExpr = append(existExpr, "("+strings.Join(guards, " OR ")+")")
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, key.Name))
|
||||
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, key.Name))
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, members[0]))
|
||||
existExpr = append(existExpr, fmt.Sprintf("mapContains(%s, '%s')", columnName, members[0]))
|
||||
}
|
||||
default:
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "exists operator is not supported for map column type %s", valueType)
|
||||
|
||||
@@ -2,6 +2,7 @@ package logstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -14,6 +15,32 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestFieldForSemconvFamily(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper(flaggertest.New(t)).(*fieldMapper)
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalLogs,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
expression, err := fm.FieldFor(ctx, valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "COALESCE(NULLIF(attributes_string['db.system.name'], ''), NULLIF(attributes_string['db.system'], ''))", expression)
|
||||
assert.Less(t, strings.Index(expression, "db.system.name"), strings.Index(expression, "db.system']"), "current spelling must win")
|
||||
|
||||
exists, err := fm.existsExpressionFor(ctx, valuer.UUID{}, 0, 0, key, true)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "(mapContains(attributes_string, 'db.system.name') OR mapContains(attributes_string, 'db.system'))", exists)
|
||||
|
||||
exact := *key
|
||||
exact.SemconvMembers = []string{"db.system"}
|
||||
expression, err = fm.FieldFor(ctx, valuer.UUID{}, 0, 0, &exact)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "attributes_string['db.system']", expression)
|
||||
}
|
||||
|
||||
func TestGetColumn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
@@ -135,10 +136,22 @@ func (c *conditionBuilder) conditionFor(
|
||||
return "true", nil
|
||||
}
|
||||
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", key.Name), nil
|
||||
members := metricAttributeMembers(key)
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
guards = append(guards, fmt.Sprintf("has(JSONExtractKeys(labels), '%s')", member))
|
||||
}
|
||||
return fmt.Sprintf("not has(JSONExtractKeys(labels), '%s')", key.Name), nil
|
||||
guard := strings.Join(guards, " OR ")
|
||||
if len(guards) > 1 {
|
||||
guard = "(" + guard + ")"
|
||||
}
|
||||
if operator == qbtypes.FilterOperatorExists {
|
||||
return guard, nil
|
||||
}
|
||||
if len(guards) == 1 {
|
||||
return "not " + guard, nil
|
||||
}
|
||||
return "NOT " + guard, nil
|
||||
}
|
||||
return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "unsupported operator: %v", operator)
|
||||
}
|
||||
@@ -151,7 +164,7 @@ func (c *conditionBuilder) ConditionFor(
|
||||
endNs uint64,
|
||||
key *telemetrytypes.TelemetryFieldKey,
|
||||
fieldKeys map[string][]*telemetrytypes.TelemetryFieldKey,
|
||||
_ qbtypes.ConditionBuilderOptions,
|
||||
options qbtypes.ConditionBuilderOptions,
|
||||
operator qbtypes.FilterOperator,
|
||||
value any,
|
||||
sb *sqlbuilder.SelectBuilder,
|
||||
@@ -162,7 +175,15 @@ func (c *conditionBuilder) ConditionFor(
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
requestedKey := *key
|
||||
if requestedKey.Signal == telemetrytypes.SignalUnspecified {
|
||||
requestedKey.Signal = telemetrytypes.SignalMetrics
|
||||
}
|
||||
key = &requestedKey
|
||||
keys := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
|
||||
}
|
||||
var warnings []string
|
||||
if len(keys) == 0 {
|
||||
if _, isColumn := timeSeriesV4Columns[key.Name]; isColumn {
|
||||
@@ -180,6 +201,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
}
|
||||
}
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.ExactSemconvKeys(keys)
|
||||
}
|
||||
|
||||
conds := make([]string, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
|
||||
@@ -390,3 +390,45 @@ func TestConditionForKeyNotInMetadata(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConditionForSemconvMetricLabels(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fm := NewFieldMapper()
|
||||
conditionBuilder := NewConditionBuilder(fm)
|
||||
requested := telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
current := requested
|
||||
legacyNormalized := requested
|
||||
legacyNormalized.Name = "resource_db_system"
|
||||
fieldKeys := map[string][]*telemetrytypes.TelemetryFieldKey{
|
||||
current.Name: {¤t},
|
||||
legacyNormalized.Name: {&legacyNormalized},
|
||||
}
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
conditions, warnings, err := conditionBuilder.ConditionFor(
|
||||
ctx, valuer.UUID{}, 0, 0, &requested, fieldKeys, qbtypes.ConditionBuilderOptions{},
|
||||
qbtypes.FilterOperatorEqual, "postgresql", sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, warnings)
|
||||
sb.Where(conditions...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, query, "COALESCE(NULLIF(JSONExtractString(labels, 'db.system.name'), ''), NULLIF(JSONExtractString(labels, 'resource_db_system'), '')) = ?")
|
||||
assert.Equal(t, []any{"postgresql"}, args)
|
||||
|
||||
sb = sqlbuilder.NewSelectBuilder()
|
||||
conditions, _, err = conditionBuilder.ConditionFor(
|
||||
ctx, valuer.UUID{}, 0, 0, &requested, fieldKeys, qbtypes.ConditionBuilderOptions{ExactSemconv: true},
|
||||
qbtypes.FilterOperatorExists, nil, sb,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
sb.Where(conditions...)
|
||||
query, _ = sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
assert.Contains(t, query, "has(JSONExtractKeys(labels), 'db.system.name')")
|
||||
assert.NotContains(t, query, "resource_db_system")
|
||||
}
|
||||
|
||||
@@ -4,8 +4,10 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/semconv"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -38,6 +40,23 @@ var (
|
||||
|
||||
type fieldMapper struct{}
|
||||
|
||||
func metricAttributeMembers(key *telemetrytypes.TelemetryFieldKey) []string {
|
||||
if key.FieldContext != telemetrytypes.FieldContextResource &&
|
||||
key.FieldContext != telemetrytypes.FieldContextScope &&
|
||||
key.FieldContext != telemetrytypes.FieldContextAttribute &&
|
||||
key.FieldContext != telemetrytypes.FieldContextUnspecified {
|
||||
return []string{key.Name}
|
||||
}
|
||||
if len(key.SemconvMembers) > 0 {
|
||||
return key.SemconvMembers
|
||||
}
|
||||
return semconv.AttributeMembers(telemetrytypes.FieldKeySelector{
|
||||
Name: key.Name,
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: key.FieldContext,
|
||||
})
|
||||
}
|
||||
|
||||
// CandidateKeys returns nil: metrics has no attribute-map fallback, so a context-missing
|
||||
// key stays unresolved and the caller errors.
|
||||
func (m *fieldMapper) CandidateKeys(_ context.Context, _ valuer.UUID, _ *telemetrytypes.TelemetryFieldKey, _ any, _ map[string][]*telemetrytypes.TelemetryFieldKey) []*telemetrytypes.TelemetryFieldKey {
|
||||
@@ -80,19 +99,30 @@ func (m *fieldMapper) FieldFor(ctx context.Context, _ valuer.UUID, startNs, endN
|
||||
|
||||
switch key.FieldContext {
|
||||
case telemetrytypes.FieldContextResource, telemetrytypes.FieldContextScope, telemetrytypes.FieldContextAttribute:
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
return metricLabelExpression(columns[0].Name, metricAttributeMembers(key)), nil
|
||||
case telemetrytypes.FieldContextMetric:
|
||||
return columns[0].Name, nil
|
||||
case telemetrytypes.FieldContextUnspecified:
|
||||
if slices.Contains(IntrinsicFields, key.Name) {
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columns[0].Name, key.Name), nil
|
||||
return metricLabelExpression(columns[0].Name, metricAttributeMembers(key)), nil
|
||||
}
|
||||
|
||||
return columns[0].Name, nil
|
||||
}
|
||||
|
||||
func metricLabelExpression(columnName string, members []string) string {
|
||||
if len(members) == 1 {
|
||||
return fmt.Sprintf("JSONExtractString(%s, '%s')", columnName, members[0])
|
||||
}
|
||||
values := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
values = append(values, fmt.Sprintf("NULLIF(JSONExtractString(%s, '%s'), '')", columnName, member))
|
||||
}
|
||||
return "COALESCE(" + strings.Join(values, ", ") + ")"
|
||||
}
|
||||
|
||||
func (m *fieldMapper) ColumnFor(ctx context.Context, _ valuer.UUID, tsStart, tsEnd uint64, key *telemetrytypes.TelemetryFieldKey) ([]*schema.Column, error) {
|
||||
return m.getColumn(ctx, tsStart, tsEnd, key)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package metricstelemetryschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
@@ -12,6 +13,32 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestMetricLabelSemconvSpellings(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalMetrics,
|
||||
FieldContext: telemetrytypes.FieldContextResource,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
}
|
||||
|
||||
expression, err := fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
for _, member := range []string{
|
||||
"resource_db.system.name", "resource_db_system_name", "db.system.name", "db_system_name",
|
||||
"resource_db.system", "resource_db_system", "db.system", "db_system",
|
||||
} {
|
||||
assert.Contains(t, expression, "'"+member+"'")
|
||||
}
|
||||
assert.Less(t, strings.Index(expression, "resource_db.system.name"), strings.Index(expression, "resource_db.system'"))
|
||||
|
||||
exact := *key
|
||||
exact.SemconvMembers = []string{"resource_db_system"}
|
||||
expression, err = fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, &exact)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "JSONExtractString(labels, 'resource_db_system')", expression)
|
||||
}
|
||||
|
||||
func TestGetColumn(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
|
||||
@@ -220,6 +220,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
}
|
||||
|
||||
matches := querybuilder.MatchingFieldKeys(key, fieldKeys)
|
||||
if options.ExactSemconv {
|
||||
matches = querybuilder.MatchingFieldKeysExact(key, fieldKeys)
|
||||
}
|
||||
skipResourceFilter := options.SkipResourceFilter
|
||||
|
||||
keys, warning := querybuilder.ResolveKeys(key, matches)
|
||||
@@ -265,6 +268,9 @@ func (c *conditionBuilder) ConditionFor(
|
||||
synthesized = true
|
||||
warnings = append(warnings, querybuilder.NewKeyNotFoundWarning(key.Name))
|
||||
}
|
||||
if options.ExactSemconv {
|
||||
keys = querybuilder.ExactSemconvKeys(keys)
|
||||
}
|
||||
|
||||
// When a resource sub-query already covers the term, drop resource keys from the main
|
||||
// query. Synthesized keys are exempt: the sub-query skips keys absent from metadata.
|
||||
|
||||
@@ -362,7 +362,10 @@ func (m *fieldMapper) resolveColumnExprs(
|
||||
switch valueType := column.Type.(schema.MapColumnType).ValueType; valueType.GetType() {
|
||||
case schema.ColumnTypeEnumString, schema.ColumnTypeEnumFloat64, schema.ColumnTypeEnumBool:
|
||||
members := traceSemconvMembers(key)
|
||||
if len(members) > 1 {
|
||||
if key.Materialized && (len(members) == 1 || key.MaterializedSemconv) {
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(key))
|
||||
existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key))
|
||||
} else if len(members) > 1 {
|
||||
guards := make([]string, 0, len(members))
|
||||
for _, member := range members {
|
||||
guards = append(guards, fmt.Sprintf("mapContains(%s, '%s')", columnName, member))
|
||||
@@ -381,12 +384,6 @@ func (m *fieldMapper) resolveColumnExprs(
|
||||
exprs = append(exprs, "multiIf("+strings.Join(branches, ", ")+", NULL)")
|
||||
}
|
||||
existExprs = append(existExprs, "("+strings.Join(guards, " OR ")+")")
|
||||
} else if key.Materialized {
|
||||
// a key could have been materialized, if so return the materialized column name
|
||||
physicalKey := *key
|
||||
physicalKey.Name = members[0]
|
||||
exprs = append(exprs, telemetrytypes.FieldKeyToMaterializedColumnName(&physicalKey))
|
||||
existExprs = append(existExprs, telemetrytypes.FieldKeyToMaterializedColumnNameForExists(&physicalKey))
|
||||
} else {
|
||||
exprs = append(exprs, fmt.Sprintf("%s['%s']", columnName, members[0]))
|
||||
existExprs = append(existExprs, fmt.Sprintf("mapContains(%s, '%s')", columnName, members[0]))
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -348,3 +350,27 @@ func TestColumnExpressionForTimestampAttributeCollision(t *testing.T) {
|
||||
assert.Contains(t, result, "attributes_number['timestamp']")
|
||||
})
|
||||
}
|
||||
|
||||
func TestDBSystemFamilyUsesSemconvAwareMaterializedColumn(t *testing.T) {
|
||||
fm := NewFieldMapper()
|
||||
key := &telemetrytypes.TelemetryFieldKey{
|
||||
Name: "db.system.name",
|
||||
Signal: telemetrytypes.SignalTraces,
|
||||
FieldContext: telemetrytypes.FieldContextAttribute,
|
||||
FieldDataType: telemetrytypes.FieldDataTypeString,
|
||||
Materialized: true,
|
||||
MaterializedColumnName: "attribute_string_db$$system",
|
||||
MaterializedSemconv: true,
|
||||
SemconvMembers: []string{"db.system.name", "db.system"},
|
||||
}
|
||||
|
||||
expression, err := fm.FieldFor(context.Background(), valuer.UUID{}, 0, 0, key)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "`attribute_string_db$$system`", expression)
|
||||
|
||||
exists, err := querybuilder.ExistsExpression(
|
||||
[]*schema.Column{indexV3Columns["attributes_string"]}, key, 0, 0, expression, true,
|
||||
)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "`attribute_string_db$$system_exists`", exists)
|
||||
}
|
||||
|
||||
@@ -45,6 +45,10 @@ type ConditionBuilder interface {
|
||||
type ConditionBuilderOptions struct {
|
||||
// SkipResourceFilter drops the resource context from the candidate set.
|
||||
SkipResourceFilter bool
|
||||
// ExactSemconv disables semantic-convention family expansion. It is an
|
||||
// internal escape hatch for diagnostics and migrations that must address one
|
||||
// physical spelling only; public query APIs continue to resolve families.
|
||||
ExactSemconv bool
|
||||
}
|
||||
type AggExprRewriter interface {
|
||||
// Rewrite rewrites the aggregation expression to be used in the query.
|
||||
|
||||
@@ -25,10 +25,22 @@ type Result struct {
|
||||
}
|
||||
|
||||
type ExecStats struct {
|
||||
RowsScanned uint64 `json:"rowsScanned"`
|
||||
BytesScanned uint64 `json:"bytesScanned"`
|
||||
DurationMS uint64 `json:"durationMs"`
|
||||
StepIntervals map[string]uint64 `json:"stepIntervals,omitempty"`
|
||||
RowsScanned uint64 `json:"rowsScanned"`
|
||||
BytesScanned uint64 `json:"bytesScanned"`
|
||||
DurationMS uint64 `json:"durationMs"`
|
||||
StepIntervals map[string]uint64 `json:"stepIntervals,omitempty"`
|
||||
SemconvResolutions []SemconvResolution `json:"semconvResolutions,omitempty"`
|
||||
}
|
||||
|
||||
// SemconvResolution records a semantic-convention family that the query
|
||||
// builder resolved. Requested preserves the spelling supplied by the caller so
|
||||
// agents and editors can update their next query without changing response
|
||||
// labels in the current response.
|
||||
type SemconvResolution struct {
|
||||
Requested string `json:"requested"`
|
||||
Current string `json:"current"`
|
||||
Members []string `json:"members"`
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
|
||||
var _ jsonschema.Preparer = &ExecStats{}
|
||||
|
||||
@@ -46,6 +46,13 @@ type TelemetryFieldKey struct {
|
||||
JSONPlan JSONAccessPlan `json:"-"`
|
||||
Indexes []TelemetryFieldKeySkipIndex `json:"-"`
|
||||
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
|
||||
// MaterializedColumnName preserves the physical column when its DEFAULT
|
||||
// expression resolves a newer semantic-convention key than the historical
|
||||
// column identifier (for example db.system.name in db$$system).
|
||||
MaterializedColumnName string `json:"-"`
|
||||
// MaterializedSemconv is true when the column's DEFAULT expression already
|
||||
// coalesces every enabled family member and is therefore safe for a family query.
|
||||
MaterializedSemconv bool `json:"-"`
|
||||
|
||||
Evolutions []*EvolutionEntry `json:"-"`
|
||||
SemconvMembers []string `json:"-"`
|
||||
@@ -127,6 +134,8 @@ func (f *TelemetryFieldKey) OverrideMetadataFrom(src *TelemetryFieldKey) {
|
||||
f.FieldDataType = src.FieldDataType
|
||||
f.Indexes = src.Indexes
|
||||
f.Materialized = src.Materialized
|
||||
f.MaterializedColumnName = src.MaterializedColumnName
|
||||
f.MaterializedSemconv = src.MaterializedSemconv
|
||||
f.JSONPlan = src.JSONPlan
|
||||
f.Evolutions = src.Evolutions
|
||||
f.SemconvMembers = src.SemconvMembers
|
||||
@@ -204,6 +213,9 @@ func TelemetryFieldKeyToText(key *TelemetryFieldKey) string {
|
||||
}
|
||||
|
||||
func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string {
|
||||
if key.MaterializedColumnName != "" {
|
||||
return fmt.Sprintf("`%s`", key.MaterializedColumnName)
|
||||
}
|
||||
return fmt.Sprintf("`%s_%s_%s`",
|
||||
key.FieldContext.String,
|
||||
fieldDataTypes[key.FieldDataType.StringValue()].StringValue(),
|
||||
@@ -212,6 +224,9 @@ func FieldKeyToMaterializedColumnName(key *TelemetryFieldKey) string {
|
||||
}
|
||||
|
||||
func FieldKeyToMaterializedColumnNameForExists(key *TelemetryFieldKey) string {
|
||||
if key.MaterializedColumnName != "" {
|
||||
return fmt.Sprintf("`%s_exists`", key.MaterializedColumnName)
|
||||
}
|
||||
return fmt.Sprintf("`%s_%s_%s_exists`",
|
||||
key.FieldContext.String,
|
||||
fieldDataTypes[key.FieldDataType.StringValue()].StringValue(),
|
||||
@@ -278,6 +293,28 @@ type GettableFieldValues struct {
|
||||
Complete bool `json:"complete" required:"true"`
|
||||
}
|
||||
|
||||
// PostableSemconvMigrationReportParams selects the metadata window used to
|
||||
// find services that still emit only historical semantic-convention names.
|
||||
type PostableSemconvMigrationReportParams struct {
|
||||
StartUnixMilli int64 `query:"startUnixMilli"`
|
||||
EndUnixMilli int64 `query:"endUnixMilli"`
|
||||
}
|
||||
|
||||
type SemconvMigrationReportEntry struct {
|
||||
Current string `json:"current"`
|
||||
Old string `json:"old"`
|
||||
Signal string `json:"signal"`
|
||||
Services []string `json:"services"`
|
||||
ResourceSets uint64 `json:"resourceSets"`
|
||||
LastSeenUnixMilli int64 `json:"lastSeenUnixMilli"`
|
||||
}
|
||||
|
||||
type GettableSemconvMigrationReport struct {
|
||||
StartUnixMilli int64 `json:"startUnixMilli"`
|
||||
EndUnixMilli int64 `json:"endUnixMilli"`
|
||||
Entries []*SemconvMigrationReportEntry `json:"entries" required:"true"`
|
||||
}
|
||||
|
||||
type PostableFieldValueParams struct {
|
||||
PostableFieldKeysParams
|
||||
Name string `query:"name"`
|
||||
|
||||
@@ -26,6 +26,10 @@ type MetadataStore interface {
|
||||
// GetAllValues returns a list of all values.
|
||||
GetAllValues(ctx context.Context, orgID valuer.UUID, fieldValueSelector *FieldValueSelector) (*TelemetryFieldValues, bool, error)
|
||||
|
||||
// GetSemconvMigrationReport returns services whose metadata contains an old
|
||||
// semantic-convention name but no current member of that family.
|
||||
GetSemconvMigrationReport(ctx context.Context, orgID valuer.UUID, startUnixMilli, endUnixMilli int64) (*GettableSemconvMigrationReport, error)
|
||||
|
||||
// FetchTemporality fetches the temporality for metric
|
||||
FetchTemporality(ctx context.Context, orgID valuer.UUID, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricName string) (metrictypes.Temporality, error)
|
||||
|
||||
|
||||
@@ -23,7 +23,19 @@ type MockMetadataStore struct {
|
||||
ColumnEvolutionMetadataMap map[string][]*telemetrytypes.EvolutionEntry
|
||||
LookupKeysMap map[telemetrytypes.MetricMetadataLookupKey]int64
|
||||
// StaticFields holds signal-specific intrinsic field definitions (e.g. logstelemetryschema.IntrinsicFields).
|
||||
StaticFields map[string]telemetrytypes.TelemetryFieldKey
|
||||
StaticFields map[string]telemetrytypes.TelemetryFieldKey
|
||||
SemconvMigrationReport *telemetrytypes.GettableSemconvMigrationReport
|
||||
}
|
||||
|
||||
func (m *MockMetadataStore) GetSemconvMigrationReport(_ context.Context, _ valuer.UUID, startUnixMilli, endUnixMilli int64) (*telemetrytypes.GettableSemconvMigrationReport, error) {
|
||||
if m.SemconvMigrationReport != nil {
|
||||
return m.SemconvMigrationReport, nil
|
||||
}
|
||||
return &telemetrytypes.GettableSemconvMigrationReport{
|
||||
StartUnixMilli: startUnixMilli,
|
||||
EndUnixMilli: endUnixMilli,
|
||||
Entries: []*telemetrytypes.SemconvMigrationReportEntry{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewMockMetadataStore creates a new instance of MockMetadataStore with initialized maps.
|
||||
|
||||
@@ -7,5 +7,31 @@ default_enabled: false
|
||||
families:
|
||||
deployment.environment.name:
|
||||
enabled: true
|
||||
contexts: [resource, attribute]
|
||||
signals: [traces, logs, metrics]
|
||||
db.system.name:
|
||||
enabled: true
|
||||
contexts: [resource, attribute]
|
||||
signals: [traces, logs, metrics]
|
||||
|
||||
# These metric renames predate the schema history vendored above. Keep them
|
||||
# in the same generated registry so every v5 metric query uses one source of
|
||||
# truth instead of the legacy hand-written transition table.
|
||||
k8s.pod.cpu.usage:
|
||||
enabled: true
|
||||
kind: metric
|
||||
old: [k8s.pod.cpu.utilization]
|
||||
contexts: [metric]
|
||||
signals: [metrics]
|
||||
k8s.node.cpu.usage:
|
||||
enabled: true
|
||||
kind: metric
|
||||
old: [k8s.node.cpu.utilization]
|
||||
contexts: [metric]
|
||||
signals: [metrics]
|
||||
container.cpu.usage:
|
||||
enabled: true
|
||||
kind: metric
|
||||
old: [container.cpu.utilization]
|
||||
contexts: [metric]
|
||||
signals: [metrics]
|
||||
|
||||
157
tests/integration/tests/queriersemconv/02_cross_signal.py
Normal file
157
tests/integration/tests/queriersemconv/02_cross_signal.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""Phase 2 semantic-convention checks across logs and metrics."""
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
||||
import requests
|
||||
|
||||
from fixtures import querier, types
|
||||
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
|
||||
from fixtures.logs import Logs
|
||||
from fixtures.metrics import Metrics
|
||||
|
||||
DB_CURRENT = "db.system.name"
|
||||
DB_OLD = "db.system"
|
||||
METRIC_CURRENT = "container.cpu.usage"
|
||||
METRIC_OLD = "container.cpu.utilization"
|
||||
PREFIX = "semconv-phase2"
|
||||
|
||||
|
||||
def _raw_log_bodies(
|
||||
signoz: types.SigNoz,
|
||||
token: str,
|
||||
now: datetime,
|
||||
expression: str,
|
||||
) -> set[str]:
|
||||
response = querier.make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=querier.RequestType.RAW,
|
||||
queries=[querier.build_raw_query("A", "logs", limit=100, filter_expression=expression)],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
return {row["data"]["body"] for row in querier.get_rows(response)}
|
||||
|
||||
|
||||
def _field_values(signoz: types.SigNoz, token: str, signal: str, name: str, context: str) -> set[str]:
|
||||
response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/values"),
|
||||
timeout=5,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"signal": signal,
|
||||
"name": name,
|
||||
"fieldContext": context,
|
||||
"fieldDataType": "string",
|
||||
},
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
return set(response.json()["data"]["values"].get("stringValues") or [])
|
||||
|
||||
|
||||
def test_logs_resolve_db_system_family(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_logs: Callable[[list[Logs]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(microsecond=0) - timedelta(minutes=1)
|
||||
rows = [
|
||||
("old", {DB_OLD: "postgresql"}),
|
||||
("current", {DB_CURRENT: "postgresql"}),
|
||||
("conflict", {DB_OLD: "mysql", DB_CURRENT: "postgresql"}),
|
||||
("missing", {}),
|
||||
]
|
||||
insert_logs(
|
||||
[
|
||||
Logs(
|
||||
timestamp=now + timedelta(seconds=index),
|
||||
resources={"service.name": PREFIX, **attributes},
|
||||
attributes=attributes,
|
||||
body=f"{PREFIX}-{suffix}",
|
||||
)
|
||||
for index, (suffix, attributes) in enumerate(rows)
|
||||
]
|
||||
)
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
present = {f"{PREFIX}-old", f"{PREFIX}-current", f"{PREFIX}-conflict"}
|
||||
for context in ("attribute", "resource"):
|
||||
for requested in (DB_CURRENT, DB_OLD):
|
||||
field = f"{context}.{requested}"
|
||||
assert _raw_log_bodies(signoz, token, now, f'{field} = "postgresql"') == present
|
||||
assert _raw_log_bodies(signoz, token, now, f"{field} EXISTS") == present
|
||||
assert _raw_log_bodies(signoz, token, now, f"{field} NOT EXISTS") == {f"{PREFIX}-missing"}
|
||||
assert _field_values(signoz, token, "logs", requested, context) == {"postgresql", "mysql"}
|
||||
|
||||
|
||||
def test_metrics_resolve_label_and_metric_name_families(
|
||||
signoz: types.SigNoz,
|
||||
create_user_admin: None, # pylint: disable=unused-argument
|
||||
get_token: Callable[[str, str], str],
|
||||
insert_metrics: Callable[[list[Metrics]], None],
|
||||
) -> None:
|
||||
now = datetime.now(tz=UTC).replace(second=0, microsecond=0)
|
||||
insert_metrics(
|
||||
[
|
||||
Metrics(
|
||||
metric_name=METRIC_CURRENT,
|
||||
labels={DB_CURRENT: "postgresql"},
|
||||
timestamp=now - timedelta(seconds=3),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=10,
|
||||
),
|
||||
Metrics(
|
||||
metric_name=METRIC_OLD,
|
||||
labels={"db_system": "mysql"},
|
||||
timestamp=now - timedelta(seconds=2),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=20,
|
||||
),
|
||||
Metrics(
|
||||
metric_name=METRIC_OLD,
|
||||
labels={DB_OLD: "mysql", DB_CURRENT: "postgresql", "series": "conflict"},
|
||||
timestamp=now - timedelta(seconds=1),
|
||||
temporality="Unspecified",
|
||||
type_="Gauge",
|
||||
is_monotonic=False,
|
||||
value=30,
|
||||
),
|
||||
]
|
||||
)
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
for metric_name in (METRIC_CURRENT, METRIC_OLD):
|
||||
for requested_label in (DB_CURRENT, DB_OLD):
|
||||
response = querier.make_scalar_query_request(
|
||||
signoz,
|
||||
token,
|
||||
now,
|
||||
[
|
||||
querier.build_scalar_query(
|
||||
name="A",
|
||||
signal="metrics",
|
||||
aggregations=[
|
||||
querier.build_metrics_aggregation(
|
||||
metric_name,
|
||||
"latest",
|
||||
"sum",
|
||||
"unspecified",
|
||||
reduce_to="last",
|
||||
)
|
||||
],
|
||||
group_by=[querier.build_group_by_field(requested_label, "string", "attribute")],
|
||||
filter_expression=f"attribute.{requested_label} EXISTS",
|
||||
)
|
||||
],
|
||||
)
|
||||
assert response.status_code == HTTPStatus.OK, response.text
|
||||
data = {row[0]: row[-1] for row in querier.get_scalar_table_data(response.json())}
|
||||
assert data == {"postgresql": 40.0, "mysql": 20.0}, (metric_name, requested_label, data)
|
||||
@@ -84,10 +84,7 @@ def semconv_phase1_data(
|
||||
yield now
|
||||
|
||||
cluster = clickhouse.env["SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER"]
|
||||
clickhouse.conn.command(
|
||||
f"ALTER TABLE signoz_traces.dependency_graph_minutes_v2 ON CLUSTER '{cluster}' "
|
||||
f"DELETE WHERE startsWith(src, '{PREFIX}-map-') SETTINGS mutations_sync = 1"
|
||||
)
|
||||
clickhouse.conn.command(f"ALTER TABLE signoz_traces.dependency_graph_minutes_v2 ON CLUSTER '{cluster}' DELETE WHERE startsWith(src, '{PREFIX}-map-') SETTINGS mutations_sync = 1")
|
||||
|
||||
|
||||
def _result(response: requests.Response) -> dict[str, Any]:
|
||||
@@ -148,6 +145,31 @@ def test_semconv_phase1_mixed_sdk_generations( # pylint: disable=too-many-state
|
||||
now = semconv_phase1_data
|
||||
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
|
||||
|
||||
# A builder response records the exact spelling that was resolved. Raw SQL
|
||||
# and PromQL deliberately do not use this resolver.
|
||||
resolution_response = make_query_request(
|
||||
signoz,
|
||||
token,
|
||||
start_ms=int((now - timedelta(minutes=2)).timestamp() * 1000),
|
||||
end_ms=int((now + timedelta(minutes=1)).timestamp() * 1000),
|
||||
request_type=RequestType.RAW,
|
||||
queries=[
|
||||
BuilderQuery(
|
||||
signal="traces",
|
||||
name="A",
|
||||
limit=1,
|
||||
filter_expression=f"resource.{OLD} EXISTS",
|
||||
).to_dict()
|
||||
],
|
||||
)
|
||||
assert resolution_response.status_code == HTTPStatus.OK, resolution_response.text
|
||||
assert {
|
||||
"requested": OLD,
|
||||
"current": CURRENT,
|
||||
"members": [CURRENT, OLD],
|
||||
"kind": "attribute",
|
||||
} in resolution_response.json()["data"]["meta"]["semconvResolutions"]
|
||||
|
||||
# Resource and span-attribute paths share the same matrix. Run every
|
||||
# operator with both the saved-query (old) and current request spellings.
|
||||
for context in ("resource", "attribute"):
|
||||
@@ -237,3 +259,16 @@ def test_semconv_phase1_mixed_sdk_generations( # pylint: disable=too-many-state
|
||||
)
|
||||
assert map_response.status_code == HTTPStatus.OK, map_response.text
|
||||
assert {edge["parent"] for edge in map_response.json()} == {f"{PREFIX}-map-production"}
|
||||
|
||||
report_response = requests.get(
|
||||
signoz.self.host_configs["8080"].get("/api/v1/fields/semconv-migration"),
|
||||
timeout=30,
|
||||
headers={"authorization": f"Bearer {token}"},
|
||||
params={
|
||||
"startUnixMilli": int((now - timedelta(minutes=2)).timestamp() * 1000),
|
||||
"endUnixMilli": int((now + timedelta(minutes=1)).timestamp() * 1000),
|
||||
},
|
||||
)
|
||||
assert report_response.status_code == HTTPStatus.OK, report_response.text
|
||||
entry = next(item for item in report_response.json()["data"]["entries"] if item["current"] == CURRENT and item["old"] == OLD and item["signal"] == "traces")
|
||||
assert set(entry["services"]) == {f"{PREFIX}-old", f"{PREFIX}-staging"}
|
||||
|
||||
Reference in New Issue
Block a user