Compare commits

..

1 Commits

Author SHA1 Message Date
Vinícius Lourenço
d5971c59da chore(yarn): bump to v4 2026-04-26 15:57:09 -03:00
37 changed files with 24101 additions and 20198 deletions

View File

@@ -23,7 +23,7 @@ jobs:
node-version: lts/*
- name: install
run: |
cd tests/e2e && yarn install --frozen-lockfile
cd tests/e2e && yarn install --immutable
- name: fmt
run: |
cd tests/e2e && yarn fmt:check
@@ -59,7 +59,7 @@ jobs:
cd tests && uv sync
- name: yarn-install
run: |
cd tests/e2e && yarn install --frozen-lockfile
cd tests/e2e && yarn install --immutable
- name: playwright-browsers
run: |
cd tests/e2e && yarn playwright install --with-deps ${{ matrix.project }}

View File

@@ -79,7 +79,8 @@ jobs:
- name: deps-install
working-directory: ./frontend
run: |
yarn install
corepack enable
yarn install --immutable
- name: uv-install
uses: astral-sh/setup-uv@v5
- name: uv-deps
@@ -117,7 +118,7 @@ jobs:
with:
node-version: "22"
- name: install-frontend
run: cd frontend && yarn install
run: cd frontend && corepack enable && yarn install --immutable
- name: generate-api-clients
run: |
cd frontend && yarn generate:api

5
.gitignore vendored
View File

@@ -10,9 +10,14 @@ node_modules
deploy/docker/environment_tiny/common_test
frontend/node_modules
frontend/.pnp
frontend/.pnp.*
frontend/i18n-translations-hash.json
*.pnp.js
# Yarn Berry
frontend/.yarn/cache
frontend/.yarn/install-state.gz
# testing
frontend/coverage

View File

@@ -2933,8 +2933,8 @@ components:
type: object
PromotetypesWrappedIndex:
properties:
fieldDataType:
$ref: '#/components/schemas/TelemetrytypesFieldDataType'
column_type:
type: string
granularity:
type: integer
type:

View File

@@ -5,7 +5,12 @@ import (
"fmt"
"net/http"
"github.com/SigNoz/signoz/ee/query-service/constants"
"github.com/SigNoz/signoz/ee/query-service/model"
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/types/authtypes"
"github.com/SigNoz/signoz/pkg/types/featuretypes"
"github.com/SigNoz/signoz/pkg/valuer"
)
type DayWiseBreakdown struct {
@@ -65,18 +70,53 @@ func (ah *APIHandler) getBilling(w http.ResponseWriter, r *http.Request) {
return
}
data, err := ah.Signoz.Zeus.GetMeters(r.Context(), licenseKey)
claims, err := authtypes.ClaimsFromContext(r.Context())
if err != nil {
RespondError(w, model.InternalError(err), nil)
return
}
var billing billingData
if err := json.Unmarshal(data, &billing); err != nil {
orgID := valuer.MustNewUUID(claims.OrgID)
evalCtx := featuretypes.NewFlaggerEvaluationContext(orgID)
useZeus := ah.Signoz.Flagger.BooleanOrEmpty(r.Context(), flagger.FeatureGetMetersFromZeus, evalCtx)
if useZeus {
data, err := ah.Signoz.Zeus.GetMeters(r.Context(), licenseKey)
if err != nil {
RespondError(w, model.InternalError(err), nil)
return
}
var billing billingData
if err := json.Unmarshal(data, &billing); err != nil {
RespondError(w, model.InternalError(err), nil)
return
}
ah.Respond(w, billing)
return
}
billingURL := fmt.Sprintf("%s/usage?licenseKey=%s", constants.LicenseSignozIo, licenseKey)
hClient := &http.Client{}
req, err := http.NewRequest("GET", billingURL, nil)
if err != nil {
RespondError(w, model.InternalError(err), nil)
return
}
req.Header.Add("X-SigNoz-SecretKey", constants.LicenseAPIKey)
billingResp, err := hClient.Do(req)
if err != nil {
RespondError(w, model.InternalError(err), nil)
return
}
ah.Respond(w, billing)
return
var billingResponse billingDetails
if err := json.NewDecoder(billingResp.Body).Decode(&billingResponse); err != nil {
RespondError(w, model.InternalError(err), nil)
return
}
ah.Respond(w, billingResponse.Data)
}

View File

@@ -1,2 +0,0 @@
network-timeout 600000
save-prefix ""

9
frontend/.yarnrc.yml Normal file
View File

@@ -0,0 +1,9 @@
nodeLinker: node-modules
httpTimeout: 600000
defaultSemverRangePrefix: ""
enableGlobalCache: false
enableScripts: true

View File

@@ -246,6 +246,8 @@
"resolutions": {
"@types/react": "18.0.26",
"@types/react-dom": "18.0.10",
"@types/styled-components": "5.1.26",
"@types/lodash": "4.14.194",
"debug": "4.3.4",
"semver": "7.5.4",
"xml2js": "0.5.0",
@@ -262,5 +264,6 @@
"on-headers": "^1.1.0",
"tmp": "0.2.4",
"vite": "npm:rolldown-vite@7.3.1"
}
}
},
"packageManager": "yarn@4.14.1"
}

View File

@@ -3701,7 +3701,10 @@ export interface PromotetypesPromotePathDTO {
}
export interface PromotetypesWrappedIndexDTO {
fieldDataType?: TelemetrytypesFieldDataTypeDTO;
/**
* @type string
*/
column_type?: string;
/**
* @type integer
*/

View File

@@ -1,3 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" fill="currentColor" height="34" viewBox="0 0 131 34">
<path fill="currentColor" d="M.36 8.6h16.7v5.6H6.04c-.2 0-.35.16-.35.35v4.9c0 .2.16.35.35.35h11.02v5.6h-5.33c-.2 0-.35.16-.35.35v4.9c0 .2.16.35.35.35h4.98c.2 0 .35-.15.35-.35V25.4h5.34c.2 0 .35-.16.35-.35v-4.9c0-.2-.16-.35-.35-.35h-5.34v-5.6h5.34c.2 0 .35-.16.35-.35v-4.9c0-.2-.16-.35-.35-.35h-5.34V3.35c0-.2-.16-.35-.35-.35H.36c-.2 0-.36.16-.36.35v4.9c0 .2.16.35.36.35ZM44.41 14.7c-.5-.5-1.1-.9-1.76-1.18a5.62 5.62 0 0 0-4.6.17c-.73.37-1.32.91-1.75 1.62h-.17V8.59H34.1v16.83h2.04v-1.81h.17c.21.36.47.67.77.94.31.25.65.48 1.01.67.37.18.77.31 1.18.39a6.2 6.2 0 0 0 3.39-.24 5.36 5.36 0 0 0 3.02-3.1c.29-.75.44-1.62.44-2.6v-.47c0-.96-.16-1.83-.47-2.58-.3-.75-.7-1.4-1.23-1.9v-.01Zm-5.87.66a3.9 3.9 0 0 1 4.34.84c.36.35.64.8.83 1.3.2.5.3 1.07.3 1.7v.47c0 .64-.1 1.23-.3 1.74a3.75 3.75 0 0 1-2.06 2.15 4.27 4.27 0 0 1-3.12-.03 3.86 3.86 0 0 1-2.09-2.2c-.2-.52-.3-1.11-.3-1.75v-.29c0-.62.1-1.2.3-1.7v-.01c.21-.53.5-.99.84-1.36.36-.37.78-.66 1.26-.86ZM97.04 8.59H95v4.86h-2.94v1.86H95v8.17c0 .56.17 1.03.53 1.4.37.35.84.54 1.4.54h4.18v-1.87h-3.5c-.2 0-.33-.05-.43-.15-.1-.1-.14-.27-.14-.49v-7.6h4.65v-1.86h-4.65V8.59ZM114.61 15a5.48 5.48 0 0 0-1.8-1.33 5.6 5.6 0 0 0-2.57-.56 6.17 6.17 0 0 0-4.26 1.7 5.6 5.6 0 0 0-1.72 4.2v.57c0 .9.15 1.75.44 2.5a5.58 5.58 0 0 0 5.5 3.67c1.55 0 2.8-.35 3.72-1.04a5.35 5.35 0 0 0 1.91-2.73l.03-.07-1.94-.52-.02.07c-.11.33-.27.64-.46.94-.17.27-.4.52-.7.74-.28.22-.63.39-1.04.51-.41.13-.9.19-1.46.19a3.8 3.8 0 0 1-2.84-1.05 4.07 4.07 0 0 1-1.1-2.7h9.68v-1.6c0-.54-.11-1.12-.34-1.75a5.04 5.04 0 0 0-1.03-1.74Zm-8.25 3.21a3.8 3.8 0 0 1 1.22-2.25 4.19 4.19 0 0 1 3.99-.7c.44.16.83.38 1.17.66.34.27.62.62.82 1.02.21.38.34.8.38 1.27h-7.58ZM129.09 14.42a4.47 4.47 0 0 0-3.37-1.3c-.93 0-1.73.2-2.4.59-.64.39-1.15.97-1.52 1.74h-.17v-2h-2.04v11.97h2.04v-6.23c0-1.26.32-2.28.95-3.02a3.31 3.31 0 0 1 2.65-1.14c.94 0 1.7.3 2.24.9.56.6.83 1.52.83 2.74v6.75h2.04v-7.13c0-1.71-.42-3.02-1.25-3.87ZM88.1 15a5.48 5.48 0 0 0-1.78-1.33 5.6 5.6 0 0 0-2.58-.56 6.17 6.17 0 0 0-4.27 1.7 5.59 5.59 0 0 0-1.71 4.2v.56c0 .92.14 1.76.44 2.51a5.6 5.6 0 0 0 5.5 3.67c1.55 0 2.8-.35 3.72-1.04a5.36 5.36 0 0 0 1.91-2.73l.03-.07-1.94-.52-.03.07c-.1.32-.26.64-.45.94-.17.27-.4.52-.7.74-.29.21-.64.39-1.05.51-.4.12-.9.19-1.45.19a3.8 3.8 0 0 1-2.85-1.05 4.07 4.07 0 0 1-1.09-2.7h9.68v-1.61c0-.53-.12-1.12-.34-1.74A5.03 5.03 0 0 0 88.1 15Zm-8.24 3.21a3.83 3.83 0 0 1 1.22-2.25 4.2 4.2 0 0 1 3.99-.7c.44.16.83.38 1.16.66.35.27.62.62.83 1.02.2.38.33.8.37 1.27h-7.57ZM73.65 19.42a6.11 6.11 0 0 0-3.23-1.02 6.63 6.63 0 0 1-2.68-.58c-.47-.3-.7-.7-.7-1.25 0-.27.08-.5.21-.7.14-.2.33-.38.56-.52a4.05 4.05 0 0 1 1.78-.42c.85 0 1.54.21 2.06.63.53.41.83 1 .91 1.73l.01.1 1.95-.47-.01-.07c-.07-.45-.22-.91-.45-1.36a3.46 3.46 0 0 0-.94-1.2 4.6 4.6 0 0 0-1.52-.84 6.05 6.05 0 0 0-2.1-.34c-.58 0-1.13.07-1.65.22-.52.14-1 .36-1.43.65-.41.3-.74.66-.99 1.1a2.9 2.9 0 0 0-.37 1.49v.14c0 1.06.38 1.87 1.14 2.42a6.2 6.2 0 0 0 3.24.97c1.18.08 2.04.26 2.56.54.5.28.75.72.75 1.36 0 .6-.25 1.06-.78 1.4-.52.32-1.22.48-2.1.48a3.68 3.68 0 0 1-2.46-.79 3.13 3.13 0 0 1-1.04-2.14v-.09l-1.93.46h-.02v.07a4.4 4.4 0 0 0 3.1 4c.7.24 1.52.36 2.46.36.7 0 1.35-.09 1.93-.27.6-.16 1.12-.4 1.53-.72A3.38 3.38 0 0 0 74.8 22v-.14c0-1.07-.39-1.9-1.14-2.44ZM60.25 23.4c-.1-.1-.14-.27-.14-.49v-9.46h-2.05v1.85h-.16a3.78 3.78 0 0 0-1.61-1.63 4.62 4.62 0 0 0-2.26-.56c-.77 0-1.5.14-2.19.41a5.27 5.27 0 0 0-3.02 3.12c-.29.75-.44 1.63-.44 2.6v.38c0 .99.15 1.87.44 2.63.3.75.7 1.4 1.2 1.93a5.48 5.48 0 0 0 4.05 1.57c.8 0 1.51-.2 2.2-.58.69-.38 1.24-.97 1.63-1.75h.16v.06c0 .56.18 1.03.53 1.4.37.35.85.54 1.41.54h1.36v-1.87h-.68c-.2 0-.34-.05-.43-.15Zm-4.46.13c-.46.2-.97.3-1.52.3a3.68 3.68 0 0 1-2.75-1.09 4.42 4.42 0 0 1-1.05-3.12v-.38c0-.62.1-1.2.29-1.71a3.65 3.65 0 0 1 5-2.17c.47.2.88.49 1.21.86.35.37.62.83.8 1.36.2.5.3 1.08.3 1.7v.3c0 .63-.1 1.23-.3 1.76-.18.5-.45.96-.78 1.33-.33.37-.73.66-1.2.86Z"/>
</svg>

Before

Width:  |  Height:  |  Size: 3.9 KiB

View File

@@ -1,13 +0,0 @@
<svg width="109" height="24" viewBox="0 0 109 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<g clip-path="url(#clip0_125_22125)">
<path d="M0 -2.08616e-07V24H17.9352C22.9911 24 26.0999 21.0858 26.0999 17.04V6.96C26.0999 2.91432 22.9911 -2.08616e-07 17.9352 -2.08616e-07H0ZM6.76413 5.82864H19.1992V18.1714H6.76413V5.82864Z" fill="currentColor"/>
<path d="M46.7659 18.6172H35.0824V14.16H46.7659V18.6172ZM46.595 5.38296V9.5658H35.0824V5.38296H46.595ZM50.2846 12.1373V11.5886C52.5734 10.5258 53.7008 8.8458 53.7008 6.13728C53.7008 2.64012 50.9337 0.000116183 45.5361 0.000116183H28.3184V24H45.7752C51.1728 24 53.9399 21.8401 53.9399 18.0685C53.9399 15.0172 52.6418 13.2001 50.2846 12.1373Z" fill="currentColor"/>
<path d="M62.397 18.1714H74.8319V5.82864H62.397V18.1714ZM63.6609 24C58.6049 24 55.4961 21.0858 55.4961 17.04V6.96012C55.4961 2.91432 58.6049 0.000116183 63.6609 0.000116183H73.568C78.6238 0.000116183 81.7326 2.91432 81.7326 6.96012V17.04C81.7326 21.0858 78.6238 24 73.568 24H63.6609Z" fill="currentColor"/>
<path d="M101.66 15.12L90.8995 14.3658C85.5361 13.9886 83.418 11.1772 83.418 7.47432V6.96012C83.418 2.91432 86.5266 0.000116183 91.5827 0.000116183H100.157C105.214 0.000116183 108.323 2.91432 108.323 6.96012V7.98864H101.968V5.14284H90.2504V8.43432L100.601 9.18864C105.999 9.56568 108.493 12.8572 108.493 16.5257V17.04C108.493 20.7428 105.384 24 100.328 24H91.5827C86.5266 24 83.418 20.7428 83.418 17.04V16.0115H89.7722V18.8572H101.66V15.12Z" fill="currentColor"/>
</g>
<defs>
<clipPath id="clip0_125_22125">
<rect width="108.494" height="24" fill="white"/>
</clipPath>
</defs>
</svg>

Before

Width:  |  Height:  |  Size: 1.6 KiB

View File

@@ -1 +0,0 @@
<svg width='234' height='42' viewBox='0 0 234 42' fill='none' xmlns='http://www.w3.org/2000/svg'><path d='M18 30H6V18H18V30Z' fill='#CFCECD'/><path d='M18 12H6V30H18V12ZM24 36H0V6H24V36Z' fill='#656363'/><path d='M48 30H36V18H48V30Z' fill='#CFCECD'/><path d='M36 30H48V12H36V30ZM54 36H36V42H30V6H54V36Z' fill='#656363'/><path d='M84 24V30H66V24H84Z' fill='#CFCECD'/><path d='M84 24H66V30H84V36H60V6H84V24ZM66 18H78V12H66V18Z' fill='#656363'/><path d='M108 36H96V18H108V36Z' fill='#CFCECD'/><path d='M108 12H96V36H90V6H108V12ZM114 36H108V12H114V36Z' fill='#656363'/><path d='M144 30H126V18H144V30Z' fill='#CFCECD'/><path d='M144 12H126V30H144V36H120V6H144V12Z' fill='#211E1E'/><path d='M168 30H156V18H168V30Z' fill='#CFCECD'/><path d='M168 12H156V30H168V12ZM174 36H150V6H174V36Z' fill='#211E1E'/><path d='M198 30H186V18H198V30Z' fill='#CFCECD'/><path d='M198 12H186V30H198V12ZM204 36H180V6H198V0H204V36Z' fill='#211E1E'/><path d='M234 24V30H216V24H234Z' fill='#CFCECD'/><path d='M216 12V18H228V12H216ZM234 24H216V30H234V36H210V6H234V24Z' fill='#211E1E'/></svg>

Before

Width:  |  Height:  |  Size: 1.0 KiB

View File

@@ -10,7 +10,6 @@ import cx from 'classnames';
import { LogType } from 'components/Logs/LogStateIndicator/LogStateIndicator';
import QuerySearch from 'components/QueryBuilderV2/QueryV2/QuerySearch/QuerySearch';
import { convertExpressionToFilters } from 'components/QueryBuilderV2/utils';
import { FeatureKeys } from 'constants/features';
import { LOCALSTORAGE } from 'constants/localStorage';
import { QueryParams } from 'constants/query';
import { initialQueriesMap, PANEL_TYPES } from 'constants/queryBuilder';
@@ -47,7 +46,6 @@ import {
TextSelect,
X,
} from 'lucide-react';
import { useAppContext } from 'providers/App/App';
import { AppState } from 'store/reducers';
import { Query, TagFilter } from 'types/api/queryBuilder/queryBuilderData';
import { DataSource, StringOperators } from 'types/common/queryBuilder';
@@ -81,10 +79,6 @@ function LogDetailInner({
const [selectedView, setSelectedView] = useState<VIEWS>(selectedTab);
const [isFilterVisible, setIsFilterVisible] = useState<boolean>(false);
const { featureFlags } = useAppContext();
const isBodyJsonQueryEnabled =
featureFlags?.find((flag) => flag.name === FeatureKeys.BODY_JSON_ENABLED)
?.active || false;
const [filters, setFilters] = useState<TagFilter | null>(null);
const [isEdit, setIsEdit] = useState<boolean>(false);
@@ -214,29 +208,11 @@ function LogDetailInner({
}
};
const logBody = useMemo(() => {
if (!isBodyJsonQueryEnabled) {
return log?.body || '';
}
try {
const json = JSON.parse(log?.body || '');
if (typeof json?.message === 'string' && json.message !== '') {
return json.message;
}
return log?.body || '';
} catch (error) {
return log?.body || '';
}
}, [isBodyJsonQueryEnabled, log?.body]);
const htmlBody = useMemo(
() => ({
__html: getSanitizedLogBody(logBody || '', { shouldEscapeHtml: true }),
__html: getSanitizedLogBody(log?.body || '', { shouldEscapeHtml: true }),
}),
[logBody],
[log?.body],
);
const handleJSONCopy = (): void => {
@@ -442,7 +418,7 @@ function LogDetailInner({
<div className="log-detail-drawer__log">
<Divider type="vertical" className={cx('log-type-indicator', logType)} />
<Tooltip
title={removeEscapeCharacters(logBody)}
title={removeEscapeCharacters(log?.body)}
placement="left"
mouseLeaveDelay={0}
>

View File

@@ -9,5 +9,4 @@ export enum FeatureKeys {
ANOMALY_DETECTION = 'anomaly_detection',
ONBOARDING_V3 = 'onboarding_v3',
DOT_METRICS_ENABLED = 'dot_metrics_enabled',
BODY_JSON_ENABLED = 'body_json_enabled',
}

View File

@@ -8,19 +8,8 @@ import {
OPERATORS,
QUERY_BUILDER_FUNCTIONS,
} from 'constants/antlrQueryConstants';
import { FeatureKeys } from 'constants/features';
import { QueryParams } from 'constants/query';
import { useActiveLog } from 'hooks/logs/useActiveLog';
import { useGetSearchQueryParam } from 'hooks/queryBuilder/useGetSearchQueryParam';
import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
import { ICurrentQueryData } from 'hooks/useHandleExplorerTabChange';
import { useNotifications } from 'hooks/useNotifications';
import { ExplorerViews } from 'pages/LogsExplorer/utils';
import { useAppContext } from 'providers/App/App';
import {
BaseAutocompleteData,
DataTypes,
} from 'types/api/queryBuilder/queryAutocompleteResponse';
import { TitleWrapper } from './BodyTitleRenderer.styles';
import { DROPDOWN_KEY } from './constant';
@@ -36,32 +25,17 @@ function BodyTitleRenderer({
parentIsArray = false,
nodeKey,
value,
handleChangeSelectedView,
}: BodyTitleRendererProps): JSX.Element {
const { onAddToQuery } = useActiveLog();
const { stagedQuery, updateQueriesData } = useQueryBuilder();
const { featureFlags } = useAppContext();
const [, setCopy] = useCopyToClipboard();
const { notifications } = useNotifications();
const viewName = useGetSearchQueryParam(QueryParams.viewName) || '';
const cleanedNodeKey = removeObjectFromString(nodeKey);
const isBodyJsonQueryEnabled =
featureFlags?.find((flag) => flag.name === FeatureKeys.BODY_JSON_ENABLED)
?.active || false;
// Group by is supported only for body json query enabled and not for array elements
const isGroupBySupported =
isBodyJsonQueryEnabled && !cleanedNodeKey.includes('[]');
const filterHandler = (isFilterIn: boolean) => (): void => {
if (parentIsArray) {
onAddToQuery(
generateFieldKeyForArray(
cleanedNodeKey,
removeObjectFromString(nodeKey),
getDataTypes(value),
isBodyJsonQueryEnabled,
),
`${value}`,
isFilterIn
@@ -71,7 +45,7 @@ function BodyTitleRenderer({
);
} else {
onAddToQuery(
`body.${cleanedNodeKey}`,
`body.${removeObjectFromString(nodeKey)}`,
`${value}`,
isFilterIn ? OPERATORS['='] : OPERATORS['!='],
getDataTypes(value),
@@ -79,67 +53,10 @@ function BodyTitleRenderer({
}
};
const groupByHandler = useCallback((): void => {
if (!stagedQuery) {
return;
}
const groupByKey = parentIsArray
? generateFieldKeyForArray(
cleanedNodeKey,
getDataTypes(value),
isBodyJsonQueryEnabled,
)
: `body.${cleanedNodeKey}`;
const fieldDataType = getDataTypes(value);
const normalizedDataType: DataTypes | undefined = Object.values(
DataTypes,
).includes(fieldDataType as DataTypes)
? (fieldDataType as DataTypes)
: undefined;
const updatedQuery = updateQueriesData(
stagedQuery,
'queryData',
(item, index) => {
if (index === 0) {
const newGroupByItem: BaseAutocompleteData = {
key: groupByKey,
type: '',
dataType: normalizedDataType,
};
return { ...item, groupBy: [...(item.groupBy || []), newGroupByItem] };
}
return item;
},
);
const queryData: ICurrentQueryData = {
name: viewName,
id: updatedQuery.id,
query: updatedQuery,
};
handleChangeSelectedView?.(ExplorerViews.TIMESERIES, queryData);
}, [
cleanedNodeKey,
handleChangeSelectedView,
isBodyJsonQueryEnabled,
parentIsArray,
stagedQuery,
updateQueriesData,
value,
viewName,
]);
const onClickHandler: MenuProps['onClick'] = (props): void => {
const mapper = {
[DROPDOWN_KEY.FILTER_IN]: filterHandler(true),
[DROPDOWN_KEY.FILTER_OUT]: filterHandler(false),
[DROPDOWN_KEY.GROUP_BY]: groupByHandler,
};
const handler = mapper[props.key];
@@ -159,14 +76,6 @@ function BodyTitleRenderer({
key: DROPDOWN_KEY.FILTER_OUT,
label: `Filter out ${value}`,
},
...(isGroupBySupported
? [
{
key: DROPDOWN_KEY.GROUP_BY,
label: `Group by ${nodeKey}`,
},
]
: []),
],
onClick: onClickHandler,
};
@@ -175,6 +84,7 @@ function BodyTitleRenderer({
(e: React.MouseEvent): void => {
// Prevent tree node expansion/collapse
e.stopPropagation();
const cleanedKey = removeObjectFromString(nodeKey);
let copyText: string;
// Check if value is an object or array
@@ -196,8 +106,8 @@ function BodyTitleRenderer({
if (copyText) {
const notificationMessage = isObject
? `${cleanedNodeKey} object copied to clipboard`
: `${cleanedNodeKey} copied to clipboard`;
? `${cleanedKey} object copied to clipboard`
: `${cleanedKey} copied to clipboard`;
notifications.success({
message: notificationMessage,
@@ -205,7 +115,7 @@ function BodyTitleRenderer({
});
}
},
[cleanedNodeKey, parentIsArray, setCopy, value, notifications],
[nodeKey, parentIsArray, setCopy, value, notifications],
);
return (

View File

@@ -1,4 +1,3 @@
import { ChangeViewFunctionType } from 'container/ExplorerOptions/types';
import { MetricsType } from 'container/MetricsApplication/constant';
import { ILog } from 'types/api/logs/log';
@@ -7,7 +6,6 @@ export interface BodyTitleRendererProps {
nodeKey: string;
value: unknown;
parentIsArray?: boolean;
handleChangeSelectedView?: ChangeViewFunctionType;
}
export type AnyObject = { [key: string]: any };

View File

@@ -2,7 +2,6 @@ import React, { useCallback, useMemo, useState } from 'react';
import { useLocation } from 'react-router-dom';
import { Color } from '@signozhq/design-tokens';
import { Button, Popover, Spin, Tooltip, Tree } from 'antd';
import type { DataNode } from 'antd/es/tree';
import GroupByIcon from 'assets/CustomIcons/GroupByIcon';
import cx from 'classnames';
import CopyClipboardHOC from 'components/Logs/CopyClipboardHOC';
@@ -58,7 +57,7 @@ interface ITableViewActionsProps {
}
// Memoized Tree Component
const MemoizedTree = React.memo<{ treeData: DataNode[] }>(({ treeData }) => (
const MemoizedTree = React.memo<{ treeData: any[] }>(({ treeData }) => (
<Tree
defaultExpandAll
showLine
@@ -75,54 +74,50 @@ const BodyContent: React.FC<{
record: DataType;
bodyHtml: { __html: string };
textToCopy: string;
handleChangeSelectedView?: ChangeViewFunctionType;
}> = React.memo(
({ fieldData, record, bodyHtml, textToCopy, handleChangeSelectedView }) => {
const { isLoading, treeData, error } = useAsyncJSONProcessing(
fieldData.value,
record.field === 'body',
handleChangeSelectedView,
}> = React.memo(({ fieldData, record, bodyHtml, textToCopy }) => {
const { isLoading, treeData, error } = useAsyncJSONProcessing(
fieldData.value,
record.field === 'body',
);
// Show JSON tree if available, otherwise show HTML content
if (record.field === 'body' && treeData) {
return <MemoizedTree treeData={treeData} />;
}
if (record.field === 'body' && isLoading) {
return (
<div style={{ display: 'flex', alignItems: 'center', gap: '8px' }}>
<Spin size="small" />
<span style={{ color: Color.BG_SIENNA_400 }}>Processing JSON...</span>
</div>
);
}
// Show JSON tree if available, otherwise show HTML content
if (record.field === 'body' && treeData) {
return <MemoizedTree treeData={treeData} />;
}
if (record.field === 'body' && error) {
return (
<span
style={{ color: Color.BG_SIENNA_400, whiteSpace: 'pre-wrap', tabSize: 4 }}
>
Error parsing Body JSON
</span>
);
}
if (record.field === 'body' && isLoading) {
return (
<div style={{ display: 'flex', alignItems: 'center', gap: '8px' }}>
<Spin size="small" />
<span style={{ color: Color.BG_SIENNA_400 }}>Processing JSON...</span>
</div>
);
}
if (record.field === 'body' && error) {
return (
if (record.field === 'body') {
return (
<CopyClipboardHOC entityKey="body" textToCopy={textToCopy}>
<span
style={{ color: Color.BG_SIENNA_400, whiteSpace: 'pre-wrap', tabSize: 4 }}
>
Error parsing Body JSON
<span dangerouslySetInnerHTML={bodyHtml} />
</span>
);
}
</CopyClipboardHOC>
);
}
if (record.field === 'body') {
return (
<CopyClipboardHOC entityKey="body" textToCopy={textToCopy}>
<span
style={{ color: Color.BG_SIENNA_400, whiteSpace: 'pre-wrap', tabSize: 4 }}
>
<span dangerouslySetInnerHTML={bodyHtml} />
</span>
</CopyClipboardHOC>
);
}
return null;
},
);
return null;
});
BodyContent.displayName = 'BodyContent';
@@ -324,7 +319,6 @@ export default function TableViewActions(
record={record}
bodyHtml={bodyHtml}
textToCopy={textToCopy}
handleChangeSelectedView={handleChangeSelectedView}
/>
);
@@ -348,7 +342,6 @@ export default function TableViewActions(
fieldData,
bodyHtml,
textToCopy,
handleChangeSelectedView,
formatTimezoneAdjustedTimestamp,
cleanTimestamp,
]);
@@ -362,7 +355,6 @@ export default function TableViewActions(
record={record}
bodyHtml={bodyHtml}
textToCopy={textToCopy}
handleChangeSelectedView={handleChangeSelectedView}
/>
{!isListViewPanel &&
!RESTRICTED_SELECTED_FIELDS.includes(fieldFilterKey) && (

View File

@@ -1,8 +1,5 @@
import { useEffect, useRef, useState } from 'react';
import { FeatureKeys } from 'constants/features';
import { ChangeViewFunctionType } from 'container/ExplorerOptions/types';
import { isEmpty } from 'lodash-es';
import { useAppContext } from 'providers/App/App';
import { jsonToDataNodes, recursiveParseJSON } from '../utils';
@@ -12,7 +9,6 @@ const MAX_BODY_BYTES = 100 * 1024; // 100 KB
const useAsyncJSONProcessing = (
value: string,
shouldProcess: boolean,
handleChangeSelectedView?: ChangeViewFunctionType,
): {
isLoading: boolean;
treeData: any[] | null;
@@ -29,10 +25,6 @@ const useAsyncJSONProcessing = (
});
const processingRef = useRef<boolean>(false);
const { featureFlags } = useAppContext();
const isBodyJsonQueryEnabled =
featureFlags?.find((flag) => flag.name === FeatureKeys.BODY_JSON_ENABLED)
?.active || false;
// eslint-disable-next-line sonarjs/cognitive-complexity
useEffect((): (() => void) => {
@@ -55,10 +47,7 @@ const useAsyncJSONProcessing = (
try {
const parsedBody = recursiveParseJSON(value);
if (!isEmpty(parsedBody)) {
const treeData = jsonToDataNodes(parsedBody, {
isBodyJsonQueryEnabled,
handleChangeSelectedView,
});
const treeData = jsonToDataNodes(parsedBody);
setJsonState({ isLoading: false, treeData, error: null });
} else {
setJsonState({ isLoading: false, treeData: null, error: null });
@@ -84,10 +73,7 @@ const useAsyncJSONProcessing = (
try {
const parsedBody = recursiveParseJSON(value);
if (!isEmpty(parsedBody)) {
const treeData = jsonToDataNodes(parsedBody, {
isBodyJsonQueryEnabled,
handleChangeSelectedView,
});
const treeData = jsonToDataNodes(parsedBody);
setJsonState({ isLoading: false, treeData, error: null });
} else {
setJsonState({ isLoading: false, treeData: null, error: null });
@@ -115,7 +101,7 @@ const useAsyncJSONProcessing = (
return (): void => {
processingRef.current = false;
};
}, [value, shouldProcess, isBodyJsonQueryEnabled, handleChangeSelectedView]);
}, [value, shouldProcess]);
return jsonState;
};

View File

@@ -1,5 +1,4 @@
export const DROPDOWN_KEY = {
FILTER_IN: 'filterIn',
FILTER_OUT: 'filterOut',
GROUP_BY: 'groupBy',
};

View File

@@ -1,6 +1,5 @@
import Convert from 'ansi-to-html';
import type { DataNode } from 'antd/es/tree';
import { ChangeViewFunctionType } from 'container/ExplorerOptions/types';
import { MetricsType } from 'container/MetricsApplication/constant';
import dompurify from 'dompurify';
import { uniqueId } from 'lodash-es';
@@ -35,32 +34,13 @@ export const recursiveParseJSON = (obj: string): Record<string, unknown> => {
}
};
type JsonToDataNodesOptions = {
parentKey?: string;
parentIsArray?: boolean;
isBodyJsonQueryEnabled?: boolean;
handleChangeSelectedView?: ChangeViewFunctionType;
};
type ComputeDataNodeOptions = {
key: string;
valueIsArray: boolean;
value: unknown;
nodeKey: string;
parentIsArray: boolean;
isBodyJsonQueryEnabled?: boolean;
handleChangeSelectedView?: ChangeViewFunctionType;
};
export const computeDataNode = ({
key,
valueIsArray,
value,
nodeKey,
parentIsArray,
isBodyJsonQueryEnabled = false,
handleChangeSelectedView,
}: ComputeDataNodeOptions): DataNode => ({
export const computeDataNode = (
key: string,
valueIsArray: boolean,
value: unknown,
nodeKey: string,
parentIsArray: boolean,
): DataNode => ({
key: uniqueId(),
title: (
<BodyTitleRenderer
@@ -68,30 +48,20 @@ export const computeDataNode = ({
nodeKey={nodeKey}
value={value}
parentIsArray={parentIsArray}
handleChangeSelectedView={handleChangeSelectedView}
/>
),
children: jsonToDataNodes(value as Record<string, unknown>, {
parentKey: valueIsArray
? `${nodeKey}${isBodyJsonQueryEnabled ? '[]' : '[*]'}`
: nodeKey,
parentIsArray: valueIsArray,
isBodyJsonQueryEnabled,
handleChangeSelectedView,
}),
children: jsonToDataNodes(
value as Record<string, unknown>,
valueIsArray ? `${nodeKey}[*]` : nodeKey,
valueIsArray,
),
});
export function jsonToDataNodes(
json: Record<string, unknown>,
options: JsonToDataNodesOptions = {},
parentKey = '',
parentIsArray = false,
): DataNode[] {
const {
parentKey = '',
parentIsArray = false,
isBodyJsonQueryEnabled = false,
handleChangeSelectedView,
} = options;
return Object.entries(json).map(([key, value]) => {
let nodeKey = parentKey || key;
if (parentIsArray) {
@@ -104,15 +74,7 @@ export function jsonToDataNodes(
if (parentIsArray) {
if (typeof value === 'object' && value !== null) {
return computeDataNode({
key,
valueIsArray,
value,
nodeKey,
parentIsArray,
isBodyJsonQueryEnabled,
handleChangeSelectedView,
});
return computeDataNode(key, valueIsArray, value, nodeKey, parentIsArray);
}
return {
@@ -123,31 +85,14 @@ export function jsonToDataNodes(
nodeKey={nodeKey}
value={value}
parentIsArray={parentIsArray}
handleChangeSelectedView={handleChangeSelectedView}
/>
),
children: jsonToDataNodes(
{},
{
parentKey: nodeKey,
parentIsArray: valueIsArray,
isBodyJsonQueryEnabled,
handleChangeSelectedView,
},
),
children: jsonToDataNodes({}, nodeKey, valueIsArray),
};
}
if (typeof value === 'object' && value !== null) {
return computeDataNode({
key,
valueIsArray,
value,
nodeKey,
parentIsArray,
isBodyJsonQueryEnabled,
handleChangeSelectedView,
});
return computeDataNode(key, valueIsArray, value, nodeKey, parentIsArray);
}
return {
key: uniqueId(),
@@ -157,7 +102,6 @@ export function jsonToDataNodes(
nodeKey={nodeKey}
value={value}
parentIsArray={parentIsArray}
handleChangeSelectedView={handleChangeSelectedView}
/>
),
};
@@ -179,7 +123,6 @@ export function flattenObject(obj: AnyObject, prefix = ''): AnyObject {
export const generateFieldKeyForArray = (
fieldKey: string,
dataType: DataTypes,
isBodyJsonQueryEnabled = false,
): string => {
let lastDotIndex = fieldKey.lastIndexOf('.');
let resultNodeKey = fieldKey;
@@ -195,16 +138,6 @@ export const generateFieldKeyForArray = (
newResultNodeKey = resultNodeKey.substring(0, lastDotIndex);
}
}
// When filtering for a value inside an array, the query builder expects the
// last array segment to be referenced without the trailing `[]`.
// Examples:
// - has(body.config.features, 'fast_checkout')
// - has(body.config.features[].items, 'pen')
// - has(body.config.features[].items[].variants, 'ballpen')
if (isBodyJsonQueryEnabled && newResultNodeKey.endsWith('[]')) {
newResultNodeKey = newResultNodeKey.slice(0, -2);
}
return `body.${newResultNodeKey}`;
};

View File

@@ -16,7 +16,6 @@ import azureMysqlUrl from '@/assets/Logos/azure-mysql.svg';
import azureOpenaiUrl from '@/assets/Logos/azure-openai.svg';
import azureSqlDatabaseMetricsUrl from '@/assets/Logos/azure-sql-database-metrics.svg';
import azureVmUrl from '@/assets/Logos/azure-vm.svg';
import basetenUrl from '@/assets/Logos/baseten.svg';
import celeryUrl from '@/assets/Logos/celery.svg';
import claudeCodeUrl from '@/assets/Logos/claude-code.svg';
import clickhouseUrl from '@/assets/Logos/clickhouse.svg';
@@ -28,7 +27,6 @@ import cppUrl from '@/assets/Logos/cpp.svg';
import crewAiUrl from '@/assets/Logos/crew-ai.svg';
import dashboardsUrl from '@/assets/Logos/dashboards.svg';
import datadogUrl from '@/assets/Logos/datadog.svg';
import dbosUrl from '@/assets/Logos/dbos.svg';
import deepseekUrl from '@/assets/Logos/deepseek.svg';
import denoUrl from '@/assets/Logos/deno.svg';
import dockerUrl from '@/assets/Logos/docker.svg';
@@ -107,7 +105,6 @@ import nvidiaDcgmUrl from '@/assets/Logos/nvidia-dcgm.svg';
import ollamaUrl from '@/assets/Logos/ollama.svg';
import openaiUrl from '@/assets/Logos/openai.svg';
import openclawUrl from '@/assets/Logos/openclaw.svg';
import opencodeUrl from '@/assets/Logos/opencode.svg';
import openlitUrl from '@/assets/Logos/openlit.svg';
import openrouterUrl from '@/assets/Logos/openrouter.svg';
import opentelemetryUrl from '@/assets/Logos/opentelemetry.svg';
@@ -5171,79 +5168,6 @@ const onboardingConfigWithLinks = [
id: 'hasura',
link: '/docs/integrations/opentelemetry-hasura/',
},
{
dataSource: 'opencode-observability',
label: 'OpenCode',
imgUrl: opencodeUrl,
tags: ['LLM Monitoring'],
module: 'apm',
relatedSearchKeywords: [
'ai coding',
'ai editor',
'code editor',
'llm',
'llm monitoring',
'monitoring',
'observability',
'opencode',
'opencode logs',
'opencode metrics',
'opencode monitoring',
'opencode observability',
'opencode traces',
'opentelemetry opencode',
],
id: 'opencode-observability',
link: '/docs/opencode-observability/',
},
{
dataSource: 'baseten-monitoring',
label: 'Baseten',
imgUrl: basetenUrl,
tags: ['LLM Monitoring'],
module: 'apm',
relatedSearchKeywords: [
'baseten',
'baseten logs',
'baseten metrics',
'baseten monitoring',
'baseten observability',
'baseten traces',
'llm',
'llm monitoring',
'ml model serving',
'model inference',
'monitoring',
'observability',
'opentelemetry baseten',
],
id: 'baseten-monitoring',
link: '/docs/baseten-monitoring/',
},
{
dataSource: 'dbos',
label: 'DBOS',
imgUrl: dbosUrl,
tags: ['apm/traces'],
module: 'apm',
relatedSearchKeywords: [
'database oriented',
'dbos',
'dbos instrumentation',
'dbos logs',
'dbos metrics',
'dbos monitoring',
'dbos observability',
'dbos traces',
'monitoring',
'observability',
'opentelemetry dbos',
'transactional runtime',
'workflow monitoring',
],
id: 'dbos',
link: '/docs/integrations/opentelemetry-dbos/',
},
{
dataSource: 'supabase',
label: 'Supabase',

View File

@@ -32,6 +32,9 @@
],
"@/*": [
"./src/*"
],
"react-router": [
"./node_modules/@types/react-router"
]
},
"downlevelIteration": true,

File diff suppressed because it is too large Load Diff

2
go.mod
View File

@@ -8,7 +8,7 @@ require (
github.com/ClickHouse/clickhouse-go/v2 v2.40.1
github.com/DATA-DOG/go-sqlmock v1.5.2
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
github.com/SigNoz/signoz-otel-collector v0.144.3
github.com/SigNoz/signoz-otel-collector v0.144.3-rc.4
github.com/antlr4-go/antlr/v4 v4.13.1
github.com/antonmedv/expr v1.15.3
github.com/cespare/xxhash/v2 v2.3.0

4
go.sum
View File

@@ -108,8 +108,8 @@ github.com/SigNoz/expr v1.17.7-beta h1:FyZkleM5dTQ0O6muQfwGpoH5A2ohmN/XTasRCO72g
github.com/SigNoz/expr v1.17.7-beta/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkbj57eGXx8H3ZJ4zhmQXBnrW523ktj8=
github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
github.com/SigNoz/signoz-otel-collector v0.144.3 h1:/7PPIqIpPsaWtrgnfHam2XVYP41ZlgEKLHzQO8oVxcA=
github.com/SigNoz/signoz-otel-collector v0.144.3/go.mod h1:9pLVpcIQvUT88ZiNnZN/MI+XLruvwC+Xm2UzPmGjNfA=
github.com/SigNoz/signoz-otel-collector v0.144.3-rc.4 h1:EskJkEMfuuIyArWhV8SleDV/fuKxiaEGTXrCZIFqDT4=
github.com/SigNoz/signoz-otel-collector v0.144.3-rc.4/go.mod h1:9pLVpcIQvUT88ZiNnZN/MI+XLruvwC+Xm2UzPmGjNfA=
github.com/Yiling-J/theine-go v0.6.2 h1:1GeoXeQ0O0AUkiwj2S9Jc0Mzx+hpqzmqsJ4kIC4M9AY=
github.com/Yiling-J/theine-go v0.6.2/go.mod h1:08QpMa5JZ2pKN+UJCRrCasWYO1IKCdl54Xa836rpmDU=
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=

View File

@@ -3,10 +3,11 @@ package flagger
import "github.com/SigNoz/signoz/pkg/types/featuretypes"
var (
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
FeatureUseSpanMetrics = featuretypes.MustNewName("use_span_metrics")
FeatureKafkaSpanEval = featuretypes.MustNewName("kafka_span_eval")
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
)
func MustNewRegistry() featuretypes.Registry {
@@ -35,6 +36,14 @@ func MustNewRegistry() featuretypes.Registry {
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
&featuretypes.Feature{
Name: FeatureGetMetersFromZeus,
Kind: featuretypes.KindBoolean,
Stage: featuretypes.StageExperimental,
Description: "Controls whether billing details are fetched from Zeus instead of the legacy subscriptions service",
DefaultVariant: featuretypes.MustNewName("disabled"),
Variants: featuretypes.NewBooleanVariants(),
},
&featuretypes.Feature{
Name: FeaturePutMetersInZeus,
Kind: featuretypes.KindBoolean,

View File

@@ -32,17 +32,27 @@ func NewModule(metadataStore telemetrytypes.MetadataStore, telemetrystore teleme
}
func (m *module) ListPromotedAndIndexedPaths(ctx context.Context) ([]promotetypes.PromotePath, error) {
indexes, err := m.metadataStore.ListLogsJSONIndexes(ctx)
logsIndexes, err := m.metadataStore.ListLogsJSONIndexes(ctx)
if err != nil {
return nil, err
}
// Flatten the map values (which are slices) into a single slice
indexes := slices.Concat(slices.Collect(maps.Values(logsIndexes))...)
aggr := map[string][]promotetypes.WrappedIndex{}
for _, index := range indexes {
aggr[index.Name] = append(aggr[index.Name], promotetypes.WrappedIndex{
FieldDataType: index.FieldDataType,
Type: index.IndexType,
Granularity: index.Granularity,
path, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(index.Expression)
if err != nil {
return nil, err
}
// clean backticks from the path
path = strings.ReplaceAll(path, "`", "")
aggr[path] = append(aggr[path], promotetypes.WrappedIndex{
ColumnType: columnType,
Type: index.Type,
Granularity: index.Granularity,
})
}
promotedPaths, err := m.listPromotedPaths(ctx)

View File

@@ -204,7 +204,7 @@ func AdjustKey(key *telemetrytypes.TelemetryFieldKey, keys map[string][]*telemet
// Downstream query builder should handle multiple matching keys with their own metadata
// and not rely on this function to do so.
materialized := true
indexes := []telemetrytypes.TelemetryFieldKeySkipIndex{}
indexes := []telemetrytypes.JSONDataTypeIndex{}
fieldContextsSeen := map[telemetrytypes.FieldContext]bool{}
dataTypesSeen := map[telemetrytypes.FieldDataType]bool{}
for _, matchingKey := range matchingKeys {

View File

@@ -195,8 +195,8 @@ func (c *jsonConditionBuilder) buildPrimitiveTerminalCondition(node *telemetryty
// the field genuinely holds the empty/zero value.
//
// Note: indexing is also skipped for Array Nested fields because they cannot be indexed.
indexed := slices.ContainsFunc(node.TerminalConfig.Key.Indexes, func(index telemetrytypes.TelemetryFieldKeySkipIndex) bool {
return telemetrytypes.MappingFieldDataTypeToJSONDataType[index.FieldDataType] == node.TerminalConfig.ElemType
indexed := slices.ContainsFunc(node.TerminalConfig.Key.Indexes, func(index telemetrytypes.JSONDataTypeIndex) bool {
return index.Type == node.TerminalConfig.ElemType
})
isExistsCheck := operator == qbtypes.FilterOperatorExists || operator == qbtypes.FilterOperatorNotExists
if node.TerminalConfig.ElemType.IndexSupported && indexed && !isExistsCheck {

View File

@@ -1127,12 +1127,9 @@ func buildTestTelemetryMetadataStore(t *testing.T, addIndexes bool) *telemetryty
return entry.Path == path && entry.Type == jsonType
})
if idx >= 0 {
key.Indexes = append(key.Indexes, telemetrytypes.TelemetryFieldKeySkipIndex{
Name: path,
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: fdt,
BaseColumn: LogsV2BodyV2Column,
IndexExpression: schemamigrator.JSONSubColumnIndexExpr(LogsV2BodyV2Column, path, jsonType.StringValue()),
key.Indexes = append(key.Indexes, telemetrytypes.JSONDataTypeIndex{
Type: jsonType,
ColumnExpression: schemamigrator.JSONSubColumnIndexExpr(LogsV2BodyV2Column, path, jsonType.StringValue()),
})
}
}

View File

@@ -106,7 +106,7 @@ func (t *telemetryMetaStore) buildJSONPlans(keys []*telemetrytypes.TelemetryFiel
return nil
}
func (t *telemetryMetaStore) getJSONPathIndexes(ctx context.Context, paths ...string) (map[string][]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
func (t *telemetryMetaStore) getJSONPathIndexes(ctx context.Context, paths ...string) (map[string][]telemetrytypes.JSONDataTypeIndex, error) {
filteredPaths := []string{}
for _, path := range paths {
// skip array paths; since they don't have any indexes
@@ -116,22 +116,47 @@ func (t *telemetryMetaStore) getJSONPathIndexes(ctx context.Context, paths ...st
filteredPaths = append(filteredPaths, path)
}
if len(filteredPaths) == 0 {
return make(map[string][]telemetrytypes.TelemetryFieldKeySkipIndex), nil
return make(map[string][]telemetrytypes.JSONDataTypeIndex), nil
}
// list indexes for the paths
indexes, err := t.ListLogsJSONIndexes(ctx, filteredPaths...)
indexesMap, err := t.ListLogsJSONIndexes(ctx, filteredPaths...)
if err != nil {
return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to list JSON path indexes")
}
// build a set of indexes
fieldPathToIndexes := make(map[string][]telemetrytypes.TelemetryFieldKeySkipIndex)
for _, index := range indexes {
fieldPathToIndexes[index.Name] = append(fieldPathToIndexes[index.Name], index)
cleanIndexes := make(map[string][]telemetrytypes.JSONDataTypeIndex)
for path, indexes := range indexesMap {
for _, index := range indexes {
columnExpr, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(index.Expression)
if err != nil {
return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to unfold JSON sub column index expression: %s", index.Expression)
}
jsonDataType, found := telemetrytypes.MappingStringToJSONDataType[columnType]
if !found {
t.logger.ErrorContext(ctx, "failed to map column type to JSON data type", slog.String("column_type", columnType), slog.String("column_expr", columnExpr))
continue
}
if jsonDataType == telemetrytypes.String {
cleanIndexes[path] = append(cleanIndexes[path], telemetrytypes.JSONDataTypeIndex{
Type: telemetrytypes.String,
ColumnExpression: columnExpr,
IndexExpression: index.Expression,
})
} else if strings.HasPrefix(index.Type, "minmax") {
cleanIndexes[path] = append(cleanIndexes[path], telemetrytypes.JSONDataTypeIndex{
Type: jsonDataType,
ColumnExpression: columnExpr,
IndexExpression: index.Expression,
})
}
}
}
return fieldPathToIndexes, nil
return cleanIndexes, nil
}
func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, []any) {
@@ -148,15 +173,14 @@ func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, [
filterExprs := []string{}
for _, filter := range filters {
// Remove backticks from actual expr cuz paths from metadata doesn't have backticks
filterExprs = append(filterExprs, sb.ILike("replaceAll(expr, '`', '')", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(filter))))
filterExprs = append(filterExprs, sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(filter))))
}
sb.Where(sb.Or(filterExprs...))
return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
}
func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) (map[string][]schemamigrator.Index, error) {
ctx = withTelemetryContext(ctx, "ListLogsJSONIndexes")
query, args := buildListLogsJSONIndexesQuery(t.telemetrystore.Cluster(), filters...)
rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
@@ -165,7 +189,7 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
}
defer rows.Close()
indexes := []telemetrytypes.TelemetryFieldKeySkipIndex{}
indexes := make(map[string][]schemamigrator.Index)
for rows.Next() {
var name string
var typeFull string
@@ -174,39 +198,11 @@ func (t *telemetryMetaStore) ListLogsJSONIndexes(ctx context.Context, filters ..
if err := rows.Scan(&name, &typeFull, &expr, &granularity); err != nil {
return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to scan string indexed column")
}
columnExpr, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(expr)
if err != nil {
return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to unfold JSON sub column index expression: %s", expr)
}
fdt, found := telemetrytypes.MappingJSONDataTypeToFieldDataType[columnType]
if !found {
t.logger.ErrorContext(ctx, "failed to map JSON data type to field data type", slog.String("column_type", columnType), slog.String("column_expr", columnExpr))
continue
}
baseColumn := ""
fieldName := ""
switch {
case strings.HasPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix):
baseColumn = telemetrylogs.BodyV2ColumnPrefix
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyV2ColumnPrefix)
case strings.HasPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix):
baseColumn = telemetrylogs.BodyPromotedColumnPrefix
fieldName = strings.TrimPrefix(columnExpr, telemetrylogs.BodyPromotedColumnPrefix)
}
fieldName = strings.ReplaceAll(fieldName, "`", "")
indexes = append(indexes, telemetrytypes.TelemetryFieldKeySkipIndex{
Name: fieldName,
FieldContext: telemetrytypes.FieldContextBody,
FieldDataType: fdt,
BaseColumn: baseColumn,
IndexName: name,
IndexType: typeFull,
IndexExpression: expr,
Granularity: int(granularity),
indexes[name] = append(indexes[name], schemamigrator.Index{
Name: name,
Type: typeFull,
Expression: expr,
Granularity: int(granularity),
})
}

View File

@@ -36,7 +36,7 @@ func TestBuildListLogsJSONIndexesQuery(t *testing.T) {
cluster: "test-cluster",
filters: []string{"foo", "bar"},
expectedSQL: "SELECT name, type_full, expr, granularity FROM clusterAllReplicas('test-cluster', system.data_skipping_indices) " +
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?)) AND (LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?) OR LOWER(replaceAll(expr, '`', '')) LIKE LOWER(?))",
"WHERE database = ? AND table = ? AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?)) AND (LOWER(expr) LIKE LOWER(?) OR LOWER(expr) LIKE LOWER(?))",
expectedArgs: []any{
telemetrylogs.DBName,
telemetrylogs.LogsV2LocalTableName,

View File

@@ -10,10 +10,10 @@ import (
)
type WrappedIndex struct {
JSONDataType telemetrytypes.JSONDataType `json:"-"`
FieldDataType telemetrytypes.FieldDataType `json:"fieldDataType"`
Type string `json:"type"`
Granularity int `json:"granularity"`
JSONDataType telemetrytypes.JSONDataType `json:"-"`
ColumnType string `json:"column_type"`
Type string `json:"type"`
Granularity int `json:"granularity"`
}
type PromotePath struct {
@@ -60,12 +60,12 @@ func (i *PromotePath) ValidateAndSetDefaults() error {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index granularity must be greater than 0")
}
jsonDataType, ok := telemetrytypes.MappingFieldDataTypeToJSONDataType[index.FieldDataType]
jsonDataType, ok := telemetrytypes.MappingStringToJSONDataType[index.ColumnType]
if !ok {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid column type: %s", index.FieldDataType)
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid column type: %s", index.ColumnType)
}
if !jsonDataType.IndexSupported {
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index is not supported for column type: %s", index.FieldDataType)
return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index is not supported for column type: %s", index.ColumnType)
}
i.Indexes[idx].JSONDataType = jsonDataType

View File

@@ -37,9 +37,9 @@ type TelemetryFieldKey struct {
FieldContext FieldContext `json:"fieldContext,omitzero"`
FieldDataType FieldDataType `json:"fieldDataType,omitzero"`
JSONPlan JSONAccessPlan `json:"-"`
Indexes []TelemetryFieldKeySkipIndex `json:"-"`
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
JSONPlan JSONAccessPlan `json:"-"`
Indexes []JSONDataTypeIndex `json:"-"`
Materialized bool `json:"-"` // refers to promoted in case of body.... fields
Evolutions []*EvolutionEntry `json:"-"`
}
@@ -102,7 +102,7 @@ func (f TelemetryFieldKey) String() string {
if i > 0 {
sb.WriteString("; ")
}
fmt.Fprintf(&sb, "{type=%s, indexExpr=%s}", MappingFieldDataTypeToJSONDataType[index.FieldDataType].StringValue(), index.IndexExpression)
fmt.Fprintf(&sb, "{type=%s, columnExpr=%s, indexExpr=%s}", index.Type.StringValue(), index.ColumnExpression, index.IndexExpression)
}
sb.WriteString("]")
}
@@ -400,14 +400,3 @@ func NewFieldValueSelectorFromPostableFieldValueParams(params PostableFieldValue
return fieldValueSelector
}
type TelemetryFieldKeySkipIndex struct {
Name string `json:"name"` // Name is TelemetryFieldKey.Name not IndexName from ClickHouse
FieldContext FieldContext `json:"fieldContext,omitzero"`
FieldDataType FieldDataType `json:"fieldDataType,omitzero"`
BaseColumn string `json:"baseColumn"`
IndexName string `json:"indexName"`
IndexType string `json:"indexType"`
IndexExpression string `json:"indexExpression"`
Granularity int `json:"granularity"`
}

View File

@@ -1,5 +1,11 @@
package telemetrytypes
type JSONDataTypeIndex struct {
Type JSONDataType
ColumnExpression string
IndexExpression string
}
type JSONDataType struct {
str string // Store the correct case for ClickHouse
IsArray bool
@@ -26,17 +32,18 @@ var (
ArrayJSON = JSONDataType{"Array(JSON)", true, "JSON", false}
)
var MappingJSONDataTypeToFieldDataType = map[string]FieldDataType{
"String": FieldDataTypeString,
"Int64": FieldDataTypeInt64,
"Float64": FieldDataTypeFloat64,
"Bool": FieldDataTypeBool,
"Array(Nullable(String))": FieldDataTypeArrayString,
"Array(Nullable(Int64))": FieldDataTypeArrayInt64,
"Array(Nullable(Float64))": FieldDataTypeArrayFloat64,
"Array(Nullable(Bool))": FieldDataTypeArrayBool,
"Array(Dynamic)": FieldDataTypeArrayDynamic,
"Array(JSON)": FieldDataTypeArrayJSON,
var MappingStringToJSONDataType = map[string]JSONDataType{
"String": String,
"Int64": Int64,
"Float64": Float64,
"Bool": Bool,
"Dynamic": Dynamic,
"Array(Nullable(String))": ArrayString,
"Array(Nullable(Int64))": ArrayInt64,
"Array(Nullable(Float64))": ArrayFloat64,
"Array(Nullable(Bool))": ArrayBool,
"Array(Dynamic)": ArrayDynamic,
"Array(JSON)": ArrayJSON,
}
var MappingFieldDataTypeToJSONDataType = map[FieldDataType]JSONDataType{

View File

@@ -3,6 +3,7 @@ package telemetrytypes
import (
"context"
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
)
@@ -34,7 +35,7 @@ type MetadataStore interface {
FetchTemporalityAndTypeMulti(ctx context.Context, queryTimeRangeStartTs, queryTimeRangeEndTs uint64, metricNames ...string) (map[string]metrictypes.Temporality, map[string]metrictypes.Type, error)
// ListLogsJSONIndexes lists the JSON indexes for the logs table.
ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]TelemetryFieldKeySkipIndex, error)
ListLogsJSONIndexes(ctx context.Context, filters ...string) (map[string][]schemamigrator.Index, error)
// ListPromotedPaths lists the promoted paths.
GetPromotedPaths(ctx context.Context, paths ...string) (map[string]bool, error)

View File

@@ -4,6 +4,7 @@ import (
"context"
"strings"
schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
@@ -17,7 +18,7 @@ type MockMetadataStore struct {
TemporalityMap map[string]metrictypes.Temporality
TypeMap map[string]metrictypes.Type
PromotedPathsMap map[string]bool
LogsJSONIndexes []telemetrytypes.TelemetryFieldKeySkipIndex
LogsJSONIndexesMap map[string][]schemamigrator.Index
ColumnEvolutionMetadataMap map[string][]*telemetrytypes.EvolutionEntry
LookupKeysMap map[telemetrytypes.MetricMetadataLookupKey]int64
// StaticFields holds signal-specific intrinsic field definitions (e.g. telemetrylogs.IntrinsicFields).
@@ -33,7 +34,7 @@ func NewMockMetadataStore() *MockMetadataStore {
TemporalityMap: make(map[string]metrictypes.Temporality),
TypeMap: make(map[string]metrictypes.Type),
PromotedPathsMap: make(map[string]bool),
LogsJSONIndexes: []telemetrytypes.TelemetryFieldKeySkipIndex{},
LogsJSONIndexesMap: make(map[string][]schemamigrator.Index),
ColumnEvolutionMetadataMap: make(map[string][]*telemetrytypes.EvolutionEntry),
LookupKeysMap: make(map[telemetrytypes.MetricMetadataLookupKey]int64),
StaticFields: make(map[string]telemetrytypes.TelemetryFieldKey),
@@ -368,8 +369,8 @@ func (m *MockMetadataStore) GetPromotedPaths(ctx context.Context, paths ...strin
}
// ListLogsJSONIndexes lists the JSON indexes for the logs table.
func (m *MockMetadataStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) ([]telemetrytypes.TelemetryFieldKeySkipIndex, error) {
return m.LogsJSONIndexes, nil
func (m *MockMetadataStore) ListLogsJSONIndexes(ctx context.Context, filters ...string) (map[string][]schemamigrator.Index, error) {
return m.LogsJSONIndexesMap, nil
}
func (m *MockMetadataStore) updateColumnEvolutionMetadataForKeys(_ context.Context, keysToUpdate []*telemetrytypes.TelemetryFieldKey) map[string][]*telemetrytypes.EvolutionEntry {