Compare commits

...

5 Commits

Author SHA1 Message Date
Tushar Vats
4c5381a1d9 fix: added download button in trace page 2026-03-17 18:11:29 +05:30
Amaresh S M
2f012715b4 fix(frontend/vite): avoid inlining whole process.env into bundle (#10605) 2026-03-17 11:51:30 +00:00
Vikrant Gupta
aa05a7bf14 chore(identn): add me as codeowner for identn (#10612) 2026-03-17 11:29:34 +00:00
Vikrant Gupta
99327960b0 feat(authn): move identn to factory and config (#10608)
* feat(authn): move identn to factory and config

* feat(authn): add support for enabling identNs

* feat(authn): add support for enabling identNs
2026-03-17 11:22:26 +00:00
Pandey
12b02a1002 feat(sqlschema): add support for partial unique indexes (#10604)
* feat(sqlschema): add support for partial unique indexes

* feat(sqlschema): add support for multiple indexes

* feat(sqlschema): add support for multiple indexes

* feat(sqlschema): move normalizer to its own struct

* feat(sqlschema): move normalizer tests to normalizer

* feat(sqlschema): move normalizer tests to normalizer

* feat(sqlschema): add more index tests from docs
2026-03-17 11:22:11 +00:00
30 changed files with 1576 additions and 643 deletions

4
.github/CODEOWNERS vendored
View File

@@ -105,6 +105,10 @@ go.mod @therealpandey
/pkg/modules/authdomain/ @vikrantgupta25
/pkg/modules/role/ @vikrantgupta25
# IdentN Owners
/pkg/identn/ @vikrantgupta25
/pkg/http/middleware/identn.go @vikrantgupta25
# Integration tests
/tests/integration/ @vikrantgupta25

View File

@@ -321,3 +321,19 @@ user:
org:
name: default
id: 00000000-0000-0000-0000-000000000000
##################### IdentN #####################
identn:
tokenizer:
# toggle the identN resolver
enabled: true
# headers to use for tokenizer identN resolver
headers:
- Authorization
- Sec-WebSocket-Protocol
apikey:
# toggle the identN resolver
enabled: true
# headers to use for apikey identN resolver
headers:
- SIGNOZ-API-KEY

View File

@@ -223,7 +223,8 @@ SELECT
i.indisunique AS unique,
i.indisprimary AS primary,
a.attname AS column_name,
array_position(i.indkey, a.attnum) AS column_position
array_position(i.indkey, a.attnum) AS column_position,
pg_get_expr(i.indpred, i.indrelid) AS predicate
FROM
pg_index i
LEFT JOIN pg_class ct ON ct.oid = i.indrelid
@@ -246,7 +247,12 @@ ORDER BY index_name, column_position`, string(name))
}
}()
uniqueIndicesMap := make(map[string]*sqlschema.UniqueIndex)
type indexEntry struct {
columns []sqlschema.ColumnName
predicate *string
}
uniqueIndicesMap := make(map[string]*indexEntry)
for rows.Next() {
var (
tableName string
@@ -256,30 +262,50 @@ ORDER BY index_name, column_position`, string(name))
columnName string
// starts from 0 and is unused in this function, this is to ensure that the column names are in the correct order
columnPosition int
predicate *string
)
if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition); err != nil {
if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition, &predicate); err != nil {
return nil, err
}
if unique {
if _, ok := uniqueIndicesMap[indexName]; !ok {
uniqueIndicesMap[indexName] = &sqlschema.UniqueIndex{
TableName: name,
ColumnNames: []sqlschema.ColumnName{sqlschema.ColumnName(columnName)},
uniqueIndicesMap[indexName] = &indexEntry{
columns: []sqlschema.ColumnName{sqlschema.ColumnName(columnName)},
predicate: predicate,
}
} else {
uniqueIndicesMap[indexName].ColumnNames = append(uniqueIndicesMap[indexName].ColumnNames, sqlschema.ColumnName(columnName))
uniqueIndicesMap[indexName].columns = append(uniqueIndicesMap[indexName].columns, sqlschema.ColumnName(columnName))
}
}
}
indices := make([]sqlschema.Index, 0)
for indexName, index := range uniqueIndicesMap {
if index.Name() == indexName {
indices = append(indices, index)
for indexName, entry := range uniqueIndicesMap {
if entry.predicate != nil {
index := &sqlschema.PartialUniqueIndex{
TableName: name,
ColumnNames: entry.columns,
Where: *entry.predicate,
}
if index.Name() == indexName {
indices = append(indices, index)
} else {
indices = append(indices, index.Named(indexName))
}
} else {
indices = append(indices, index.Named(indexName))
index := &sqlschema.UniqueIndex{
TableName: name,
ColumnNames: entry.columns,
}
if index.Name() == indexName {
indices = append(indices, index)
} else {
indices = append(indices, index.Named(indexName))
}
}
}

View File

@@ -8,42 +8,32 @@ export const downloadExportData = async (
props: ExportRawDataProps,
): Promise<void> => {
try {
const queryParams = new URLSearchParams();
queryParams.append('start', String(props.start));
queryParams.append('end', String(props.end));
queryParams.append('filter', props.filter);
props.columns.forEach((col) => {
queryParams.append('columns', col);
});
queryParams.append('order_by', props.orderBy);
queryParams.append('limit', String(props.limit));
queryParams.append('format', props.format);
const response = await axios.get<Blob>(`export_raw_data?${queryParams}`, {
responseType: 'blob', // Important: tell axios to handle response as blob
decompress: true, // Enable automatic decompression
headers: {
Accept: 'application/octet-stream', // Tell server we expect binary data
const response = await axios.post<Blob>(
`export_raw_data?format=${encodeURIComponent(props.format)}`,
props.body,
{
responseType: 'blob',
decompress: true,
headers: {
Accept: 'application/octet-stream',
'Content-Type': 'application/json',
},
timeout: 0,
},
timeout: 0,
});
);
// Only proceed if the response status is 200
if (response.status !== 200) {
throw new Error(
`Failed to download data: server returned status ${response.status}`,
);
}
// Create blob URL from response data
const blob = new Blob([response.data], { type: 'application/octet-stream' });
const url = window.URL.createObjectURL(blob);
// Create and configure download link
const link = document.createElement('a');
link.href = url;
// Get filename from Content-Disposition header or generate timestamped default
const filename =
response.headers['content-disposition']
?.split('filename=')[1]
@@ -51,7 +41,6 @@ export const downloadExportData = async (
link.setAttribute('download', filename);
// Trigger download
document.body.appendChild(link);
link.click();
link.remove();

View File

@@ -1,4 +1,4 @@
.logs-download-popover {
.download-popover {
.ant-popover-inner {
border-radius: 4px;
border: 1px solid var(--bg-slate-400);
@@ -59,7 +59,7 @@
}
.lightMode {
.logs-download-popover {
.download-popover {
.ant-popover-inner {
border: 1px solid var(--bg-vanilla-300);
background: linear-gradient(

View File

@@ -0,0 +1,323 @@
// eslint-disable-next-line no-restricted-imports
import { Provider } from 'react-redux';
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import { message } from 'antd';
import configureStore from 'redux-mock-store';
import store from 'store';
import { Query } from 'types/api/queryBuilder/queryBuilderData';
import { EQueryType } from 'types/common/dashboard';
import { DataSource, StringOperators } from 'types/common/queryBuilder';
import '@testing-library/jest-dom';
import { DownloadFormats, DownloadRowCounts } from './constants';
import DownloadOptionsMenu from './DownloadOptionsMenu';
const mockDownloadExportData = jest.fn().mockResolvedValue(undefined);
jest.mock('api/v1/download/downloadExportData', () => ({
downloadExportData: (...args: any[]): any => mockDownloadExportData(...args),
default: (...args: any[]): any => mockDownloadExportData(...args),
}));
jest.mock('antd', () => {
const actual = jest.requireActual('antd');
return {
...actual,
message: {
success: jest.fn(),
error: jest.fn(),
},
};
});
const mockStore = configureStore([]);
const createMockReduxStore = (): any =>
mockStore({
...store.getState(),
});
const createMockStagedQuery = (dataSource: DataSource): Query => ({
id: 'test-query-id',
queryType: EQueryType.QUERY_BUILDER,
builder: {
queryData: [
{
queryName: 'A',
dataSource,
aggregateOperator: StringOperators.NOOP,
aggregateAttribute: {
id: '',
dataType: '' as any,
key: '',
type: '',
},
aggregations: [{ expression: 'count()' }],
functions: [],
filter: { expression: 'status = 200' },
filters: { items: [], op: 'AND' },
groupBy: [],
expression: 'A',
disabled: false,
having: { expression: '' } as any,
limit: null,
stepInterval: null,
orderBy: [{ columnName: 'timestamp', order: 'desc' }],
legend: '',
selectColumns: [],
},
],
queryFormulas: [],
queryTraceOperator: [],
},
promql: [],
clickhouse_sql: [],
});
const renderWithStore = (
stagedQuery: Query | null,
dataSource: DataSource,
): void => {
const mockReduxStore = createMockReduxStore();
render(
<Provider store={mockReduxStore}>
<DownloadOptionsMenu stagedQuery={stagedQuery} dataSource={dataSource} />
</Provider>,
);
};
describe.each([
[DataSource.LOGS, 'logs'],
[DataSource.TRACES, 'traces'],
])('DownloadOptionsMenu for %s', (dataSource, signal) => {
const testId = `periscope-btn-download-${dataSource}`;
beforeEach(() => {
mockDownloadExportData.mockReset().mockResolvedValue(undefined);
(message.success as jest.Mock).mockReset();
(message.error as jest.Mock).mockReset();
});
it('renders download button', () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
const button = screen.getByTestId(testId);
expect(button).toBeInTheDocument();
expect(button).toHaveClass('periscope-btn', 'ghost');
});
it('shows popover with export options when download button is clicked', () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
expect(screen.getByRole('dialog')).toBeInTheDocument();
expect(screen.getByText('FORMAT')).toBeInTheDocument();
expect(screen.getByText('Number of Rows')).toBeInTheDocument();
expect(screen.getByText('Columns')).toBeInTheDocument();
});
it('allows changing export format', () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
const csvRadio = screen.getByRole('radio', { name: 'csv' });
const jsonlRadio = screen.getByRole('radio', { name: 'jsonl' });
expect(csvRadio).toBeChecked();
fireEvent.click(jsonlRadio);
expect(jsonlRadio).toBeChecked();
expect(csvRadio).not.toBeChecked();
});
it('allows changing row limit', () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
const tenKRadio = screen.getByRole('radio', { name: '10k' });
const fiftyKRadio = screen.getByRole('radio', { name: '50k' });
expect(tenKRadio).toBeChecked();
fireEvent.click(fiftyKRadio);
expect(fiftyKRadio).toBeChecked();
expect(tenKRadio).not.toBeChecked();
});
it('allows changing columns scope', () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
const allColumnsRadio = screen.getByRole('radio', { name: 'All' });
const selectedColumnsRadio = screen.getByRole('radio', { name: 'Selected' });
expect(allColumnsRadio).toBeChecked();
fireEvent.click(selectedColumnsRadio);
expect(selectedColumnsRadio).toBeChecked();
expect(allColumnsRadio).not.toBeChecked();
});
it('calls downloadExportData with correct format and POST body', async () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(mockDownloadExportData).toHaveBeenCalledTimes(1);
const callArgs = mockDownloadExportData.mock.calls[0][0];
expect(callArgs.format).toBe(DownloadFormats.CSV);
expect(callArgs.body).toBeDefined();
expect(callArgs.body.requestType).toBe('raw');
expect(callArgs.body.compositeQuery.queries).toHaveLength(1);
const query = callArgs.body.compositeQuery.queries[0];
expect(query.type).toBe('builder_query');
expect(query.spec.signal).toBe(signal);
expect(query.spec.limit).toBe(DownloadRowCounts.TEN_K);
});
});
it('clears groupBy and having in the export payload', async () => {
const mockQuery = createMockStagedQuery(dataSource);
mockQuery.builder.queryData[0].groupBy = [
{ key: 'service', dataType: 'string' as any, type: '' },
];
mockQuery.builder.queryData[0].having = {
expression: 'count() > 10',
} as any;
renderWithStore(mockQuery, dataSource);
fireEvent.click(screen.getByTestId(testId));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(mockDownloadExportData).toHaveBeenCalledTimes(1);
const callArgs = mockDownloadExportData.mock.calls[0][0];
const query = callArgs.body.compositeQuery.queries[0];
expect(query.spec.groupBy).toBeUndefined();
expect(query.spec.having).toEqual({ expression: '' });
});
});
it('keeps selectColumns when column scope is Selected', async () => {
const mockQuery = createMockStagedQuery(dataSource);
mockQuery.builder.queryData[0].selectColumns = [
{ name: 'http.status', fieldDataType: 'int64', fieldContext: 'attribute' },
] as any;
renderWithStore(mockQuery, dataSource);
fireEvent.click(screen.getByTestId(testId));
fireEvent.click(screen.getByRole('radio', { name: 'Selected' }));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(mockDownloadExportData).toHaveBeenCalledTimes(1);
const callArgs = mockDownloadExportData.mock.calls[0][0];
const query = callArgs.body.compositeQuery.queries[0];
expect(query.spec.selectFields).toEqual([
expect.objectContaining({
name: 'http.status',
fieldDataType: 'int64',
}),
]);
});
});
it('sends empty selectFields when column scope is All', async () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
fireEvent.click(screen.getByRole('radio', { name: 'All' }));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(mockDownloadExportData).toHaveBeenCalledTimes(1);
const callArgs = mockDownloadExportData.mock.calls[0][0];
const query = callArgs.body.compositeQuery.queries[0];
expect(query.spec.selectFields).toEqual([]);
});
});
it('handles successful export with success message', async () => {
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(message.success).toHaveBeenCalledWith(
'Export completed successfully',
);
});
});
it('handles export failure with error message', async () => {
mockDownloadExportData.mockRejectedValueOnce(new Error('Server error'));
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(message.error).toHaveBeenCalledWith(
`Failed to export ${dataSource}. Please try again.`,
);
});
});
it('handles UI state correctly during export process', async () => {
let resolveDownload: () => void;
mockDownloadExportData.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
resolveDownload = resolve;
}),
);
renderWithStore(createMockStagedQuery(dataSource), dataSource);
fireEvent.click(screen.getByTestId(testId));
expect(screen.getByRole('dialog')).toBeInTheDocument();
fireEvent.click(screen.getByText('Export'));
expect(screen.getByTestId(testId)).toBeDisabled();
expect(screen.queryByRole('dialog')).not.toBeInTheDocument();
resolveDownload!();
await waitFor(() => {
expect(screen.getByTestId(testId)).not.toBeDisabled();
});
});
});
describe('DownloadOptionsMenu for traces with queryTraceOperator', () => {
const dataSource = DataSource.TRACES;
const testId = `periscope-btn-download-${dataSource}`;
beforeEach(() => {
mockDownloadExportData.mockReset().mockResolvedValue(undefined);
(message.success as jest.Mock).mockReset();
});
it('applies limit and clears groupBy on queryTraceOperator entries', async () => {
const query = createMockStagedQuery(dataSource);
query.builder.queryTraceOperator = [
{
...query.builder.queryData[0],
queryName: 'TraceOp1',
expression: 'TraceOp1',
groupBy: [{ key: 'service', dataType: 'string' as any, type: '' }],
},
];
renderWithStore(query, dataSource);
fireEvent.click(screen.getByTestId(testId));
fireEvent.click(screen.getByRole('radio', { name: '50k' }));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(mockDownloadExportData).toHaveBeenCalledTimes(1);
const callArgs = mockDownloadExportData.mock.calls[0][0];
const queries = callArgs.body.compositeQuery.queries;
const traceOpQuery = queries.find((q: any) => q.spec.name === 'TraceOp1');
if (traceOpQuery) {
expect(traceOpQuery.spec.limit).toBe(DownloadRowCounts.FIFTY_K);
expect(traceOpQuery.spec.groupBy).toBeUndefined();
}
});
});
});

View File

@@ -0,0 +1,189 @@
import { useCallback, useMemo, useState } from 'react';
// eslint-disable-next-line no-restricted-imports
import { useSelector } from 'react-redux';
import { Button, message, Popover, Radio, Tooltip, Typography } from 'antd';
import { downloadExportData } from 'api/v1/download/downloadExportData';
import { prepareQueryRangePayloadV5 } from 'api/v5/v5';
import { PANEL_TYPES } from 'constants/queryBuilder';
import { Download, DownloadIcon, Loader2 } from 'lucide-react';
import { AppState } from 'store/reducers';
import { Query } from 'types/api/queryBuilder/queryBuilderData';
import { DataSource } from 'types/common/queryBuilder';
import { GlobalReducer } from 'types/reducer/globalTime';
import {
DownloadColumnsScopes,
DownloadFormats,
DownloadRowCounts,
} from './constants';
import './DownloadOptionsMenu.styles.scss';
interface DownloadOptionsMenuProps {
stagedQuery: Query | null;
dataSource: DataSource;
}
export default function DownloadOptionsMenu({
stagedQuery,
dataSource,
}: DownloadOptionsMenuProps): JSX.Element {
const [exportFormat, setExportFormat] = useState<string>(DownloadFormats.CSV);
const [rowLimit, setRowLimit] = useState<number>(DownloadRowCounts.TEN_K);
const [columnsScope, setColumnsScope] = useState<string>(
DownloadColumnsScopes.ALL,
);
const [isDownloading, setIsDownloading] = useState<boolean>(false);
const [isPopoverOpen, setIsPopoverOpen] = useState<boolean>(false);
const { selectedTime: globalSelectedInterval } = useSelector<
AppState,
GlobalReducer
>((state) => state.globalTime);
const handleExportRawData = useCallback(async (): Promise<void> => {
setIsPopoverOpen(false);
if (!stagedQuery) {
return;
}
try {
setIsDownloading(true);
const clearSelectColumns = columnsScope === DownloadColumnsScopes.ALL;
const exportQuery: Query = {
...stagedQuery,
builder: {
...stagedQuery.builder,
queryData: stagedQuery.builder.queryData.map((qd) => ({
...qd,
groupBy: [],
having: { expression: '' },
limit: rowLimit,
...(clearSelectColumns && { selectColumns: [] }),
})),
queryTraceOperator: (stagedQuery.builder.queryTraceOperator || []).map(
(traceOp) => ({
...traceOp,
groupBy: [],
having: { expression: '' },
limit: rowLimit,
...(clearSelectColumns && { selectColumns: [] }),
}),
),
},
};
const { queryPayload } = prepareQueryRangePayloadV5({
query: exportQuery,
graphType: PANEL_TYPES.LIST,
selectedTime: 'GLOBAL_TIME',
globalSelectedInterval,
});
await downloadExportData({ format: exportFormat, body: queryPayload });
message.success('Export completed successfully');
} catch (error) {
console.error(`Error exporting ${dataSource}:`, error);
message.error(`Failed to export ${dataSource}. Please try again.`);
} finally {
setIsDownloading(false);
}
}, [
stagedQuery,
columnsScope,
exportFormat,
rowLimit,
globalSelectedInterval,
dataSource,
]);
const popoverContent = useMemo(
() => (
<div
className="export-options-container"
role="dialog"
aria-label="Export options"
aria-modal="true"
>
<div className="export-format">
<Typography.Text className="title">FORMAT</Typography.Text>
<Radio.Group
value={exportFormat}
onChange={(e): void => setExportFormat(e.target.value)}
>
<Radio value={DownloadFormats.CSV}>csv</Radio>
<Radio value={DownloadFormats.JSONL}>jsonl</Radio>
</Radio.Group>
</div>
<div className="horizontal-line" />
<div className="row-limit">
<Typography.Text className="title">Number of Rows</Typography.Text>
<Radio.Group
value={rowLimit}
onChange={(e): void => setRowLimit(e.target.value)}
>
<Radio value={DownloadRowCounts.TEN_K}>10k</Radio>
<Radio value={DownloadRowCounts.THIRTY_K}>30k</Radio>
<Radio value={DownloadRowCounts.FIFTY_K}>50k</Radio>
</Radio.Group>
</div>
<div className="horizontal-line" />
<div className="columns-scope">
<Typography.Text className="title">Columns</Typography.Text>
<Radio.Group
value={columnsScope}
onChange={(e): void => setColumnsScope(e.target.value)}
>
<Radio value={DownloadColumnsScopes.ALL}>All</Radio>
<Radio value={DownloadColumnsScopes.SELECTED}>Selected</Radio>
</Radio.Group>
</div>
<Button
type="primary"
icon={<Download size={16} />}
onClick={handleExportRawData}
className="export-button"
disabled={isDownloading}
loading={isDownloading}
>
Export
</Button>
</div>
),
[exportFormat, rowLimit, columnsScope, isDownloading, handleExportRawData],
);
return (
<Popover
content={popoverContent}
trigger="click"
placement="bottomRight"
arrow={false}
open={isPopoverOpen}
onOpenChange={setIsPopoverOpen}
rootClassName="download-popover"
>
<Tooltip title="Download" placement="top">
<Button
className="periscope-btn ghost"
icon={
isDownloading ? (
<Loader2 size={18} className="animate-spin" />
) : (
<DownloadIcon size={15} />
)
}
data-testid={`periscope-btn-download-${dataSource}`}
disabled={isDownloading}
/>
</Tooltip>
</Popover>
);
}

View File

@@ -1,341 +0,0 @@
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import { message } from 'antd';
import { ENVIRONMENT } from 'constants/env';
import { server } from 'mocks-server/server';
import { rest } from 'msw';
import { TelemetryFieldKey } from 'types/api/v5/queryRange';
import '@testing-library/jest-dom';
import { DownloadFormats, DownloadRowCounts } from './constants';
import LogsDownloadOptionsMenu from './LogsDownloadOptionsMenu';
// Mock antd message
jest.mock('antd', () => {
const actual = jest.requireActual('antd');
return {
...actual,
message: {
success: jest.fn(),
error: jest.fn(),
},
};
});
const TEST_IDS = {
DOWNLOAD_BUTTON: 'periscope-btn-download-options',
} as const;
interface TestProps {
startTime: number;
endTime: number;
filter: string;
columns: TelemetryFieldKey[];
orderBy: string;
}
const createTestProps = (): TestProps => ({
startTime: 1631234567890,
endTime: 1631234567999,
filter: 'status = 200',
columns: [
{
name: 'http.status',
fieldContext: 'attribute',
fieldDataType: 'int64',
} as TelemetryFieldKey,
],
orderBy: 'timestamp:desc',
});
const testRenderContent = (props: TestProps): void => {
render(
<LogsDownloadOptionsMenu
startTime={props.startTime}
endTime={props.endTime}
filter={props.filter}
columns={props.columns}
orderBy={props.orderBy}
/>,
);
};
const testSuccessResponse = (res: any, ctx: any): any =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/octet-stream'),
ctx.set('Content-Disposition', 'attachment; filename="export.csv"'),
ctx.body('id,value\n1,2\n'),
);
describe('LogsDownloadOptionsMenu', () => {
const BASE_URL = ENVIRONMENT.baseURL;
const EXPORT_URL = `${BASE_URL}/api/v1/export_raw_data`;
let requestSpy: jest.Mock<any, any>;
const setupDefaultServer = (): void => {
server.use(
rest.get(EXPORT_URL, (req, res, ctx) => {
const params = req.url.searchParams;
const payload = {
start: Number(params.get('start')),
end: Number(params.get('end')),
filter: params.get('filter'),
columns: params.getAll('columns'),
order_by: params.get('order_by'),
limit: Number(params.get('limit')),
format: params.get('format'),
};
requestSpy(payload);
return testSuccessResponse(res, ctx);
}),
);
};
// Mock URL.createObjectURL used by download logic
const originalCreateObjectURL = URL.createObjectURL;
const originalRevokeObjectURL = URL.revokeObjectURL;
beforeEach(() => {
requestSpy = jest.fn();
setupDefaultServer();
(message.success as jest.Mock).mockReset();
(message.error as jest.Mock).mockReset();
// jsdom doesn't implement it by default
((URL as unknown) as {
createObjectURL: (b: Blob) => string;
}).createObjectURL = jest.fn(() => 'blob:mock');
((URL as unknown) as {
revokeObjectURL: (u: string) => void;
}).revokeObjectURL = jest.fn();
});
beforeAll(() => {
server.listen();
});
afterEach(() => {
server.resetHandlers();
});
afterAll(() => {
server.close();
// restore
URL.createObjectURL = originalCreateObjectURL;
URL.revokeObjectURL = originalRevokeObjectURL;
});
it('renders download button', () => {
const props = createTestProps();
testRenderContent(props);
const button = screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON);
expect(button).toBeInTheDocument();
expect(button).toHaveClass('periscope-btn', 'ghost');
});
it('shows popover with export options when download button is clicked', () => {
const props = createTestProps();
render(
<LogsDownloadOptionsMenu
startTime={props.startTime}
endTime={props.endTime}
filter={props.filter}
columns={props.columns}
orderBy={props.orderBy}
/>,
);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
expect(screen.getByRole('dialog')).toBeInTheDocument();
expect(screen.getByText('FORMAT')).toBeInTheDocument();
expect(screen.getByText('Number of Rows')).toBeInTheDocument();
expect(screen.getByText('Columns')).toBeInTheDocument();
});
it('allows changing export format', () => {
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
const csvRadio = screen.getByRole('radio', { name: 'csv' });
const jsonlRadio = screen.getByRole('radio', { name: 'jsonl' });
expect(csvRadio).toBeChecked();
fireEvent.click(jsonlRadio);
expect(jsonlRadio).toBeChecked();
expect(csvRadio).not.toBeChecked();
});
it('allows changing row limit', () => {
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
const tenKRadio = screen.getByRole('radio', { name: '10k' });
const fiftyKRadio = screen.getByRole('radio', { name: '50k' });
expect(tenKRadio).toBeChecked();
fireEvent.click(fiftyKRadio);
expect(fiftyKRadio).toBeChecked();
expect(tenKRadio).not.toBeChecked();
});
it('allows changing columns scope', () => {
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
const allColumnsRadio = screen.getByRole('radio', { name: 'All' });
const selectedColumnsRadio = screen.getByRole('radio', { name: 'Selected' });
expect(allColumnsRadio).toBeChecked();
fireEvent.click(selectedColumnsRadio);
expect(selectedColumnsRadio).toBeChecked();
expect(allColumnsRadio).not.toBeChecked();
});
it('calls downloadExportData with correct parameters when export button is clicked (Selected columns)', async () => {
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
fireEvent.click(screen.getByRole('radio', { name: 'Selected' }));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(requestSpy).toHaveBeenCalledWith(
expect.objectContaining({
start: props.startTime,
end: props.endTime,
columns: ['attribute.http.status:int64'],
filter: props.filter,
order_by: props.orderBy,
format: DownloadFormats.CSV,
limit: DownloadRowCounts.TEN_K,
}),
);
});
});
it('calls downloadExportData with correct parameters when export button is clicked', async () => {
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
fireEvent.click(screen.getByRole('radio', { name: 'All' }));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(requestSpy).toHaveBeenCalledWith(
expect.objectContaining({
start: props.startTime,
end: props.endTime,
columns: [],
filter: props.filter,
order_by: props.orderBy,
format: DownloadFormats.CSV,
limit: DownloadRowCounts.TEN_K,
}),
);
});
});
it('handles successful export with success message', async () => {
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(message.success).toHaveBeenCalledWith(
'Export completed successfully',
);
});
});
it('handles export failure with error message', async () => {
// Override handler to return 500 for this test
server.use(rest.get(EXPORT_URL, (_req, res, ctx) => res(ctx.status(500))));
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(message.error).toHaveBeenCalledWith(
'Failed to export logs. Please try again.',
);
});
});
it('handles UI state correctly during export process', async () => {
server.use(
rest.get(EXPORT_URL, (_req, res, ctx) => testSuccessResponse(res, ctx)),
);
const props = createTestProps();
testRenderContent(props);
// Open popover
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
expect(screen.getByRole('dialog')).toBeInTheDocument();
// Start export
fireEvent.click(screen.getByText('Export'));
// Check button is disabled during export
expect(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON)).toBeDisabled();
// Check popover is closed immediately after export starts
expect(screen.queryByRole('dialog')).not.toBeInTheDocument();
// Wait for export to complete and verify button is enabled again
await waitFor(() => {
expect(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON)).not.toBeDisabled();
});
});
it('uses filename from Content-Disposition and triggers download click', async () => {
server.use(
rest.get(EXPORT_URL, (_req, res, ctx) =>
res(
ctx.status(200),
ctx.set('Content-Type', 'application/octet-stream'),
ctx.set('Content-Disposition', 'attachment; filename="report.jsonl"'),
ctx.body('row\n'),
),
),
);
const originalCreateElement = document.createElement.bind(document);
const anchorEl = originalCreateElement('a') as HTMLAnchorElement;
const setAttrSpy = jest.spyOn(anchorEl, 'setAttribute');
const clickSpy = jest.spyOn(anchorEl, 'click');
const removeSpy = jest.spyOn(anchorEl, 'remove');
const createElSpy = jest
.spyOn(document, 'createElement')
.mockImplementation((tagName: any): any =>
tagName === 'a' ? anchorEl : originalCreateElement(tagName),
);
const appendSpy = jest.spyOn(document.body, 'appendChild');
const props = createTestProps();
testRenderContent(props);
fireEvent.click(screen.getByTestId(TEST_IDS.DOWNLOAD_BUTTON));
fireEvent.click(screen.getByText('Export'));
await waitFor(() => {
expect(appendSpy).toHaveBeenCalledWith(anchorEl);
expect(setAttrSpy).toHaveBeenCalledWith('download', 'report.jsonl');
expect(clickSpy).toHaveBeenCalled();
expect(removeSpy).toHaveBeenCalled();
});
expect(anchorEl.getAttribute('download')).toBe('report.jsonl');
createElSpy.mockRestore();
appendSpy.mockRestore();
});
});

View File

@@ -1,170 +1,15 @@
import { useCallback, useMemo, useState } from 'react';
import { Button, message, Popover, Radio, Tooltip, Typography } from 'antd';
import { downloadExportData } from 'api/v1/download/downloadExportData';
import { Download, DownloadIcon, Loader2 } from 'lucide-react';
import { TelemetryFieldKey } from 'types/api/v5/queryRange';
import {
DownloadColumnsScopes,
DownloadFormats,
DownloadRowCounts,
} from './constants';
import './LogsDownloadOptionsMenu.styles.scss';
function convertTelemetryFieldKeyToText(key: TelemetryFieldKey): string {
const prefix = key.fieldContext ? `${key.fieldContext}.` : '';
const suffix = key.fieldDataType ? `:${key.fieldDataType}` : '';
return `${prefix}${key.name}${suffix}`;
}
import DownloadOptionsMenu from 'components/DownloadOptionsMenu/DownloadOptionsMenu';
import { Query } from 'types/api/queryBuilder/queryBuilderData';
import { DataSource } from 'types/common/queryBuilder';
interface LogsDownloadOptionsMenuProps {
startTime: number;
endTime: number;
filter: string;
columns: TelemetryFieldKey[];
orderBy: string;
stagedQuery: Query | null;
}
export default function LogsDownloadOptionsMenu({
startTime,
endTime,
filter,
columns,
orderBy,
stagedQuery,
}: LogsDownloadOptionsMenuProps): JSX.Element {
const [exportFormat, setExportFormat] = useState<string>(DownloadFormats.CSV);
const [rowLimit, setRowLimit] = useState<number>(DownloadRowCounts.TEN_K);
const [columnsScope, setColumnsScope] = useState<string>(
DownloadColumnsScopes.ALL,
);
const [isDownloading, setIsDownloading] = useState<boolean>(false);
const [isPopoverOpen, setIsPopoverOpen] = useState<boolean>(false);
const handleExportRawData = useCallback(async (): Promise<void> => {
setIsPopoverOpen(false);
try {
setIsDownloading(true);
const downloadOptions = {
source: 'logs',
start: startTime,
end: endTime,
columns:
columnsScope === DownloadColumnsScopes.SELECTED
? columns.map((col) => convertTelemetryFieldKeyToText(col))
: [],
filter,
orderBy,
format: exportFormat,
limit: rowLimit,
};
await downloadExportData(downloadOptions);
message.success('Export completed successfully');
} catch (error) {
console.error('Error exporting logs:', error);
message.error('Failed to export logs. Please try again.');
} finally {
setIsDownloading(false);
}
}, [
startTime,
endTime,
columnsScope,
columns,
filter,
orderBy,
exportFormat,
rowLimit,
setIsDownloading,
setIsPopoverOpen,
]);
const popoverContent = useMemo(
() => (
<div
className="export-options-container"
role="dialog"
aria-label="Export options"
aria-modal="true"
>
<div className="export-format">
<Typography.Text className="title">FORMAT</Typography.Text>
<Radio.Group
value={exportFormat}
onChange={(e): void => setExportFormat(e.target.value)}
>
<Radio value={DownloadFormats.CSV}>csv</Radio>
<Radio value={DownloadFormats.JSONL}>jsonl</Radio>
</Radio.Group>
</div>
<div className="horizontal-line" />
<div className="row-limit">
<Typography.Text className="title">Number of Rows</Typography.Text>
<Radio.Group
value={rowLimit}
onChange={(e): void => setRowLimit(e.target.value)}
>
<Radio value={DownloadRowCounts.TEN_K}>10k</Radio>
<Radio value={DownloadRowCounts.THIRTY_K}>30k</Radio>
<Radio value={DownloadRowCounts.FIFTY_K}>50k</Radio>
</Radio.Group>
</div>
<div className="horizontal-line" />
<div className="columns-scope">
<Typography.Text className="title">Columns</Typography.Text>
<Radio.Group
value={columnsScope}
onChange={(e): void => setColumnsScope(e.target.value)}
>
<Radio value={DownloadColumnsScopes.ALL}>All</Radio>
<Radio value={DownloadColumnsScopes.SELECTED}>Selected</Radio>
</Radio.Group>
</div>
<Button
type="primary"
icon={<Download size={16} />}
onClick={handleExportRawData}
className="export-button"
disabled={isDownloading}
loading={isDownloading}
>
Export
</Button>
</div>
),
[exportFormat, rowLimit, columnsScope, isDownloading, handleExportRawData],
);
return (
<Popover
content={popoverContent}
trigger="click"
placement="bottomRight"
arrow={false}
open={isPopoverOpen}
onOpenChange={setIsPopoverOpen}
rootClassName="logs-download-popover"
>
<Tooltip title="Download" placement="top">
<Button
className="periscope-btn ghost"
icon={
isDownloading ? (
<Loader2 size={18} className="animate-spin" />
) : (
<DownloadIcon size={15} />
)
}
data-testid="periscope-btn-download-options"
disabled={isDownloading}
/>
</Tooltip>
</Popover>
<DownloadOptionsMenu stagedQuery={stagedQuery} dataSource={DataSource.LOGS} />
);
}

View File

@@ -0,0 +1,18 @@
import DownloadOptionsMenu from 'components/DownloadOptionsMenu/DownloadOptionsMenu';
import { Query } from 'types/api/queryBuilder/queryBuilderData';
import { DataSource } from 'types/common/queryBuilder';
interface TracesDownloadOptionsMenuProps {
stagedQuery: Query | null;
}
export default function TracesDownloadOptionsMenu({
stagedQuery,
}: TracesDownloadOptionsMenuProps): JSX.Element {
return (
<DownloadOptionsMenu
stagedQuery={stagedQuery}
dataSource={DataSource.TRACES}
/>
);
}

View File

@@ -6,12 +6,14 @@ import { LOCALSTORAGE } from 'constants/localStorage';
import { PANEL_TYPES } from 'constants/queryBuilder';
import { useOptionsMenu } from 'container/OptionsMenu';
import { ArrowUp10, Minus } from 'lucide-react';
import { Query } from 'types/api/queryBuilder/queryBuilderData';
import { DataSource, StringOperators } from 'types/common/queryBuilder';
import QueryStatus from './QueryStatus';
function LogsActionsContainer({
listQuery,
stagedQuery,
selectedPanelType,
showFrequencyChart,
handleToggleFrequencyChart,
@@ -21,10 +23,9 @@ function LogsActionsContainer({
isLoading,
isError,
isSuccess,
minTime,
maxTime,
}: {
listQuery: any;
stagedQuery: Query | null;
selectedPanelType: PANEL_TYPES;
showFrequencyChart: boolean;
handleToggleFrequencyChart: () => void;
@@ -34,8 +35,6 @@ function LogsActionsContainer({
isLoading: boolean;
isError: boolean;
isSuccess: boolean;
minTime: number;
maxTime: number;
}): JSX.Element {
const { options, config } = useOptionsMenu({
storageKey: LOCALSTORAGE.LOGS_LIST_OPTIONS,
@@ -96,13 +95,7 @@ function LogsActionsContainer({
/>
</div>
<div className="download-options-container">
<LogsDownloadOptionsMenu
startTime={minTime}
endTime={maxTime}
filter={listQuery?.filter?.expression || ''}
columns={config.addColumn?.value || []}
orderBy={orderBy}
/>
<LogsDownloadOptionsMenu stagedQuery={stagedQuery} />
</div>
<div className="format-options-container">
<LogsFormatOptionsMenu

View File

@@ -435,6 +435,7 @@ function LogsExplorerViewsContainer({
{!showLiveLogs && (
<LogsActionsContainer
listQuery={listQuery}
stagedQuery={stagedQuery}
selectedPanelType={selectedPanelType}
showFrequencyChart={showFrequencyChart}
handleToggleFrequencyChart={handleToggleFrequencyChart}
@@ -444,8 +445,6 @@ function LogsExplorerViewsContainer({
isLoading={isLoading}
isError={isError}
isSuccess={isSuccess}
minTime={minTime}
maxTime={maxTime}
/>
)}

View File

@@ -1,10 +1,6 @@
import { QueryRangePayloadV5 } from 'types/api/v5/queryRange';
export interface ExportRawDataProps {
source: string;
format: string;
start: number;
end: number;
columns: string[];
filter: string;
orderBy: string;
limit: number;
body: QueryRangePayloadV5;
}

View File

@@ -1,9 +1,8 @@
import { sentryVitePlugin } from '@sentry/vite-plugin';
import react from '@vitejs/plugin-react';
import { readFileSync } from 'fs';
import { resolve } from 'path';
import { visualizer } from 'rollup-plugin-visualizer';
import type { Plugin, UserConfig } from 'vite';
import type { Plugin, TransformResult, UserConfig } from 'vite';
import { defineConfig, loadEnv } from 'vite';
import vitePluginChecker from 'vite-plugin-checker';
import viteCompression from 'vite-plugin-compression';
@@ -14,15 +13,14 @@ import tsconfigPaths from 'vite-tsconfig-paths';
function rawMarkdownPlugin(): Plugin {
return {
name: 'raw-markdown',
transform(_, id): any {
if (id.endsWith('.md')) {
const content = readFileSync(id, 'utf-8');
return {
code: `export default ${JSON.stringify(content)};`,
map: null,
};
transform(code, id): TransformResult | undefined {
if (!id.endsWith('.md')) {
return undefined;
}
return undefined;
return {
code: `export default ${JSON.stringify(code)};`,
map: null,
};
},
};
}
@@ -71,7 +69,7 @@ export default defineConfig(
);
}
if (env.NODE_ENV === 'production') {
if (mode === 'production') {
plugins.push(
ViteImageOptimizer({
jpeg: { quality: 80 },
@@ -102,22 +100,25 @@ export default defineConfig(
},
define: {
// TODO: Remove this in favor of import.meta.env
'process.env': JSON.stringify({
NODE_ENV: mode,
FRONTEND_API_ENDPOINT: env.VITE_FRONTEND_API_ENDPOINT,
WEBSOCKET_API_ENDPOINT: env.VITE_WEBSOCKET_API_ENDPOINT,
PYLON_APP_ID: env.VITE_PYLON_APP_ID,
PYLON_IDENTITY_SECRET: env.VITE_PYLON_IDENTITY_SECRET,
APPCUES_APP_ID: env.VITE_APPCUES_APP_ID,
POSTHOG_KEY: env.VITE_POSTHOG_KEY,
SENTRY_AUTH_TOKEN: env.VITE_SENTRY_AUTH_TOKEN,
SENTRY_ORG: env.VITE_SENTRY_ORG,
SENTRY_PROJECT_ID: env.VITE_SENTRY_PROJECT_ID,
SENTRY_DSN: env.VITE_SENTRY_DSN,
TUNNEL_URL: env.VITE_TUNNEL_URL,
TUNNEL_DOMAIN: env.VITE_TUNNEL_DOMAIN,
DOCS_BASE_URL: env.VITE_DOCS_BASE_URL,
}),
'process.env.NODE_ENV': JSON.stringify(mode),
'process.env.FRONTEND_API_ENDPOINT': JSON.stringify(
env.VITE_FRONTEND_API_ENDPOINT,
),
'process.env.WEBSOCKET_API_ENDPOINT': JSON.stringify(
env.VITE_WEBSOCKET_API_ENDPOINT,
),
'process.env.PYLON_APP_ID': JSON.stringify(env.VITE_PYLON_APP_ID),
'process.env.PYLON_IDENTITY_SECRET': JSON.stringify(
env.VITE_PYLON_IDENTITY_SECRET,
),
'process.env.APPCUES_APP_ID': JSON.stringify(env.VITE_APPCUES_APP_ID),
'process.env.POSTHOG_KEY': JSON.stringify(env.VITE_POSTHOG_KEY),
'process.env.SENTRY_ORG': JSON.stringify(env.VITE_SENTRY_ORG),
'process.env.SENTRY_PROJECT_ID': JSON.stringify(env.VITE_SENTRY_PROJECT_ID),
'process.env.SENTRY_DSN': JSON.stringify(env.VITE_SENTRY_DSN),
'process.env.TUNNEL_URL': JSON.stringify(env.VITE_TUNNEL_URL),
'process.env.TUNNEL_DOMAIN': JSON.stringify(env.VITE_TUNNEL_DOMAIN),
'process.env.DOCS_BASE_URL': JSON.stringify(env.VITE_DOCS_BASE_URL),
},
build: {
sourcemap: true,

View File

@@ -17,28 +17,34 @@ import (
// todo: will move this in types layer with service account integration
type apiKeyTokenKey struct{}
type resolver struct {
type provider struct {
store sqlstore.SQLStore
headers []string
config identn.Config
settings factory.ScopedProviderSettings
sfGroup *singleflight.Group
}
func New(providerSettings factory.ProviderSettings, store sqlstore.SQLStore, headers []string) identn.IdentN {
return &resolver{
store: store,
headers: headers,
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/apikeyidentn"),
sfGroup: &singleflight.Group{},
}
func NewFactory(store sqlstore.SQLStore) factory.ProviderFactory[identn.IdentN, identn.Config] {
return factory.NewProviderFactory(factory.MustNewName(authtypes.IdentNProviderAPIkey.StringValue()), func(ctx context.Context, providerSettings factory.ProviderSettings, config identn.Config) (identn.IdentN, error) {
return New(providerSettings, store, config)
})
}
func (r *resolver) Name() authtypes.IdentNProvider {
func New(providerSettings factory.ProviderSettings, store sqlstore.SQLStore, config identn.Config) (identn.IdentN, error) {
return &provider{
store: store,
config: config,
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/apikeyidentn"),
sfGroup: &singleflight.Group{},
}, nil
}
func (provider *provider) Name() authtypes.IdentNProvider {
return authtypes.IdentNProviderAPIkey
}
func (r *resolver) Test(req *http.Request) bool {
for _, header := range r.headers {
func (provider *provider) Test(req *http.Request) bool {
for _, header := range provider.config.APIKeyConfig.Headers {
if req.Header.Get(header) != "" {
return true
}
@@ -46,8 +52,12 @@ func (r *resolver) Test(req *http.Request) bool {
return false
}
func (r *resolver) Pre(req *http.Request) *http.Request {
token := r.extractToken(req)
func (provider *provider) Enabled() bool {
return provider.config.APIKeyConfig.Enabled
}
func (provider *provider) Pre(req *http.Request) *http.Request {
token := provider.extractToken(req)
if token == "" {
return req
}
@@ -56,16 +66,16 @@ func (r *resolver) Pre(req *http.Request) *http.Request {
return req.WithContext(ctx)
}
func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
func (provider *provider) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
ctx := req.Context()
apiKeyToken, ok := ctx.Value(apiKeyTokenKey{}).(string)
if !ok || apiKeyToken == "" {
return nil, errors.New(errors.TypeUnauthenticated, errors.CodeUnauthenticated, "missing api key")
}
var apiKey types.StorableAPIKey
err := r.store.
err := provider.
store.
BunDB().
NewSelect().
Model(&apiKey).
@@ -80,7 +90,8 @@ func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
}
var user types.User
err = r.store.
err = provider.
store.
BunDB().
NewSelect().
Model(&user).
@@ -99,14 +110,15 @@ func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
return &identity, nil
}
func (r *resolver) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
func (provider *provider) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
apiKeyToken, ok := ctx.Value(apiKeyTokenKey{}).(string)
if !ok || apiKeyToken == "" {
return
}
_, _, _ = r.sfGroup.Do(apiKeyToken, func() (any, error) {
_, err := r.store.
_, _, _ = provider.sfGroup.Do(apiKeyToken, func() (any, error) {
_, err := provider.
store.
BunDB().
NewUpdate().
Model(new(types.StorableAPIKey)).
@@ -115,14 +127,14 @@ func (r *resolver) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims
Where("revoked = false").
Exec(ctx)
if err != nil {
r.settings.Logger().ErrorContext(ctx, "failed to update last used of api key", "error", err)
provider.settings.Logger().ErrorContext(ctx, "failed to update last used of api key", "error", err)
}
return true, nil
})
}
func (r *resolver) extractToken(req *http.Request) string {
for _, header := range r.headers {
func (provider *provider) extractToken(req *http.Request) string {
for _, header := range provider.config.APIKeyConfig.Headers {
if v := req.Header.Get(header); v != "" {
return v
}

48
pkg/identn/config.go Normal file
View File

@@ -0,0 +1,48 @@
package identn
import (
"github.com/SigNoz/signoz/pkg/factory"
)
type Config struct {
// Config for tokenizer identN resolver
Tokenizer TokenizerConfig `mapstructure:"tokenizer"`
// Config for apikey identN resolver
APIKeyConfig APIKeyConfig `mapstructure:"apikey"`
}
type TokenizerConfig struct {
// Toggles the identN resolver
Enabled bool `mapstructure:"enabled"`
// Headers to extract from incoming requests
Headers []string `mapstructure:"headers"`
}
type APIKeyConfig struct {
// Toggles the identN resolver
Enabled bool `mapstructure:"enabled"`
// Headers to extract from incoming requests
Headers []string `mapstructure:"headers"`
}
func NewConfigFactory() factory.ConfigFactory {
return factory.NewConfigFactory(factory.MustNewName("identn"), newConfig)
}
func newConfig() factory.Config {
return &Config{
Tokenizer: TokenizerConfig{
Enabled: true,
Headers: []string{"Authorization", "Sec-WebSocket-Protocol"},
},
APIKeyConfig: APIKeyConfig{
Enabled: true,
Headers: []string{"SIGNOZ-API-KEY"},
},
}
}
func (c Config) Validate() error {
return nil
}

View File

@@ -23,6 +23,8 @@ type IdentN interface {
GetIdentity(r *http.Request) (*authtypes.Identity, error)
Name() authtypes.IdentNProvider
Enabled() bool
}
// IdentNWithPreHook is optionally implemented by resolvers that need to

View File

@@ -12,8 +12,16 @@ type identNResolver struct {
}
func NewIdentNResolver(providerSettings factory.ProviderSettings, identNs ...IdentN) IdentNResolver {
enabledIdentNs := []IdentN{}
for _, identN := range identNs {
if identN.Enabled() {
enabledIdentNs = append(enabledIdentNs, identN)
}
}
return &identNResolver{
identNs: identNs,
identNs: enabledIdentNs,
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn"),
}
}

View File

@@ -13,28 +13,34 @@ import (
"golang.org/x/sync/singleflight"
)
type resolver struct {
type provider struct {
tokenizer tokenizer.Tokenizer
headers []string
config identn.Config
settings factory.ScopedProviderSettings
sfGroup *singleflight.Group
}
func New(providerSettings factory.ProviderSettings, tokenizer tokenizer.Tokenizer, headers []string) identn.IdentN {
return &resolver{
tokenizer: tokenizer,
headers: headers,
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"),
sfGroup: &singleflight.Group{},
}
func NewFactory(tokenizer tokenizer.Tokenizer) factory.ProviderFactory[identn.IdentN, identn.Config] {
return factory.NewProviderFactory(factory.MustNewName(authtypes.IdentNProviderTokenizer.StringValue()), func(ctx context.Context, providerSettings factory.ProviderSettings, config identn.Config) (identn.IdentN, error) {
return New(providerSettings, tokenizer, config)
})
}
func (r *resolver) Name() authtypes.IdentNProvider {
func New(providerSettings factory.ProviderSettings, tokenizer tokenizer.Tokenizer, config identn.Config) (identn.IdentN, error) {
return &provider{
tokenizer: tokenizer,
config: config,
settings: factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"),
sfGroup: &singleflight.Group{},
}, nil
}
func (provider *provider) Name() authtypes.IdentNProvider {
return authtypes.IdentNProviderTokenizer
}
func (r *resolver) Test(req *http.Request) bool {
for _, header := range r.headers {
func (provider *provider) Test(req *http.Request) bool {
for _, header := range provider.config.Tokenizer.Headers {
if req.Header.Get(header) != "" {
return true
}
@@ -42,8 +48,12 @@ func (r *resolver) Test(req *http.Request) bool {
return false
}
func (r *resolver) Pre(req *http.Request) *http.Request {
accessToken := r.extractToken(req)
func (provider *provider) Enabled() bool {
return provider.config.Tokenizer.Enabled
}
func (provider *provider) Pre(req *http.Request) *http.Request {
accessToken := provider.extractToken(req)
if accessToken == "" {
return req
}
@@ -52,7 +62,7 @@ func (r *resolver) Pre(req *http.Request) *http.Request {
return req.WithContext(ctx)
}
func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
func (provider *provider) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
ctx := req.Context()
accessToken, err := authtypes.AccessTokenFromContext(ctx)
@@ -60,41 +70,45 @@ func (r *resolver) GetIdentity(req *http.Request) (*authtypes.Identity, error) {
return nil, err
}
return r.tokenizer.GetIdentity(ctx, accessToken)
return provider.tokenizer.GetIdentity(ctx, accessToken)
}
func (r *resolver) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
func (provider *provider) Post(ctx context.Context, _ *http.Request, _ authtypes.Claims) {
if !provider.config.Tokenizer.Enabled {
return
}
accessToken, err := authtypes.AccessTokenFromContext(ctx)
if err != nil {
return
}
_, _, _ = r.sfGroup.Do(accessToken, func() (any, error) {
if err := r.tokenizer.SetLastObservedAt(ctx, accessToken, time.Now()); err != nil {
r.settings.Logger().ErrorContext(ctx, "failed to set last observed at", "error", err)
_, _, _ = provider.sfGroup.Do(accessToken, func() (any, error) {
if err := provider.tokenizer.SetLastObservedAt(ctx, accessToken, time.Now()); err != nil {
provider.settings.Logger().ErrorContext(ctx, "failed to set last observed at", "error", err)
return false, err
}
return true, nil
})
}
func (r *resolver) extractToken(req *http.Request) string {
func (provider *provider) extractToken(req *http.Request) string {
var value string
for _, header := range r.headers {
for _, header := range provider.config.Tokenizer.Headers {
if v := req.Header.Get(header); v != "" {
value = v
break
}
}
accessToken, ok := r.parseBearerAuth(value)
accessToken, ok := provider.parseBearerAuth(value)
if !ok {
return value
}
return accessToken
}
func (r *resolver) parseBearerAuth(auth string) (string, bool) {
func (provider *provider) parseBearerAuth(auth string) (string, bool) {
const prefix = "Bearer "
if len(auth) < len(prefix) || !strings.EqualFold(auth[:len(prefix)], prefix) {
return "", false

View File

@@ -20,6 +20,7 @@ import (
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/gateway"
"github.com/SigNoz/signoz/pkg/global"
"github.com/SigNoz/signoz/pkg/identn"
"github.com/SigNoz/signoz/pkg/instrumentation"
"github.com/SigNoz/signoz/pkg/modules/metricsexplorer"
"github.com/SigNoz/signoz/pkg/modules/user"
@@ -113,6 +114,9 @@ type Config struct {
// User config
User user.Config `mapstructure:"user"`
// IdentN config
IdentN identn.Config `mapstructure:"identn"`
}
// DeprecatedFlags are the flags that are deprecated and scheduled for removal.
@@ -176,6 +180,7 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
metricsexplorer.NewConfigFactory(),
flagger.NewConfigFactory(),
user.NewConfigFactory(),
identn.NewConfigFactory(),
}
conf, err := config.New(ctx, resolverConfig, configFactories)

View File

@@ -22,6 +22,9 @@ import (
"github.com/SigNoz/signoz/pkg/flagger/configflagger"
"github.com/SigNoz/signoz/pkg/global"
"github.com/SigNoz/signoz/pkg/global/signozglobal"
"github.com/SigNoz/signoz/pkg/identn"
"github.com/SigNoz/signoz/pkg/identn/apikeyidentn"
"github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"
"github.com/SigNoz/signoz/pkg/modules/authdomain/implauthdomain"
"github.com/SigNoz/signoz/pkg/modules/organization"
"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
@@ -271,6 +274,13 @@ func NewTokenizerProviderFactories(cache cache.Cache, sqlstore sqlstore.SQLStore
)
}
func NewIdentNProviderFactories(sqlstore sqlstore.SQLStore, tokenizer tokenizer.Tokenizer) factory.NamedMap[factory.ProviderFactory[identn.IdentN, identn.Config]] {
return factory.MustNewNamedMap(
tokenizeridentn.NewFactory(tokenizer),
apikeyidentn.NewFactory(sqlstore),
)
}
func NewGlobalProviderFactories() factory.NamedMap[factory.ProviderFactory[global.Global, global.Config]] {
return factory.MustNewNamedMap(
signozglobal.NewFactory(),

View File

@@ -17,8 +17,6 @@ import (
"github.com/SigNoz/signoz/pkg/flagger"
"github.com/SigNoz/signoz/pkg/gateway"
"github.com/SigNoz/signoz/pkg/identn"
"github.com/SigNoz/signoz/pkg/identn/apikeyidentn"
"github.com/SigNoz/signoz/pkg/identn/tokenizeridentn"
"github.com/SigNoz/signoz/pkg/instrumentation"
"github.com/SigNoz/signoz/pkg/licensing"
"github.com/SigNoz/signoz/pkg/modules/dashboard"
@@ -395,9 +393,16 @@ func New(
modules := NewModules(sqlstore, tokenizer, emailing, providerSettings, orgGetter, alertmanager, analytics, querier, telemetrystore, telemetryMetadataStore, authNs, authz, cache, queryParser, config, dashboard, userGetter)
// Initialize identN resolver
tokenizeridentN := tokenizeridentn.New(providerSettings, tokenizer, []string{"Authorization", "Sec-WebSocket-Protocol"})
apikeyIdentN := apikeyidentn.New(providerSettings, sqlstore, []string{"SIGNOZ-API-KEY"})
identNResolver := identn.NewIdentNResolver(providerSettings, tokenizeridentN, apikeyIdentN)
identNFactories := NewIdentNProviderFactories(sqlstore, tokenizer)
identNs := []identn.IdentN{}
for _, identNFactory := range identNFactories.GetInOrder() {
identN, err := identNFactory.New(ctx, providerSettings, config.IdentN)
if err != nil {
return nil, err
}
identNs = append(identNs, identN)
}
identNResolver := identn.NewIdentNResolver(providerSettings, identNs...)
userService := impluser.NewService(providerSettings, impluser.NewStore(sqlstore, providerSettings), modules.User, orgGetter, authz, config.User.Root)

View File

@@ -8,8 +8,9 @@ import (
)
var (
IndexTypeUnique = IndexType{s: valuer.NewString("uq")}
IndexTypeIndex = IndexType{s: valuer.NewString("ix")}
IndexTypeUnique = IndexType{s: valuer.NewString("uq")}
IndexTypeIndex = IndexType{s: valuer.NewString("ix")}
IndexTypePartialUnique = IndexType{s: valuer.NewString("puq")}
)
type IndexType struct{ s valuer.String }
@@ -22,6 +23,7 @@ type Index interface {
// The name of the index.
// - Indexes are named as `ix_<table_name>_<column_names>`. The column names are separated by underscores.
// - Unique constraints are named as `uq_<table_name>_<column_names>`. The column names are separated by underscores.
// - Partial unique indexes are named as `puq_<table_name>_<column_names>_<predicate_hash>`.
// The name is autogenerated and should not be set by the user.
Name() string
@@ -133,3 +135,101 @@ func (index *UniqueIndex) ToDropSQL(fmter SQLFormatter) []byte {
return sql
}
type PartialUniqueIndex struct {
TableName TableName
ColumnNames []ColumnName
Where string
name string
}
func (index *PartialUniqueIndex) Name() string {
if index.name != "" {
return index.name
}
var b strings.Builder
b.WriteString(IndexTypePartialUnique.String())
b.WriteString("_")
b.WriteString(string(index.TableName))
b.WriteString("_")
for i, column := range index.ColumnNames {
if i > 0 {
b.WriteString("_")
}
b.WriteString(string(column))
}
b.WriteString("_")
b.WriteString((&whereNormalizer{input: index.Where}).hash())
return b.String()
}
func (index *PartialUniqueIndex) Named(name string) Index {
copyOfColumnNames := make([]ColumnName, len(index.ColumnNames))
copy(copyOfColumnNames, index.ColumnNames)
return &PartialUniqueIndex{
TableName: index.TableName,
ColumnNames: copyOfColumnNames,
Where: index.Where,
name: name,
}
}
func (index *PartialUniqueIndex) IsNamed() bool {
return index.name != ""
}
func (*PartialUniqueIndex) Type() IndexType {
return IndexTypePartialUnique
}
func (index *PartialUniqueIndex) Columns() []ColumnName {
return index.ColumnNames
}
func (index *PartialUniqueIndex) Equals(other Index) bool {
if other.Type() != IndexTypePartialUnique {
return false
}
otherPartial, ok := other.(*PartialUniqueIndex)
if !ok {
return false
}
return index.Name() == other.Name() && slices.Equal(index.Columns(), other.Columns()) && (&whereNormalizer{input: index.Where}).normalize() == (&whereNormalizer{input: otherPartial.Where}).normalize()
}
func (index *PartialUniqueIndex) ToCreateSQL(fmter SQLFormatter) []byte {
sql := []byte{}
sql = append(sql, "CREATE UNIQUE INDEX IF NOT EXISTS "...)
sql = fmter.AppendIdent(sql, index.Name())
sql = append(sql, " ON "...)
sql = fmter.AppendIdent(sql, string(index.TableName))
sql = append(sql, " ("...)
for i, column := range index.ColumnNames {
if i > 0 {
sql = append(sql, ", "...)
}
sql = fmter.AppendIdent(sql, string(column))
}
sql = append(sql, ") WHERE "...)
sql = append(sql, index.Where...)
return sql
}
func (index *PartialUniqueIndex) ToDropSQL(fmter SQLFormatter) []byte {
sql := []byte{}
sql = append(sql, "DROP INDEX IF EXISTS "...)
sql = fmter.AppendIdent(sql, index.Name())
return sql
}

View File

@@ -38,6 +38,110 @@ func TestIndexToCreateSQL(t *testing.T) {
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "my_index" ON "users" ("id", "name", "email")`,
},
{
name: "PartialUnique_1Column",
index: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_94610c77" ON "users" ("email") WHERE "deleted_at" IS NULL`,
},
{
name: "PartialUnique_2Columns",
index: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"org_id", "email"},
Where: `"deleted_at" IS NULL`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_org_id_email_94610c77" ON "users" ("org_id", "email") WHERE "deleted_at" IS NULL`,
},
{
name: "PartialUnique_Named",
index: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
name: "my_partial_index",
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "my_partial_index" ON "users" ("email") WHERE "deleted_at" IS NULL`,
},
{
name: "PartialUnique_WhereWithParentheses",
index: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `("deleted_at" IS NULL)`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_94610c77" ON "users" ("email") WHERE ("deleted_at" IS NULL)`,
},
{
name: "PartialUnique_WhereWithQuotedIdentifier",
index: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"order" IS NULL`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_14c5f5f2" ON "users" ("email") WHERE "order" IS NULL`,
},
{
name: "PartialUnique_WhereWithQuotedLiteral",
index: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `status = 'somewhere'`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_9817c709" ON "users" ("email") WHERE status = 'somewhere'`,
},
{
name: "PartialUnique_WhereWith2Columns",
index: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email", "status"},
Where: `email = 'test@example.com' AND status = 'active'`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_status_e70e78c3" ON "users" ("email", "status") WHERE email = 'test@example.com' AND status = 'active'`,
},
// postgres docs example
{
name: "PartialUnique_WhereWithPostgresDocsExample_1",
index: &PartialUniqueIndex{
TableName: "access_log",
ColumnNames: []ColumnName{"client_ip"},
Where: `NOT (client_ip > inet '192.168.100.0' AND client_ip < inet '192.168.100.255')`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_access_log_client_ip_5a596410" ON "access_log" ("client_ip") WHERE NOT (client_ip > inet '192.168.100.0' AND client_ip < inet '192.168.100.255')`,
},
// postgres docs example
{
name: "PartialUnique_WhereWithPostgresDocsExample_2",
index: &PartialUniqueIndex{
TableName: "orders",
ColumnNames: []ColumnName{"order_nr"},
Where: `billed is not true`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_orders_order_nr_6d31bb0e" ON "orders" ("order_nr") WHERE billed is not true`,
},
// sqlite docs example
{
name: "PartialUnique_WhereWithSqliteDocsExample_1",
index: &PartialUniqueIndex{
TableName: "person",
ColumnNames: []ColumnName{"team_id"},
Where: `is_team_leader`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_person_team_id_c8604a29" ON "person" ("team_id") WHERE is_team_leader`,
},
// sqlite docs example
{
name: "PartialUnique_WhereWithSqliteDocsExample_2",
index: &PartialUniqueIndex{
TableName: "purchaseorder",
ColumnNames: []ColumnName{"parent_po"},
Where: `parent_po IS NOT NULL`,
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "puq_purchaseorder_parent_po_dbe2929d" ON "purchaseorder" ("parent_po") WHERE parent_po IS NOT NULL`,
},
}
for _, testCase := range testCases {
@@ -49,3 +153,109 @@ func TestIndexToCreateSQL(t *testing.T) {
})
}
}
func TestIndexEquals(t *testing.T) {
testCases := []struct {
name string
a Index
b Index
equals bool
}{
{
name: "PartialUnique_Same",
a: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
b: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
equals: true,
},
{
name: "PartialUnique_NormalizedPostgresWhere",
a: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
b: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `(deleted_at IS NULL)`,
},
equals: true,
},
{
name: "PartialUnique_DifferentWhere",
a: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
b: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"active" = true`,
},
equals: false,
},
{
name: "PartialUnique_NotEqual_Unique",
a: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
b: &UniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
},
equals: false,
},
{
name: "Unique_NotEqual_PartialUnique",
a: &UniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
},
b: &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
equals: false,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.equals, testCase.a.Equals(testCase.b))
})
}
}
func TestPartialUniqueIndexName(t *testing.T) {
a := &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
}
b := &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `(deleted_at IS NULL)`,
}
c := &PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"active" = true`,
}
assert.Equal(t, "puq_users_email_94610c77", a.Name())
assert.Equal(t, a.Name(), b.Name())
assert.NotEqual(t, a.Name(), c.Name())
}

162
pkg/sqlschema/normalizer.go Normal file
View File

@@ -0,0 +1,162 @@
package sqlschema
import (
"fmt"
"hash/fnv"
"strings"
)
type whereNormalizer struct {
input string
}
func (n *whereNormalizer) hash() string {
hasher := fnv.New32a()
_, _ = hasher.Write([]byte(n.normalize()))
return fmt.Sprintf("%08x", hasher.Sum32())
}
func (n *whereNormalizer) normalize() string {
where := strings.TrimSpace(n.input)
where = n.stripOuterParentheses(where)
var output strings.Builder
output.Grow(len(where))
for i := 0; i < len(where); i++ {
switch where[i] {
case ' ', '\t', '\n', '\r':
if output.Len() > 0 {
last := output.String()[output.Len()-1]
if last != ' ' {
output.WriteByte(' ')
}
}
case '\'':
end := n.consumeSingleQuotedLiteral(where, i, &output)
i = end
case '"':
token, end := n.consumeDoubleQuotedToken(where, i)
output.WriteString(token)
i = end
default:
output.WriteByte(where[i])
}
}
return strings.TrimSpace(output.String())
}
func (n *whereNormalizer) stripOuterParentheses(s string) string {
for {
s = strings.TrimSpace(s)
if len(s) < 2 || s[0] != '(' || s[len(s)-1] != ')' || !n.hasWrappingParentheses(s) {
return s
}
s = s[1 : len(s)-1]
}
}
func (n *whereNormalizer) hasWrappingParentheses(s string) bool {
depth := 0
inSingleQuotedLiteral := false
inDoubleQuotedToken := false
for i := 0; i < len(s); i++ {
switch s[i] {
case '\'':
if inDoubleQuotedToken {
continue
}
if inSingleQuotedLiteral && i+1 < len(s) && s[i+1] == '\'' {
i++
continue
}
inSingleQuotedLiteral = !inSingleQuotedLiteral
case '"':
if inSingleQuotedLiteral {
continue
}
if inDoubleQuotedToken && i+1 < len(s) && s[i+1] == '"' {
i++
continue
}
inDoubleQuotedToken = !inDoubleQuotedToken
case '(':
if inSingleQuotedLiteral || inDoubleQuotedToken {
continue
}
depth++
case ')':
if inSingleQuotedLiteral || inDoubleQuotedToken {
continue
}
depth--
if depth == 0 && i != len(s)-1 {
return false
}
}
}
return depth == 0
}
func (n *whereNormalizer) consumeSingleQuotedLiteral(s string, start int, output *strings.Builder) int {
output.WriteByte(s[start])
for i := start + 1; i < len(s); i++ {
output.WriteByte(s[i])
if s[i] == '\'' {
if i+1 < len(s) && s[i+1] == '\'' {
i++
output.WriteByte(s[i])
continue
}
return i
}
}
return len(s) - 1
}
func (n *whereNormalizer) consumeDoubleQuotedToken(s string, start int) (string, int) {
var ident strings.Builder
for i := start + 1; i < len(s); i++ {
if s[i] == '"' {
if i+1 < len(s) && s[i+1] == '"' {
ident.WriteByte('"')
i++
continue
}
if n.isSimpleUnquotedIdentifier(ident.String()) {
return ident.String(), i
}
return s[start : i+1], i
}
ident.WriteByte(s[i])
}
return s[start:], len(s) - 1
}
func (n *whereNormalizer) isSimpleUnquotedIdentifier(s string) bool {
if s == "" || strings.ToLower(s) != s {
return false
}
for i := 0; i < len(s); i++ {
ch := s[i]
if (ch >= 'a' && ch <= 'z') || ch == '_' {
continue
}
if i > 0 && ch >= '0' && ch <= '9' {
continue
}
return false
}
return true
}

View File

@@ -0,0 +1,57 @@
package sqlschema
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestWhereNormalizerNormalize(t *testing.T) {
testCases := []struct {
name string
input string
output string
}{
{
name: "BooleanComparison",
input: `"active" = true`,
output: `active = true`,
},
{
name: "QuotedStringLiteralPreserved",
input: `status = 'somewhere'`,
output: `status = 'somewhere'`,
},
{
name: "EscapedStringLiteralPreserved",
input: `status = 'it''s active'`,
output: `status = 'it''s active'`,
},
{
name: "OuterParenthesesRemoved",
input: `(("deleted_at" IS NULL))`,
output: `deleted_at IS NULL`,
},
{
name: "InnerParenthesesPreserved",
input: `("deleted_at" IS NULL OR ("active" = true AND "status" = 'open'))`,
output: `deleted_at IS NULL OR (active = true AND status = 'open')`,
},
{
name: "MultipleClausesWhitespaceCollapsed",
input: " ( \"deleted_at\" IS NULL \n AND\t\"active\" = true AND status = 'open' ) ",
output: `deleted_at IS NULL AND active = true AND status = 'open'`,
},
{
name: "ComplexBooleanClauses",
input: `NOT ("deleted_at" IS NOT NULL AND ("active" = false OR "status" = 'archived'))`,
output: `NOT (deleted_at IS NOT NULL AND (active = false OR status = 'archived'))`,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.output, (&whereNormalizer{input: testCase.input}).normalize())
})
}
}

View File

@@ -1146,3 +1146,100 @@ func TestOperatorAlterTable(t *testing.T) {
})
}
}
func TestOperatorDiffIndices(t *testing.T) {
testCases := []struct {
name string
oldIndices []Index
newIndices []Index
expectedSQLs [][]byte
}{
{
name: "UniqueToPartialUnique_DropAndCreate",
oldIndices: []Index{
&UniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
},
},
newIndices: []Index{
&PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
},
expectedSQLs: [][]byte{
[]byte(`DROP INDEX IF EXISTS "uq_users_email"`),
[]byte(`CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_94610c77" ON "users" ("email") WHERE "deleted_at" IS NULL`),
},
},
{
name: "PartialUnique_SameWhere_NoOp",
oldIndices: []Index{
&PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
},
newIndices: []Index{
&PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
},
expectedSQLs: [][]byte{},
},
{
name: "PartialUnique_NormalizedWhere_NoOp",
oldIndices: []Index{
&PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `(deleted_at IS NULL)`,
},
},
newIndices: []Index{
&PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
},
expectedSQLs: [][]byte{},
},
{
name: "PartialUnique_DifferentWhere_DropAndCreate",
oldIndices: []Index{
&PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"deleted_at" IS NULL`,
},
},
newIndices: []Index{
&PartialUniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
Where: `"active" = true`,
},
},
expectedSQLs: [][]byte{
[]byte(`DROP INDEX IF EXISTS "puq_users_email_94610c77"`),
[]byte(`CREATE UNIQUE INDEX IF NOT EXISTS "puq_users_email_202121f8" ON "users" ("email") WHERE "active" = true`),
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
fmter := NewFormatter(schema.NewNopFormatter().Dialect())
operator := NewOperator(fmter, OperatorSupport{})
actuals := operator.DiffIndices(testCase.oldIndices, testCase.newIndices)
assert.Equal(t, testCase.expectedSQLs, actuals)
})
}
}

View File

@@ -3,6 +3,7 @@ package sqlitesqlschema
import (
"context"
"strconv"
"strings"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/SigNoz/signoz/pkg/factory"
@@ -114,7 +115,29 @@ func (provider *provider) GetIndices(ctx context.Context, tableName sqlschema.Ta
return nil, err
}
if unique {
if unique && partial {
var indexSQL string
if err := provider.
sqlstore.
BunDB().
NewRaw("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", name).
Scan(ctx, &indexSQL); err != nil {
return nil, err
}
where := extractWhereClause(indexSQL)
index := &sqlschema.PartialUniqueIndex{
TableName: tableName,
ColumnNames: columns,
Where: where,
}
if index.Name() == name {
indices = append(indices, index)
} else {
indices = append(indices, index.Named(name))
}
} else if unique {
index := &sqlschema.UniqueIndex{
TableName: tableName,
ColumnNames: columns,
@@ -148,3 +171,73 @@ func (provider *provider) ToggleFKEnforcement(ctx context.Context, db bun.IDB, o
return errors.NewInternalf(errors.CodeInternal, "foreign_keys(actual: %s, expected: %s), maybe a transaction is in progress?", strconv.FormatBool(val), strconv.FormatBool(on))
}
func extractWhereClause(sql string) string {
lastWhere := -1
inSingleQuotedLiteral := false
inDoubleQuotedIdentifier := false
inBacktickQuotedIdentifier := false
inBracketQuotedIdentifier := false
for i := 0; i < len(sql); i++ {
switch sql[i] {
case '\'':
if inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
continue
}
if inSingleQuotedLiteral && i+1 < len(sql) && sql[i+1] == '\'' {
i++
continue
}
inSingleQuotedLiteral = !inSingleQuotedLiteral
case '"':
if inSingleQuotedLiteral || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
continue
}
if inDoubleQuotedIdentifier && i+1 < len(sql) && sql[i+1] == '"' {
i++
continue
}
inDoubleQuotedIdentifier = !inDoubleQuotedIdentifier
case '`':
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBracketQuotedIdentifier {
continue
}
inBacktickQuotedIdentifier = !inBacktickQuotedIdentifier
case '[':
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
continue
}
inBracketQuotedIdentifier = true
case ']':
if inBracketQuotedIdentifier {
inBracketQuotedIdentifier = false
}
}
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
continue
}
if strings.EqualFold(sql[i:min(i+5, len(sql))], "WHERE") &&
(i == 0 || !isSQLiteIdentifierChar(sql[i-1])) &&
(i+5 == len(sql) || !isSQLiteIdentifierChar(sql[i+5])) {
lastWhere = i
i += 4
}
}
if lastWhere == -1 {
return ""
}
return strings.TrimSpace(sql[lastWhere+len("WHERE"):])
}
func isSQLiteIdentifierChar(ch byte) bool {
return (ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
(ch >= '0' && ch <= '9') ||
ch == '_'
}

View File

@@ -0,0 +1,52 @@
package sqlitesqlschema
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestExtractWhereClause(t *testing.T) {
testCases := []struct {
name string
sql string
where string
}{
{
name: "UppercaseWhere",
sql: `CREATE UNIQUE INDEX "idx" ON "users" ("email") WHERE "deleted_at" IS NULL`,
where: `"deleted_at" IS NULL`,
},
{
name: "LowercaseWhere",
sql: `CREATE UNIQUE INDEX "idx" ON "users" ("email") where "deleted_at" IS NULL`,
where: `"deleted_at" IS NULL`,
},
{
name: "NewlineBeforeWhere",
sql: "CREATE UNIQUE INDEX \"idx\" ON \"users\" (\"email\")\nWHERE \"deleted_at\" IS NULL",
where: `"deleted_at" IS NULL`,
},
{
name: "ExtraWhitespace",
sql: "CREATE UNIQUE INDEX \"idx\" ON \"users\" (\"email\") \n \t where \"deleted_at\" IS NULL ",
where: `"deleted_at" IS NULL`,
},
{
name: "WhereInStringLiteral",
sql: `CREATE UNIQUE INDEX "idx" ON "users" ("email") WHERE status = 'somewhere'`,
where: `status = 'somewhere'`,
},
{
name: "BooleanLiteral",
sql: `CREATE UNIQUE INDEX "idx" ON "users" ("email") WHERE active = true`,
where: `active = true`,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.where, extractWhereClause(testCase.sql))
})
}
}