test(wal): log the leaky queries

cmd/community/server.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/SigNoz/signoz/cmd"
 	"github.com/SigNoz/signoz/pkg/analytics"
-	"github.com/SigNoz/signoz/pkg/auditor"
 	"github.com/SigNoz/signoz/pkg/authn"
 	"github.com/SigNoz/signoz/pkg/authz"
 	"github.com/SigNoz/signoz/pkg/authz/openfgaauthz"
@@ -94,9 +93,6 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
 		func(_ licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config] {
 			return noopgateway.NewProviderFactory()
 		},
-		func(_ licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]] {
-			return signoz.NewAuditorProviderFactories()
-		},
 		func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
 			return querier.NewHandler(ps, q, a)
 		},

cmd/enterprise/server.go

@@ -8,7 +8,6 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/SigNoz/signoz/cmd"
-	"github.com/SigNoz/signoz/ee/auditor/otlphttpauditor"
 	"github.com/SigNoz/signoz/ee/authn/callbackauthn/oidccallbackauthn"
 	"github.com/SigNoz/signoz/ee/authn/callbackauthn/samlcallbackauthn"
 	"github.com/SigNoz/signoz/ee/authz/openfgaauthz"
@@ -25,7 +24,6 @@ import (
 	enterprisezeus "github.com/SigNoz/signoz/ee/zeus"
 	"github.com/SigNoz/signoz/ee/zeus/httpzeus"
 	"github.com/SigNoz/signoz/pkg/analytics"
-	"github.com/SigNoz/signoz/pkg/auditor"
 	"github.com/SigNoz/signoz/pkg/authn"
 	"github.com/SigNoz/signoz/pkg/authz"
 	"github.com/SigNoz/signoz/pkg/errors"
@@ -135,13 +133,6 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
 		func(licensing licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config] {
 			return httpgateway.NewProviderFactory(licensing)
 		},
-		func(licensing licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]] {
-			factories := signoz.NewAuditorProviderFactories()
-			if err := factories.Add(otlphttpauditor.NewFactory(licensing, version.Info)); err != nil {
-				panic(err)
-			}
-			return factories
-		},
 		func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
 			communityHandler := querier.NewHandler(ps, q, a)
 			return eequerier.NewHandler(ps, q, communityHandler)

conf/example.yaml

@@ -364,34 +364,3 @@ serviceaccount:
   analytics:
     # toggle service account analytics
     enabled: true
-
-##################### Auditor #####################
-auditor:
-  # Specifies the auditor provider to use.
-  # noop: discards all audit events (community default).
-  # otlphttp: exports audit events via OTLP HTTP (enterprise).
-  provider: noop
-  # The async channel capacity for audit events. Events are dropped when full (fail-open).
-  buffer_size: 1000
-  # The maximum number of events per export batch.
-  batch_size: 100
-  # The maximum time between export flushes.
-  flush_interval: 1s
-  otlphttp:
-    # The target scheme://host:port/path of the OTLP HTTP endpoint.
-    endpoint: http://localhost:4318/v1/logs
-    # Whether to use HTTP instead of HTTPS.
-    insecure: false
-    # The maximum duration for an export attempt.
-    timeout: 10s
-    # Additional HTTP headers sent with every export request.
-    headers: {}
-    retry:
-      # Whether to retry on transient failures.
-      enabled: true
-      # The initial wait time before the first retry.
-      initial_interval: 5s
-      # The upper bound on backoff interval.
-      max_interval: 30s
-      # The total maximum time spent retrying.
-      max_elapsed_time: 60s

ee/query-service/app/server.go

@@ -227,7 +227,7 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler, web web.Web) (*h
 		s.config.APIServer.Timeout.Default,
 		s.config.APIServer.Timeout.Max,
 	).Wrap)
-	r.Use(middleware.NewAudit(s.signoz.Instrumentation.Logger(), s.config.APIServer.Logging.ExcludedRoutes, s.signoz.Auditor).Wrap)
+	r.Use(middleware.NewAudit(s.signoz.Instrumentation.Logger(), s.config.APIServer.Logging.ExcludedRoutes, nil).Wrap)
 	r.Use(middleware.NewComment().Wrap)
 
 	apiHandler.RegisterRoutes(r, am)

frontend/src/api/trace/getTraceV3.tsx

@@ -1,44 +0,0 @@
-import { ApiV3Instance as axios } from 'api';
-import { omit } from 'lodash-es';
-import { ErrorResponse, SuccessResponse } from 'types/api';
-import {
-	GetTraceV3PayloadProps,
-	GetTraceV3SuccessResponse,
-} from 'types/api/trace/getTraceV3';
-
-const getTraceV3 = async (
-	props: GetTraceV3PayloadProps,
-): Promise<SuccessResponse<GetTraceV3SuccessResponse> | ErrorResponse> => {
-	let uncollapsedSpans = [...props.uncollapsedSpans];
-	if (!props.isSelectedSpanIDUnCollapsed) {
-		uncollapsedSpans = uncollapsedSpans.filter(
-			(node) => node !== props.selectedSpanId,
-		);
-	}
-	const postData: GetTraceV3PayloadProps = {
-		...props,
-		uncollapsedSpans,
-	};
-	const response = await axios.post<GetTraceV3SuccessResponse>(
-		`/traces/${props.traceId}/waterfall`,
-		omit(postData, 'traceId'),
-	);
-
-	// V3 API wraps response in { status, data }
-	const rawPayload = (response.data as any).data || response.data;
-
-	// Derive serviceName from resources for backward compatibility
-	const spans = (rawPayload.spans || []).map((span: any) => ({
-		...span,
-		serviceName: span.serviceName || span.resources?.['service.name'] || '',
-	}));
-
-	return {
-		statusCode: 200,
-		error: null,
-		message: 'Success',
-		payload: { ...rawPayload, spans },
-	};
-};
-
-export default getTraceV3;

frontend/src/constants/localStorage.ts

@@ -37,5 +37,4 @@ export enum LOCALSTORAGE {
 	SHOW_FREQUENCY_CHART = 'SHOW_FREQUENCY_CHART',
 	DISSMISSED_COST_METER_INFO = 'DISMISSED_COST_METER_INFO',
 	DISMISSED_API_KEYS_DEPRECATION_BANNER = 'DISMISSED_API_KEYS_DEPRECATION_BANNER',
-	TRACE_DETAILS_SPAN_DETAILS_POSITION = 'TRACE_DETAILS_SPAN_DETAILS_POSITION',
 }

frontend/src/constants/reactQueryKeys.ts

@@ -32,7 +32,6 @@ export const REACT_QUERY_KEY = {
 	UPDATE_ALERT_RULE: 'UPDATE_ALERT_RULE',
 	GET_ACTIVE_LICENSE_V3: 'GET_ACTIVE_LICENSE_V3',
 	GET_TRACE_V2_WATERFALL: 'GET_TRACE_V2_WATERFALL',
-	GET_TRACE_V3_WATERFALL: 'GET_TRACE_V3_WATERFALL',
 	GET_TRACE_V2_FLAMEGRAPH: 'GET_TRACE_V2_FLAMEGRAPH',
 	GET_POD_LIST: 'GET_POD_LIST',
 	GET_NODE_LIST: 'GET_NODE_LIST',

frontend/src/container/NewWidget/index.tsx

@@ -677,18 +677,6 @@ function NewWidget({
 			queryType: currentQuery.queryType,
 			isNewPanel,
 			dataSource: currentQuery?.builder?.queryData?.[0]?.dataSource,
-			...(currentQuery.queryType === EQueryType.CLICKHOUSE && {
-				clickhouseQueryCount: currentQuery.clickhouse_sql.length,
-				clickhouseQueries: currentQuery.clickhouse_sql.map((q) => ({
-					name: q.name,
-					query: (q.query ?? '')
-						.replace(/--[^\n]*/g, '') // strip line comments
-						.replace(/\/\*[\s\S]*?\*\//g, '') // strip block comments
-						.replace(/'(?:[^'\\]|\\.|'')*'/g, "'?'") // replace single-quoted strings (handles \' and '' escapes)
-						.replace(/\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b/g, '?'), // replace numeric literals (int, float, scientific)
-					disabled: q.disabled,
-				})),
-			}),
 		});
 		setSaveModal(true);
 		// eslint-disable-next-line react-hooks/exhaustive-deps

frontend/src/hooks/trace/useCopySpanLink.ts

@@ -1,7 +1,7 @@
 import { MouseEventHandler, useCallback } from 'react';
 import { useLocation } from 'react-router-dom';
 import { useCopyToClipboard } from 'react-use';
-import { toast } from '@signozhq/sonner';
+import { useNotifications } from 'hooks/useNotifications';
 import useUrlQuery from 'hooks/useUrlQuery';
 import { Span } from 'types/api/trace/getTraceV2';
 
@@ -11,6 +11,7 @@ export const useCopySpanLink = (
 	const urlQuery = useUrlQuery();
 	const { pathname } = useLocation();
 	const [, setCopy] = useCopyToClipboard();
+	const { notifications } = useNotifications();
 
 	const onSpanCopy: MouseEventHandler<HTMLElement> = useCallback(
 		(event) => {
@@ -30,12 +31,11 @@ export const useCopySpanLink = (
 			const link = `${window.location.origin}${pathname}?${urlQuery.toString()}`;
 
 			setCopy(link);
-			toast.success('Copied to clipboard', {
-				richColors: true,
-				position: 'top-right',
+			notifications.success({
+				message: 'Copied to clipboard',
 			});
 		},
-		[span, urlQuery, pathname, setCopy],
+		[span, urlQuery, pathname, setCopy, notifications],
 	);
 
 	return {

frontend/src/hooks/trace/useGetTraceV3.tsx

@@ -1,29 +0,0 @@
-import { useQuery, UseQueryResult } from 'react-query';
-import getTraceV3 from 'api/trace/getTraceV3';
-import { REACT_QUERY_KEY } from 'constants/reactQueryKeys';
-import { ErrorResponse, SuccessResponse } from 'types/api';
-import {
-	GetTraceV3PayloadProps,
-	GetTraceV3SuccessResponse,
-} from 'types/api/trace/getTraceV3';
-
-const useGetTraceV3 = (props: GetTraceV3PayloadProps): UseTraceV3 =>
-	useQuery({
-		queryFn: () => getTraceV3(props),
-		queryKey: [
-			REACT_QUERY_KEY.GET_TRACE_V3_WATERFALL,
-			props.traceId,
-			props.selectedSpanId,
-			props.isSelectedSpanIDUnCollapsed,
-		],
-		enabled: !!props.traceId,
-		keepPreviousData: true,
-		refetchOnWindowFocus: false,
-	});
-
-type UseTraceV3 = UseQueryResult<
-	SuccessResponse<GetTraceV3SuccessResponse> | ErrorResponse,
-	unknown
->;
-
-export default useGetTraceV3;

frontend/src/types/api/trace/getTraceFlamegraph.ts

@@ -4,8 +4,7 @@ export interface TraceDetailFlamegraphURLProps {
 
 export interface GetTraceFlamegraphPayloadProps {
 	traceId: string;
-	selectedSpanId?: string;
-	limit?: number;
+	selectedSpanId: string;
 }
 
 export interface Event {

frontend/src/types/api/trace/getTraceV2.ts

@@ -37,23 +37,6 @@ export interface Span {
 	hasSibling: boolean;
 	subTreeNodeCount: number;
 	level: number;
-	// V2 API format fields
-	attributes_string?: Record<string, string>;
-	attributes_number?: Record<string, number>;
-	attributes_bool?: Record<string, boolean>;
-	resources_string?: Record<string, string>;
-	// V3 API format fields
-	attributes?: Record<string, string>;
-	resources?: Record<string, string>;
-	http_method?: string;
-	http_url?: string;
-	http_host?: string;
-	db_name?: string;
-	db_operation?: string;
-	external_http_method?: string;
-	external_http_url?: string;
-	response_status_code?: string;
-	is_remote?: string;
 }
 
 export interface GetTraceV2SuccessResponse {

frontend/src/types/api/trace/getTraceV3.ts

@@ -1,14 +0,0 @@
-export interface GetTraceV3PayloadProps {
-	traceId: string;
-	selectedSpanId: string;
-	uncollapsedSpans: string[];
-	isSelectedSpanIDUnCollapsed: boolean;
-}
-
-// Re-export shared types from V2 until V3 response diverges
-export type {
-	Event,
-	GetTraceV2SuccessResponse as GetTraceV3SuccessResponse,
-	Span,
-	TraceDetailV2URLProps as TraceDetailV3URLProps,
-} from './getTraceV2';

pkg/auditor/config.go

@@ -63,7 +63,6 @@ type RetryConfig struct {
 
 func newConfig() factory.Config {
 	return Config{
-		Provider:      "noop",
 		BufferSize:    1000,
 		BatchSize:     100,
 		FlushInterval: time.Second,

pkg/query-service/app/server.go

@@ -208,7 +208,7 @@ func (s *Server) createPublicServer(api *APIHandler, web web.Web) (*http.Server,
 		s.config.APIServer.Timeout.Default,
 		s.config.APIServer.Timeout.Max,
 	).Wrap)
-	r.Use(middleware.NewAudit(s.signoz.Instrumentation.Logger(), s.config.APIServer.Logging.ExcludedRoutes, s.signoz.Auditor).Wrap)
+	r.Use(middleware.NewAudit(s.signoz.Instrumentation.Logger(), s.config.APIServer.Logging.ExcludedRoutes, nil).Wrap)
 	r.Use(middleware.NewComment().Wrap)
 
 	am := middleware.NewAuthZ(s.signoz.Instrumentation.Logger(), s.signoz.Modules.OrgGetter, s.signoz.Authz)

pkg/signoz/config.go

@@ -11,7 +11,6 @@ import (
 	"github.com/SigNoz/signoz/pkg/alertmanager"
 	"github.com/SigNoz/signoz/pkg/analytics"
 	"github.com/SigNoz/signoz/pkg/apiserver"
-	"github.com/SigNoz/signoz/pkg/auditor"
 	"github.com/SigNoz/signoz/pkg/cache"
 	"github.com/SigNoz/signoz/pkg/config"
 	"github.com/SigNoz/signoz/pkg/emailing"
@@ -124,9 +123,6 @@ type Config struct {
 
 	// ServiceAccount config
 	ServiceAccount serviceaccount.Config `mapstructure:"serviceaccount"`
-
-	// Auditor config
-	Auditor auditor.Config `mapstructure:"auditor"`
 }
 
 func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.ResolverConfig) (Config, error) {
@@ -157,7 +153,6 @@ func NewConfig(ctx context.Context, logger *slog.Logger, resolverConfig config.R
 		user.NewConfigFactory(),
 		identn.NewConfigFactory(),
 		serviceaccount.NewConfigFactory(),
-		auditor.NewConfigFactory(),
 	}
 
 	conf, err := config.New(ctx, resolverConfig, configFactories)

pkg/signoz/provider.go

@@ -3,8 +3,6 @@ package signoz
 import (
 	"github.com/SigNoz/signoz/pkg/alertmanager"
 	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager"
-	"github.com/SigNoz/signoz/pkg/auditor"
-	"github.com/SigNoz/signoz/pkg/auditor/noopauditor"
 	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager/rulebasednotification"
 	"github.com/SigNoz/signoz/pkg/alertmanager/signozalertmanager"
 	"github.com/SigNoz/signoz/pkg/analytics"
@@ -314,12 +312,6 @@ func NewGlobalProviderFactories(identNConfig identn.Config) factory.NamedMap[fac
 	)
 }
 
-func NewAuditorProviderFactories() factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]] {
-	return factory.MustNewNamedMap(
-		noopauditor.NewFactory(),
-	)
-}
-
 func NewFlaggerProviderFactories(registry featuretypes.Registry) factory.NamedMap[factory.ProviderFactory[flagger.FlaggerProvider, flagger.Config]] {
 	return factory.MustNewNamedMap(
 		configflagger.NewFactory(registry),

pkg/signoz/signoz.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/alertmanager"
 	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager"
-	"github.com/SigNoz/signoz/pkg/auditor"
 	"github.com/SigNoz/signoz/pkg/alertmanager/nfmanager/nfroutingstore/sqlroutingstore"
 	"github.com/SigNoz/signoz/pkg/analytics"
 	"github.com/SigNoz/signoz/pkg/apiserver"
@@ -76,7 +75,6 @@ type SigNoz struct {
 	QueryParser            queryparser.QueryParser
 	Flagger                flagger.Flagger
 	Gateway                gateway.Gateway
-	Auditor                auditor.Auditor
 }
 
 func New(
@@ -96,7 +94,6 @@ func New(
 	authzCallback func(context.Context, sqlstore.SQLStore, licensing.Licensing, dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error),
 	dashboardModuleCallback func(sqlstore.SQLStore, factory.ProviderSettings, analytics.Analytics, organization.Getter, queryparser.QueryParser, querier.Querier, licensing.Licensing) dashboard.Module,
 	gatewayProviderFactory func(licensing.Licensing) factory.ProviderFactory[gateway.Gateway, gateway.Config],
-	auditorProviderFactories func(licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]],
 	querierHandlerCallback func(factory.ProviderSettings, querier.Querier, analytics.Analytics) querier.Handler,
 ) (*SigNoz, error) {
 	// Initialize instrumentation
@@ -374,12 +371,6 @@ func New(
 		return nil, err
 	}
 
-	// Initialize auditor from the variant-specific provider factories
-	auditor, err := factory.NewProviderFromNamedMap(ctx, providerSettings, config.Auditor, auditorProviderFactories(licensing), config.Auditor.Provider)
-	if err != nil {
-		return nil, err
-	}
-
 	// Initialize authns
 	store := sqlauthnstore.NewStore(sqlstore)
 	authNs, err := authNsCallback(ctx, providerSettings, store, licensing)
@@ -479,7 +470,6 @@ func New(
 		factory.NewNamedService(factory.MustNewName("tokenizer"), tokenizer),
 		factory.NewNamedService(factory.MustNewName("authz"), authz),
 		factory.NewNamedService(factory.MustNewName("user"), userService, factory.MustNewName("authz")),
-		factory.NewNamedService(factory.MustNewName("auditor"), auditor),
 	)
 	if err != nil {
 		return nil, err
@@ -526,6 +516,5 @@ func New(
 		QueryParser:            queryParser,
 		Flagger:                flagger,
 		Gateway:                gateway,
-		Auditor:                auditor,
 	}, nil
 }

pkg/sqlstore/sqlitesqlstore/provider.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"log/slog"
 	"net/url"
+	"os"
+	"time"
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
@@ -23,6 +25,7 @@ type provider struct {
 	bundb     *sqlstore.BunDB
 	dialect   *dialect
 	formatter sqlstore.SQLFormatter
+	done      chan struct{}
 }
 
 func NewFactory(hookFactories ...factory.ProviderFactory[sqlstore.SQLStoreHook, sqlstore.Config]) factory.ProviderFactory[sqlstore.SQLStore, sqlstore.Config] {
@@ -59,13 +62,19 @@ func New(ctx context.Context, providerSettings factory.ProviderSettings, config
 
 	sqliteDialect := sqlitedialect.New()
 	bunDB := sqlstore.NewBunDB(settings, sqldb, sqliteDialect, hooks)
-	return &provider{
+
+	done := make(chan struct{})
+	p := &provider{
 		settings:  settings,
 		sqldb:     sqldb,
 		bundb:     bunDB,
 		dialect:   new(dialect),
 		formatter: newFormatter(bunDB.Dialect()),
-	}, nil
+		done:      done,
+	}
+	go p.walDiagnosticLoop(config.Sqlite.Path)
+
+	return p, nil
 }
 
 func (provider *provider) BunDB() *bun.DB {
@@ -109,3 +118,73 @@ func (provider *provider) WrapAlreadyExistsErrf(err error, code errors.Code, for
 
 	return err
 }
+
+// walDiagnosticLoop periodically logs pool stats, WAL file size, and busy prepared statements
+// to help diagnose WAL checkpoint failures caused by permanent read locks.
+func (provider *provider) walDiagnosticLoop(dbPath string) {
+	ticker := time.NewTicker(60 * time.Second)
+	defer ticker.Stop()
+
+	logger := provider.settings.Logger()
+	walPath := dbPath + "-wal"
+
+	for {
+		select {
+		case <-provider.done:
+			return
+		case <-ticker.C:
+			// 1. Log pool stats (no SQL needed)
+			stats := provider.sqldb.Stats()
+			logger.Info("sqlite_pool_stats",
+				slog.Int("max_open", stats.MaxOpenConnections),
+				slog.Int("open", stats.OpenConnections),
+				slog.Int("in_use", stats.InUse),
+				slog.Int("idle", stats.Idle),
+				slog.Int64("wait_count", stats.WaitCount),
+				slog.String("wait_duration", stats.WaitDuration.String()),
+				slog.Int64("max_idle_closed", stats.MaxIdleClosed),
+				slog.Int64("max_idle_time_closed", stats.MaxIdleTimeClosed),
+				slog.Int64("max_lifetime_closed", stats.MaxLifetimeClosed),
+			)
+
+			// 2. Log WAL file size (no SQL needed)
+			if info, err := os.Stat(walPath); err == nil {
+				logger.Info("sqlite_wal_size",
+					slog.Int64("bytes", info.Size()),
+					slog.String("path", walPath),
+				)
+			}
+
+			// 3. Check for busy prepared statements on a single pool connection
+			provider.checkBusyStatements(logger)
+		}
+	}
+}
+
+func (provider *provider) checkBusyStatements(logger *slog.Logger) {
+	conn, err := provider.sqldb.Conn(context.Background())
+	if err != nil {
+		logger.Warn("sqlite_diag_conn_error", slog.String("error", err.Error()))
+		return
+	}
+	defer conn.Close()
+
+	rows, err := conn.QueryContext(context.Background(), "SELECT sql FROM sqlite_stmt WHERE busy")
+	if err != nil {
+		logger.Warn("sqlite_diag_query_error", slog.String("error", err.Error()))
+		return
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var stmtSQL string
+		if err := rows.Scan(&stmtSQL); err != nil {
+			logger.Warn("sqlite_diag_scan_error", slog.String("error", err.Error()))
+			continue
+		}
+		logger.Warn("leaked_busy_statement", slog.String("sql", stmtSQL))
+	}
+	if err := rows.Err(); err != nil {
+		logger.Warn("sqlite_diag_rows_error", slog.String("error", err.Error()))
+	}
+}