Compare commits

..

3 Commits

Author SHA1 Message Date
Naman Verma
9f8102deac fix: only check for fill less than based on fill only below flag 2026-07-11 00:39:19 +05:30
Naman Verma
1d28040d9a Merge branch 'main' into nv/span-gaps-1d 2026-07-11 00:35:15 +05:30
Naman Verma
4afbaccd91 fix: make span gaps accept day and week as units 2026-07-10 10:55:35 +05:30
12 changed files with 132 additions and 840 deletions

View File

@@ -3,7 +3,6 @@ package postgressqlschema
import (
"context"
"database/sql"
"log/slog"
"github.com/uptrace/bun"
@@ -215,20 +214,6 @@ WHERE
}
func (provider *provider) GetIndices(ctx context.Context, name sqlschema.TableName) ([]sqlschema.Index, error) {
// A key is one entry an index is built on: a column (plain) or an expression
// like lower(x) (functional). Returns one row per key of each non-constraint-
// backed index on the table, ordered by position. Keys are enumerated with
// generate_series over indnkeyatts (count of key columns, excluding non-key
// INCLUDE columns), so expression keys are covered too. Reconstruction uses:
//
// index_name | index the key belongs to
// is_expression | true if the key is an expression, not a plain column
// key_def | rendered key text, e.g. org_id or lower(key)
// column_name | column name for plain keys; NULL for expression keys
// predicate | partial-index WHERE; NULL if not partial (same every row)
//
// table_name/unique/primary/column_position are carried over from the prior
// query (unique gates the loop, column_position orders the keys).
rows, err := provider.
sqlstore.
BunDB().
@@ -239,21 +224,19 @@ SELECT
i.indisunique AS unique,
i.indisprimary AS primary,
a.attname AS column_name,
n AS column_position,
(i.indkey[n - 1] = 0) AS is_expression,
pg_get_indexdef(i.indexrelid, n, true) AS key_def,
array_position(i.indkey, a.attnum) AS column_position,
pg_get_expr(i.indpred, i.indrelid) AS predicate
FROM
pg_index i
LEFT JOIN pg_class ct ON ct.oid = i.indrelid
LEFT JOIN pg_class ci ON ci.oid = i.indexrelid
LEFT JOIN pg_attribute a ON a.attrelid = ct.oid
LEFT JOIN pg_constraint con ON con.conindid = i.indexrelid
CROSS JOIN generate_series(1, i.indnkeyatts) AS n
LEFT JOIN pg_attribute a ON a.attrelid = ct.oid AND a.attnum = i.indkey[n - 1]
WHERE
ct.relname = ?
AND ct.relkind = 'r'
a.attnum = ANY(i.indkey)
AND con.oid IS NULL
AND ct.relkind = 'r'
AND ct.relname = ?
ORDER BY index_name, column_position`, string(name))
if err != nil {
return nil, provider.sqlstore.WrapNotFoundErrf(err, errors.CodeNotFound, "no indices for table (%s) found", name)
@@ -268,10 +251,6 @@ ORDER BY index_name, column_position`, string(name))
type indexEntry struct {
columns []sqlschema.ColumnName
predicate *string
// keyDefs holds every key rendered by pg_get_indexdef, in key order; used
// for functional indexes where a key may be an expression like lower(x).
keyDefs []string
hasExpression bool
}
uniqueIndicesMap := make(map[string]*indexEntry)
@@ -281,45 +260,30 @@ ORDER BY index_name, column_position`, string(name))
indexName string
unique bool
primary bool
columnName *string
// starts from 1 and is unused in this function, this is to ensure that the column names are in the correct order
columnName string
// starts from 0 and is unused in this function, this is to ensure that the column names are in the correct order
columnPosition int
isExpression bool
keyDef string
predicate *string
)
if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition, &isExpression, &keyDef, &predicate); err != nil {
if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition, &predicate); err != nil {
return nil, err
}
if !unique {
continue
}
entry, ok := uniqueIndicesMap[indexName]
if !ok {
entry = &indexEntry{predicate: predicate}
uniqueIndicesMap[indexName] = entry
}
entry.keyDefs = append(entry.keyDefs, keyDef)
if isExpression {
entry.hasExpression = true
} else if columnName != nil {
entry.columns = append(entry.columns, sqlschema.ColumnName(*columnName))
if unique {
if _, ok := uniqueIndicesMap[indexName]; !ok {
uniqueIndicesMap[indexName] = &indexEntry{
columns: []sqlschema.ColumnName{sqlschema.ColumnName(columnName)},
predicate: predicate,
}
} else {
uniqueIndicesMap[indexName].columns = append(uniqueIndicesMap[indexName].columns, sqlschema.ColumnName(columnName))
}
}
}
indices := make([]sqlschema.Index, 0)
for indexName, entry := range uniqueIndicesMap {
// functional partial indexes aren't representable (PartialUniqueIndex has no
// expressions); skip rather than misrepresent.
if entry.hasExpression && entry.predicate != nil {
provider.settings.Logger().WarnContext(ctx, "skipping functional partial unique index; not representable by sqlschema", slog.String("index", indexName), slog.String("table", string(name)))
continue
}
if entry.predicate != nil {
index := &sqlschema.PartialUniqueIndex{
TableName: name,
@@ -327,17 +291,6 @@ ORDER BY index_name, column_position`, string(name))
Where: *entry.predicate,
}
if index.Name() == indexName {
indices = append(indices, index)
} else {
indices = append(indices, index.Named(indexName))
}
} else if entry.hasExpression {
index := &sqlschema.UniqueIndexWithExpressions{
TableName: name,
Expressions: entry.keyDefs,
}
if index.Name() == indexName {
indices = append(indices, index)
} else {

View File

@@ -1,116 +0,0 @@
package postgressqlschema
import (
"context"
"database/sql/driver"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
"github.com/SigNoz/signoz/pkg/sqlschema"
"github.com/SigNoz/signoz/pkg/sqlstore"
"github.com/SigNoz/signoz/pkg/sqlstore/sqlstoretest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// TestGetIndices feeds GetIndices the exact rows the catalog query returns for a
// real table (captured from devenv postgres) and checks each unique-index shape
// is reconstructed: expression keys, partial (predicate), and plain columns.
func TestGetIndices(t *testing.T) {
// column order must match the SELECT in GetIndices.
columns := []string{"table_name", "index_name", "unique", "primary", "column_name", "column_position", "is_expression", "key_def", "predicate"}
testCases := []struct {
name string
table sqlschema.TableName
rows [][]driver.Value
want []sqlschema.Index
}{
{
name: "UniqueIndexWithExpressions",
table: "tag",
rows: [][]driver.Value{
{"tag", "uq_tag_a36c51df", true, false, "org_id", 1, false, "org_id", nil},
{"tag", "uq_tag_a36c51df", true, false, "kind", 2, false, "kind", nil},
{"tag", "uq_tag_a36c51df", true, false, nil, 3, true, "lower(key)", nil},
{"tag", "uq_tag_a36c51df", true, false, nil, 4, true, "lower(value)", nil},
},
want: []sqlschema.Index{
&sqlschema.UniqueIndexWithExpressions{
TableName: "tag",
Expressions: []string{"org_id", "kind", "lower(key)", "lower(value)"},
},
},
},
{
name: "PartialUniqueIndexes",
table: "service_account",
rows: [][]driver.Value{
{"service_account", "puq_service_account_email_org_id_471d6134", true, false, "email", 1, false, "email", "(status <> 'deleted'::text)"},
{"service_account", "puq_service_account_email_org_id_471d6134", true, false, "org_id", 2, false, "org_id", "(status <> 'deleted'::text)"},
{"service_account", "puq_service_account_name_org_id_471d6134", true, false, "name", 1, false, "name", "(status <> 'deleted'::text)"},
{"service_account", "puq_service_account_name_org_id_471d6134", true, false, "org_id", 2, false, "org_id", "(status <> 'deleted'::text)"},
},
want: []sqlschema.Index{
(&sqlschema.PartialUniqueIndex{
TableName: "service_account",
ColumnNames: []sqlschema.ColumnName{"email", "org_id"},
Where: "(status <> 'deleted'::text)",
}).Named("puq_service_account_email_org_id_471d6134"),
(&sqlschema.PartialUniqueIndex{
TableName: "service_account",
ColumnNames: []sqlschema.ColumnName{"name", "org_id"},
Where: "(status <> 'deleted'::text)",
}).Named("puq_service_account_name_org_id_471d6134"),
},
},
{
name: "PlainUniqueIndex",
table: "tag_relation",
rows: [][]driver.Value{
{"tag_relation", "uq_tag_relation_kind_resource_id_tag_id", true, false, "kind", 1, false, "kind", nil},
{"tag_relation", "uq_tag_relation_kind_resource_id_tag_id", true, false, "resource_id", 2, false, "resource_id", nil},
{"tag_relation", "uq_tag_relation_kind_resource_id_tag_id", true, false, "tag_id", 3, false, "tag_id", nil},
},
want: []sqlschema.Index{
&sqlschema.UniqueIndex{
TableName: "tag_relation",
ColumnNames: []sqlschema.ColumnName{"kind", "resource_id", "tag_id"},
},
},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
store := sqlstoretest.New(sqlstore.Config{Provider: "postgres"}, sqlmock.QueryMatcherRegexp)
rows := sqlmock.NewRows(columns)
for _, row := range testCase.rows {
rows.AddRow(row...)
}
store.Mock().ExpectQuery("pg_index").WillReturnRows(rows)
schema, err := New(context.Background(), instrumentationtest.New().ToProviderSettings(), sqlschema.Config{}, store)
require.NoError(t, err)
indices, err := schema.GetIndices(context.Background(), testCase.table)
require.NoError(t, err)
assert.Len(t, indices, len(testCase.want))
// GetIndices iterates a map, so match by name rather than slice order.
got := make(map[string]sqlschema.Index, len(indices))
for _, index := range indices {
got[index.Name()] = index
}
for _, want := range testCase.want {
actual, ok := got[want.Name()]
if !assert.True(t, ok, "expected index %s not returned", want.Name()) {
// so that other items in the for loop can also be checked.
continue
}
assert.True(t, want.Equals(actual), "index %s reconstructed differently", want.Name())
}
})
}
}

View File

@@ -1,8 +1,6 @@
package sqlschema
import (
"fmt"
"hash/fnv"
"slices"
"strings"
@@ -138,121 +136,6 @@ func (index *UniqueIndex) ToDropSQL(fmter SQLFormatter) []byte {
return sql
}
// UniqueIndexWithExpressions is a functional unique index: each key is a SQL
// expression (e.g. LOWER(col)) emitted verbatim, so the caller owns its
// well-formedness; plain columns go in as bare identifiers. The auto-generated
// name uses a hash suffix (uq_t_<hash>) since expressions aren't valid
// identifier fragments.
//
// Use this only when at least one key is a real expression: an all-identifier
// key list reconstructs as a plain UniqueIndex on read-back and won't compare
// equal — use UniqueIndex instead.
type UniqueIndexWithExpressions struct {
TableName TableName
Expressions []string
name string
}
func (index *UniqueIndexWithExpressions) Name() string {
if index.name != "" {
return index.name
}
var b strings.Builder
b.WriteString(IndexTypeUnique.String())
b.WriteString("_")
b.WriteString(string(index.TableName))
b.WriteString("_")
hasher := fnv.New32a()
_, _ = hasher.Write([]byte(strings.Join(normalizeExpressions(index.Expressions), "\x00")))
fmt.Fprintf(&b, "%08x", hasher.Sum32())
return b.String()
}
func (index *UniqueIndexWithExpressions) Named(name string) Index {
copyOfExpressions := make([]string, len(index.Expressions))
copy(copyOfExpressions, index.Expressions)
return &UniqueIndexWithExpressions{
TableName: index.TableName,
Expressions: copyOfExpressions,
name: name,
}
}
func (index *UniqueIndexWithExpressions) IsNamed() bool {
return index.name != ""
}
func (*UniqueIndexWithExpressions) Type() IndexType {
return IndexTypeUnique
}
// Columns is nil: a functional index exposes no plain key columns.
func (*UniqueIndexWithExpressions) Columns() []ColumnName {
return nil
}
func (index *UniqueIndexWithExpressions) Equals(other Index) bool {
otherIndex, ok := other.(*UniqueIndexWithExpressions)
if !ok {
return false
}
// Expressions are compared normalized so a declared `LOWER(x)` matches the
// `lower(x)` a backend renders on read-back.
return index.Name() == other.Name() && slices.Equal(normalizeExpressions(index.Expressions), normalizeExpressions(otherIndex.Expressions))
}
// normalizeExpressions canonicalizes each Expressions entry (case, whitespace,
// redundant parens, identifier quoting) so a declared LOWER(x) matches the
// lower(x) a backend renders on read-back. Reuses expressionNormalizer: an index key
// is the same SQL grammar as a WHERE predicate. It does not handle opclass,
// COLLATE, or ASC/DESC/NULLS suffixes: postgres strips them on read-back
// (pg_get_indexdef), while sqlite keeps them verbatim, so an index declared with
// one round-trips on sqlite but not on postgres.
func normalizeExpressions(expressions []string) []string {
if len(expressions) == 0 {
return nil
}
normalized := make([]string, len(expressions))
for i, expression := range expressions {
normalized[i] = (&expressionNormalizer{input: expression, foldCase: true}).normalize()
}
return normalized
}
func (index *UniqueIndexWithExpressions) ToCreateSQL(fmter SQLFormatter) []byte {
sql := []byte{}
sql = append(sql, "CREATE UNIQUE INDEX IF NOT EXISTS "...)
sql = fmter.AppendIdent(sql, index.Name())
sql = append(sql, " ON "...)
sql = fmter.AppendIdent(sql, string(index.TableName))
sql = append(sql, " ("...)
for i, expr := range index.Expressions {
if i > 0 {
sql = append(sql, ", "...)
}
sql = append(sql, expr...)
}
sql = append(sql, ")"...)
return sql
}
func (index *UniqueIndexWithExpressions) ToDropSQL(fmter SQLFormatter) []byte {
sql := []byte{}
sql = append(sql, "DROP INDEX IF EXISTS "...)
sql = fmter.AppendIdent(sql, index.Name())
return sql
}
type PartialUniqueIndex struct {
TableName TableName
ColumnNames []ColumnName
@@ -277,7 +160,7 @@ func (index *PartialUniqueIndex) Name() string {
b.WriteString(string(column))
}
b.WriteString("_")
b.WriteString((&expressionNormalizer{input: index.Where}).hash())
b.WriteString((&whereNormalizer{input: index.Where}).hash())
return b.String()
}
@@ -315,7 +198,7 @@ func (index *PartialUniqueIndex) Equals(other Index) bool {
return false
}
return index.Name() == other.Name() && slices.Equal(index.Columns(), other.Columns()) && (&expressionNormalizer{input: index.Where}).normalize() == (&expressionNormalizer{input: otherPartial.Where}).normalize()
return index.Name() == other.Name() && slices.Equal(index.Columns(), other.Columns()) && (&whereNormalizer{input: index.Where}).normalize() == (&whereNormalizer{input: otherPartial.Where}).normalize()
}
func (index *PartialUniqueIndex) ToCreateSQL(fmter SQLFormatter) []byte {

View File

@@ -38,39 +38,6 @@ func TestIndexToCreateSQL(t *testing.T) {
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "my_index" ON "users" ("id", "name", "email")`,
},
{
name: "Unique_Functional_SingleExpression",
index: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_users_72852951" ON "users" (LOWER(email))`,
},
{
name: "Unique_Functional_MixedColumnsAndExpressions",
index: &UniqueIndexWithExpressions{
TableName: "tag",
Expressions: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_tag_a36c51df" ON "tag" (org_id, kind, LOWER(key), LOWER(value))`,
},
{
name: "Unique_Functional_ComplexExpression",
index: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(TRIM(first_name) || ' ' || TRIM(last_name))"},
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_users_37e845f3" ON "users" (LOWER(TRIM(first_name) || ' ' || TRIM(last_name)))`,
},
{
name: "Unique_Functional_Named",
index: &UniqueIndexWithExpressions{
TableName: "tag",
Expressions: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
name: "uq_tag_org_kind_lower_key_lower_value",
},
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_tag_org_kind_lower_key_lower_value" ON "tag" (org_id, kind, LOWER(key), LOWER(value))`,
},
{
name: "PartialUnique_1Column",
index: &PartialUniqueIndex{
@@ -262,90 +229,6 @@ func TestIndexEquals(t *testing.T) {
},
equals: false,
},
{
name: "Unique_Functional_Same",
a: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
b: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
equals: true,
},
{
name: "Unique_Functional_CaseInsensitiveEqual",
a: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
b: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"lower(email)"},
},
equals: true,
},
{
name: "Unique_Functional_QuotedSimpleIdentifierEqualsUnquoted",
a: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
b: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{`LOWER("email")`},
},
equals: true,
},
{
name: "Unique_Functional_UnquotedMixedCaseEqualsLower",
a: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(Email)"},
},
b: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
equals: true,
},
{
name: "Unique_Functional_QuotedMixedCaseNotEqualUnquoted",
a: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{`LOWER("Email")`},
},
b: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
equals: false,
},
{
name: "Unique_Functional_DifferentExpressions",
a: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
b: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"UPPER(email)"},
},
equals: false,
},
{
name: "Unique_Functional_NotEqualToPlainSameColumns",
a: &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
},
b: &UniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
},
equals: false,
},
}
for _, testCase := range testCases {
@@ -355,138 +238,6 @@ func TestIndexEquals(t *testing.T) {
}
}
func TestUniqueIndexFunctionalName(t *testing.T) {
t.Run("autogen uses uq_<table>_<hash>", func(t *testing.T) {
idx := &UniqueIndexWithExpressions{
TableName: "tag",
Expressions: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
}
assert.Equal(t, "uq_tag_a36c51df", idx.Name())
})
t.Run("same expressions produce the same name", func(t *testing.T) {
a := &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
}
b := &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
}
assert.Equal(t, a.Name(), b.Name())
})
t.Run("different expressions produce different names", func(t *testing.T) {
a := &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
}
b := &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"UPPER(email)"},
}
assert.NotEqual(t, a.Name(), b.Name())
})
t.Run("expressions in different order produce different names", func(t *testing.T) {
a := &UniqueIndexWithExpressions{
TableName: "tag",
Expressions: []string{"org_id", "LOWER(key)"},
}
b := &UniqueIndexWithExpressions{
TableName: "tag",
Expressions: []string{"LOWER(key)", "org_id"},
}
assert.NotEqual(t, a.Name(), b.Name())
})
t.Run("functional autogen differs from plain autogen for same columns", func(t *testing.T) {
plain := &UniqueIndex{
TableName: "users",
ColumnNames: []ColumnName{"email"},
}
functional := &UniqueIndexWithExpressions{
TableName: "users",
Expressions: []string{"LOWER(email)"},
}
assert.Equal(t, "uq_users_email", plain.Name())
assert.NotEqual(t, plain.Name(), functional.Name())
})
t.Run("Named() override wins over hash", func(t *testing.T) {
idx := (&UniqueIndexWithExpressions{
TableName: "tag",
Expressions: []string{"org_id", "LOWER(key)"},
}).Named("my_functional_index")
assert.Equal(t, "my_functional_index", idx.Name())
})
}
func TestNormalizeExpressions(t *testing.T) {
testCases := []struct {
name string
expressions []string
output []string
}{
{
name: "Empty",
expressions: nil,
output: nil,
},
{
name: "PlainColumnsUnchanged",
expressions: []string{"org_id", "kind"},
output: []string{"org_id", "kind"},
},
{
name: "FunctionNameCaseFolded",
expressions: []string{"LOWER(key)", "LOWER(value)"},
output: []string{"lower(key)", "lower(value)"},
},
{
name: "PostgresRenderedMatchesDeclared",
expressions: []string{"lower(key)"},
output: []string{"lower(key)"},
},
{
name: "WhitespaceCollapsedPerExpression",
expressions: []string{"LOWER( key )", "org_id"},
output: []string{"lower( key )", "org_id"},
},
{
name: "RedundantOuterParenthesesStripped",
expressions: []string{"(LOWER(key))"},
output: []string{"lower(key)"},
},
{
name: "QuotedSimpleIdentifierUnquoted",
expressions: []string{`LOWER("email")`},
output: []string{"lower(email)"},
},
{
name: "QuotedIdentifierCaseByIdentifierPreserved",
expressions: []string{`LOWER("Key")`},
output: []string{`lower("Key")`},
},
{
name: "StringLiteralCasePreserved",
expressions: []string{"COALESCE(status, 'Deleted')"},
output: []string{"coalesce(status, 'Deleted')"},
},
{
name: "NormalizedIndependentlyPerElement",
expressions: []string{"UPPER(a)", "b", "LOWER(c)"},
output: []string{"upper(a)", "b", "lower(c)"},
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.output, normalizeExpressions(testCase.expressions))
})
}
}
func TestPartialUniqueIndexName(t *testing.T) {
a := &PartialUniqueIndex{
TableName: "users",

View File

@@ -6,26 +6,25 @@ import (
"strings"
)
type expressionNormalizer struct {
input string
foldCase bool
type whereNormalizer struct {
input string
}
func (n *expressionNormalizer) hash() string {
func (n *whereNormalizer) hash() string {
hasher := fnv.New32a()
_, _ = hasher.Write([]byte(n.normalize()))
return fmt.Sprintf("%08x", hasher.Sum32())
}
func (n *expressionNormalizer) normalize() string {
expr := strings.TrimSpace(n.input)
expr = n.stripOuterParentheses(expr)
func (n *whereNormalizer) normalize() string {
where := strings.TrimSpace(n.input)
where = n.stripOuterParentheses(where)
var output strings.Builder
output.Grow(len(expr))
output.Grow(len(where))
for i := 0; i < len(expr); i++ {
switch expr[i] {
for i := 0; i < len(where); i++ {
switch where[i] {
case ' ', '\t', '\n', '\r':
if output.Len() > 0 {
last := output.String()[output.Len()-1]
@@ -34,25 +33,21 @@ func (n *expressionNormalizer) normalize() string {
}
}
case '\'':
end := n.consumeSingleQuotedLiteral(expr, i, &output)
end := n.consumeSingleQuotedLiteral(where, i, &output)
i = end
case '"':
token, end := n.consumeDoubleQuotedToken(expr, i)
token, end := n.consumeDoubleQuotedToken(where, i)
output.WriteString(token)
i = end
default:
c := expr[i]
if n.foldCase && c >= 'A' && c <= 'Z' {
c += 'a' - 'A'
}
output.WriteByte(c)
output.WriteByte(where[i])
}
}
return strings.TrimSpace(output.String())
}
func (n *expressionNormalizer) stripOuterParentheses(s string) string {
func (n *whereNormalizer) stripOuterParentheses(s string) string {
for {
s = strings.TrimSpace(s)
if len(s) < 2 || s[0] != '(' || s[len(s)-1] != ')' || !n.hasWrappingParentheses(s) {
@@ -62,7 +57,7 @@ func (n *expressionNormalizer) stripOuterParentheses(s string) string {
}
}
func (n *expressionNormalizer) hasWrappingParentheses(s string) bool {
func (n *whereNormalizer) hasWrappingParentheses(s string) bool {
depth := 0
inSingleQuotedLiteral := false
inDoubleQuotedToken := false
@@ -106,7 +101,7 @@ func (n *expressionNormalizer) hasWrappingParentheses(s string) bool {
return depth == 0
}
func (n *expressionNormalizer) consumeSingleQuotedLiteral(s string, start int, output *strings.Builder) int {
func (n *whereNormalizer) consumeSingleQuotedLiteral(s string, start int, output *strings.Builder) int {
output.WriteByte(s[start])
for i := start + 1; i < len(s); i++ {
output.WriteByte(s[i])
@@ -123,7 +118,7 @@ func (n *expressionNormalizer) consumeSingleQuotedLiteral(s string, start int, o
return len(s) - 1
}
func (n *expressionNormalizer) consumeDoubleQuotedToken(s string, start int) (string, int) {
func (n *whereNormalizer) consumeDoubleQuotedToken(s string, start int) (string, int) {
var ident strings.Builder
for i := start + 1; i < len(s); i++ {
@@ -147,7 +142,7 @@ func (n *expressionNormalizer) consumeDoubleQuotedToken(s string, start int) (st
return s[start:], len(s) - 1
}
func (n *expressionNormalizer) isSimpleUnquotedIdentifier(s string) bool {
func (n *whereNormalizer) isSimpleUnquotedIdentifier(s string) bool {
if s == "" || strings.ToLower(s) != s {
return false
}

View File

@@ -6,7 +6,7 @@ import (
"github.com/stretchr/testify/assert"
)
func TestExpressionNormalizerNormalize(t *testing.T) {
func TestWhereNormalizerNormalize(t *testing.T) {
testCases := []struct {
name string
input string
@@ -47,16 +47,11 @@ func TestExpressionNormalizerNormalize(t *testing.T) {
input: `NOT ("deleted_at" IS NOT NULL AND ("active" = false OR "status" = 'archived'))`,
output: `NOT (deleted_at IS NOT NULL AND (active = false OR status = 'archived'))`,
},
{
name: "CaseNotFoldedWithoutFlag",
input: `LOWER("email") IS NOT NULL`,
output: `LOWER(email) IS NOT NULL`,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.output, (&expressionNormalizer{input: testCase.input}).normalize())
assert.Equal(t, testCase.output, (&whereNormalizer{input: testCase.input}).normalize())
})
}
}

View File

@@ -2,7 +2,6 @@ package sqlitesqlschema
import (
"context"
"log/slog"
"strconv"
"strings"
@@ -93,6 +92,7 @@ func (provider *provider) GetIndices(ctx context.Context, tableName sqlschema.Ta
unique bool
origin string
partial bool
columns []sqlschema.ColumnName
)
if err := rows.Scan(&seq, &name, &unique, &origin, &partial); err != nil {
return nil, err
@@ -108,103 +108,53 @@ func (provider *provider) GetIndices(ctx context.Context, tableName sqlschema.Ta
continue
}
if !unique {
continue
}
columns, hasExpression, err := provider.indexKeys(ctx, name)
if err != nil {
if err := provider.
sqlstore.
BunDB().
NewRaw("SELECT name FROM PRAGMA_index_info(?)", string(name)).
Scan(ctx, &columns); err != nil {
return nil, err
}
// SQLite exposes expression keys and the predicate only in the DDL.
var ddl string
if hasExpression || partial {
if unique && partial {
var indexSQL string
if err := provider.
sqlstore.
BunDB().
NewRaw("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", name).
Scan(ctx, &ddl); err != nil {
Scan(ctx, &indexSQL); err != nil {
return nil, err
}
}
// functional partial indexes aren't representable (PartialUniqueIndex has no
// expressions); skip rather than misrepresent.
if hasExpression && partial {
provider.settings.Logger().WarnContext(ctx, "skipping functional partial unique index; not representable by sqlschema", slog.String("index", name), slog.String("table", string(tableName)))
continue
}
var index sqlschema.Index
if partial {
index = &sqlschema.PartialUniqueIndex{
where := extractWhereClause(indexSQL)
index := &sqlschema.PartialUniqueIndex{
TableName: tableName,
ColumnNames: columns,
Where: extractWhereClause(ddl),
Where: where,
}
} else if hasExpression {
index = &sqlschema.UniqueIndexWithExpressions{
TableName: tableName,
Expressions: extractIndexColumns(ddl),
if index.Name() == name {
indices = append(indices, index)
} else {
indices = append(indices, index.Named(name))
}
} else {
index = &sqlschema.UniqueIndex{
} else if unique {
index := &sqlschema.UniqueIndex{
TableName: tableName,
ColumnNames: columns,
}
}
if index.Name() != name {
index = index.Named(name)
if index.Name() == name {
indices = append(indices, index)
} else {
indices = append(indices, index.Named(name))
}
}
indices = append(indices, index)
}
return indices, nil
}
// indexKeys returns an index's plain key columns and whether any key is an
// expression (SQLite marks expression keys with cid = -2 and a NULL name).
func (provider *provider) indexKeys(ctx context.Context, name string) ([]sqlschema.ColumnName, bool, error) {
rows, err := provider.
sqlstore.
BunDB().
QueryContext(ctx, "SELECT cid, name FROM PRAGMA_index_info(?)", name)
if err != nil {
return nil, false, err
}
defer func() {
if err := rows.Close(); err != nil {
provider.settings.Logger().ErrorContext(ctx, "error closing rows", errors.Attr(err))
}
}()
var (
columns []sqlschema.ColumnName
hasExpression bool
)
for rows.Next() {
var (
cid int
columnName *string
)
if err := rows.Scan(&cid, &columnName); err != nil {
return nil, false, err
}
if cid == -2 {
hasExpression = true
continue
}
if columnName != nil {
columns = append(columns, sqlschema.ColumnName(*columnName))
}
}
return columns, hasExpression, nil
}
func (provider *provider) ToggleFKEnforcement(ctx context.Context, db bun.IDB, on bool) error {
_, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ?", on)
if err != nil {
@@ -223,92 +173,6 @@ func (provider *provider) ToggleFKEnforcement(ctx context.Context, db bun.IDB, o
return errors.NewInternalf(errors.CodeInternal, "foreign_keys(actual: %s, expected: %s), maybe a transaction is in progress?", strconv.FormatBool(val), strconv.FormatBool(on))
}
// extractIndexColumns returns an index's key entries: the comma-separated items
// in the first top-level parenthesised list after ON, each verbatim (trimmed).
// Quotes and nested parens are respected so their commas don't split keys.
// SQLite reports expression keys with a NULL name in PRAGMA_index_info, so their
// text is only recoverable from the DDL.
func extractIndexColumns(sql string) []string {
var (
inSingleQuotedLiteral bool
inDoubleQuotedIdentifier bool
inBacktickQuotedIdentifier bool
inBracketQuotedIdentifier bool
depth int
started bool
keyStart int
columns []string
)
for i := 0; i < len(sql); i++ {
switch sql[i] {
case '\'':
if inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
break
}
if inSingleQuotedLiteral && i+1 < len(sql) && sql[i+1] == '\'' {
i++
continue
}
inSingleQuotedLiteral = !inSingleQuotedLiteral
continue
case '"':
if inSingleQuotedLiteral || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
break
}
if inDoubleQuotedIdentifier && i+1 < len(sql) && sql[i+1] == '"' {
i++
continue
}
inDoubleQuotedIdentifier = !inDoubleQuotedIdentifier
continue
case '`':
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBracketQuotedIdentifier {
break
}
inBacktickQuotedIdentifier = !inBacktickQuotedIdentifier
continue
case '[':
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
break
}
inBracketQuotedIdentifier = true
continue
case ']':
if inBracketQuotedIdentifier {
inBracketQuotedIdentifier = false
}
continue
}
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
continue
}
switch sql[i] {
case '(':
depth++
if depth == 1 && !started {
started = true
keyStart = i + 1
}
case ')':
depth--
if depth == 0 && started {
columns = append(columns, strings.TrimSpace(sql[keyStart:i]))
return columns
}
case ',':
if depth == 1 {
columns = append(columns, strings.TrimSpace(sql[keyStart:i]))
keyStart = i + 1
}
}
}
return columns
}
func extractWhereClause(sql string) string {
lastWhere := -1
inSingleQuotedLiteral := false

View File

@@ -50,93 +50,3 @@ func TestExtractWhereClause(t *testing.T) {
})
}
}
func TestExtractIndexColumns(t *testing.T) {
testCases := []struct {
name string
sql string
columns []string
}{
{
name: "SingleColumn",
sql: `CREATE UNIQUE INDEX "idx" ON "users" ("email")`,
columns: []string{`"email"`},
},
{
name: "SingleExpression",
sql: `CREATE UNIQUE INDEX "idx" ON "users" (LOWER(email))`,
columns: []string{"LOWER(email)"},
},
{
name: "MixedColumnsAndExpressions",
sql: `CREATE UNIQUE INDEX "idx" ON "tag" (org_id, kind, LOWER(key), LOWER(value))`,
columns: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
},
{
name: "CommaInsideStringLiteralNotSplit",
sql: `CREATE UNIQUE INDEX "idx" ON "users" (LOWER(TRIM(first_name) || ', ' || last_name))`,
columns: []string{"LOWER(TRIM(first_name) || ', ' || last_name)"},
},
{
name: "CommaInsideQuotedIdentifierNotSplit",
sql: `CREATE UNIQUE INDEX "idx" ON "t" ("weird,col", LOWER(x))`,
columns: []string{`"weird,col"`, "LOWER(x)"},
},
{
name: "WhereClauseAfterKeyListIgnored",
sql: `CREATE UNIQUE INDEX "idx" ON "tag" (LOWER(key)) WHERE deleted = 0`,
columns: []string{"LOWER(key)"},
},
{
name: "WhereClauseWithParensAfterKeyListIgnored",
sql: `CREATE UNIQUE INDEX "idx" ON "tag" (LOWER(key)) WHERE (deleted = 0 AND kind = 'a')`,
columns: []string{"LOWER(key)"},
},
{
name: "CommaInsideBacktickIdentifierNotSplit",
sql: "CREATE UNIQUE INDEX `idx` ON `t` (`weird,col`, x)",
columns: []string{"`weird,col`", "x"},
},
{
name: "CommaInsideBracketIdentifierNotSplit",
sql: `CREATE UNIQUE INDEX [idx] ON [t] ([weird,col], x)`,
columns: []string{"[weird,col]", "x"},
},
{
name: "EscapedDoubleQuoteInIdentifier",
sql: `CREATE UNIQUE INDEX "idx" ON "t" ("a""b", x)`,
columns: []string{`"a""b"`, "x"},
},
{
name: "EscapedSingleQuoteInLiteral",
sql: `CREATE UNIQUE INDEX "idx" ON "t" (COALESCE(x, 'it''s, ok'))`,
columns: []string{"COALESCE(x, 'it''s, ok')"},
},
{
name: "NestedFunctionCallCommasNotSplit",
sql: `CREATE UNIQUE INDEX "idx" ON "t" (COALESCE(a, b, c), d)`,
columns: []string{"COALESCE(a, b, c)", "d"},
},
{
name: "NewlinesAndTabsTrimmed",
sql: "CREATE UNIQUE INDEX \"idx\" ON \"t\" (\n\tLOWER(key),\n\torg_id\n)",
columns: []string{"LOWER(key)", "org_id"},
},
{
name: "CollateAndOrderSuffixKeptVerbatim",
sql: `CREATE UNIQUE INDEX "idx" ON "t" (LOWER(key) COLLATE NOCASE, value DESC)`,
columns: []string{"LOWER(key) COLLATE NOCASE", "value DESC"},
},
{
name: "NoKeyListReturnsNil",
sql: `CREATE INDEX "idx" ON "t"`,
columns: nil,
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
assert.Equal(t, testCase.columns, extractIndexColumns(testCase.sql))
})
}
}

View File

@@ -38,7 +38,7 @@ func newTestDashboardV2(t *testing.T, orgID valuer.UUID, source Source) *Dashboa
LineInterpolation: LineInterpolationSpline,
LineStyle: LineStyleSolid,
FillMode: FillModeSolid,
SpanGaps: SpanGaps{FillLessThan: valuer.MustParseTextDuration("60s")},
SpanGaps: SpanGaps{FillLessThan: "60s"},
},
Legend: Legend{Position: LegendPositionBottom, Mode: LegendModeList},
},

View File

@@ -6,7 +6,6 @@ import (
"os"
"strings"
"testing"
"time"
"github.com/SigNoz/signoz/pkg/errors"
"github.com/perses/spec/go/dashboard"
@@ -753,7 +752,7 @@ func TestInvalidateBadPanelSpecValues(t *testing.T) {
"spec": {
"plugin": {
"kind": "signoz/TimeSeriesPanel",
"spec": {"chartAppearance": {"spanGaps": {"fillLessThan": "notaduration"}}}
"spec": {"chartAppearance": {"spanGaps": {"fillOnlyBelow": true, "fillLessThan": "notaduration"}}}
}
}
}
@@ -1371,23 +1370,49 @@ func TestSpanGaps(t *testing.T) {
t.Run("defaults", func(t *testing.T) {
var sg SpanGaps
assert.False(t, sg.FillOnlyBelow, "expected FillOnlyBelow default false")
assert.True(t, sg.FillLessThan.IsZero(), "expected FillLessThan default zero")
assert.Empty(t, sg.FillLessThan, "expected FillLessThan default empty")
})
t.Run("fillOnlyBelow true", func(t *testing.T) {
sg := unmarshal(t, `{"fillOnlyBelow": true}`)
sg := unmarshal(t, `{"fillOnlyBelow": true, "fillLessThan": "5m"}`)
assert.True(t, sg.FillOnlyBelow)
})
t.Run("fillLessThan duration", func(t *testing.T) {
sg := unmarshal(t, `{"fillOnlyBelow": false, "fillLessThan": "5m"}`)
t.Run("fillLessThan ignored when fillOnlyBelow is false", func(t *testing.T) {
sg := unmarshal(t, `{"fillOnlyBelow": false, "fillLessThan": ""}`)
assert.False(t, sg.FillOnlyBelow)
assert.Equal(t, 5*time.Minute, sg.FillLessThan.Duration())
assert.Empty(t, sg.FillLessThan)
})
t.Run("fillLessThan duration", func(t *testing.T) {
sg := unmarshal(t, `{"fillOnlyBelow": true, "fillLessThan": "5m"}`)
assert.True(t, sg.FillOnlyBelow)
assert.Equal(t, "5m", sg.FillLessThan)
})
t.Run("fillLessThan compound duration", func(t *testing.T) {
sg := unmarshal(t, `{"fillLessThan": "1h30m"}`)
assert.Equal(t, 90*time.Minute, sg.FillLessThan.Duration())
sg := unmarshal(t, `{"fillOnlyBelow": true, "fillLessThan": "1h30m"}`)
assert.Equal(t, "1h30m", sg.FillLessThan)
})
t.Run("fillLessThan day duration", func(t *testing.T) {
sg := unmarshal(t, `{"fillOnlyBelow": true, "fillLessThan": "1d"}`)
assert.Equal(t, "1d", sg.FillLessThan)
})
t.Run("fillLessThan required when fillOnlyBelow is true", func(t *testing.T) {
var sg SpanGaps
require.Error(t, json.Unmarshal([]byte(`{"fillOnlyBelow": true}`), &sg))
})
t.Run("invalid fillLessThan rejected on unmarshal", func(t *testing.T) {
var sg SpanGaps
require.Error(t, json.Unmarshal([]byte(`{"fillOnlyBelow": true, "fillLessThan": "not-a-duration"}`), &sg))
})
t.Run("non-positive fillLessThan rejected on unmarshal", func(t *testing.T) {
var sg SpanGaps
require.Error(t, json.Unmarshal([]byte(`{"fillOnlyBelow": true, "fillLessThan": "0s"}`), &sg))
})
}

View File

@@ -8,6 +8,7 @@ import (
qb "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/prometheus/common/model"
"github.com/swaggest/jsonschema-go"
)
@@ -621,8 +622,39 @@ func (fm *FillMode) UnmarshalJSON(data []byte) error {
}
type SpanGaps struct {
FillOnlyBelow bool `json:"fillOnlyBelow" description:"Controls whether lines connect across null values. When false (default), all gaps are connected. When true, only gaps smaller than fillLessThan are connected."`
FillLessThan valuer.TextDuration `json:"fillLessThan" description:"The maximum gap size to connect when fillOnlyBelow is true. Gaps larger than this duration are left disconnected."`
FillOnlyBelow bool `json:"fillOnlyBelow" description:"Controls whether lines connect across null values. When false (default), all gaps are connected. When true, only gaps smaller than fillLessThan are connected."`
FillLessThan string `json:"fillLessThan" description:"The maximum gap size to connect when fillOnlyBelow is true. Gaps larger than this duration are left disconnected."`
}
func (sg *SpanGaps) UnmarshalJSON(data []byte) error {
type alias SpanGaps
var tmp alias
if err := json.Unmarshal(data, &tmp); err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid spanGaps")
}
*sg = SpanGaps(tmp)
return sg.validate()
}
// validate enforces FillLessThan only when FillOnlyBelow is set, since that is
// the only mode in which it applies. It must then be a valid positive duration.
// prometheus's parser accepts day/week/year units (e.g. "1d"); time.ParseDuration
// caps at hours.
func (sg SpanGaps) validate() error {
if !sg.FillOnlyBelow {
return nil
}
if sg.FillLessThan == "" {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "spanGaps.fillLessThan is required when fillOnlyBelow is true")
}
d, err := model.ParseDuration(sg.FillLessThan)
if err != nil {
return errors.WrapInvalidInputf(err, ErrCodeDashboardInvalidInput, "invalid spanGaps.fillLessThan duration %q", sg.FillLessThan)
}
if d <= 0 {
return errors.NewInvalidInputf(ErrCodeDashboardInvalidInput, "spanGaps.fillLessThan duration must be positive, got %q", sg.FillLessThan)
}
return nil
}
type PrecisionOption struct{ valuer.String }

View File

@@ -76,7 +76,7 @@
"showPoints": false,
"lineStyle": "solid",
"fillMode": "none",
"spanGaps": {"fillOnlyBelow": true}
"spanGaps": {"fillOnlyBelow": true, "fillLessThan": "5m"}
},
"legend": {
"position": "bottom"