mirror of
https://github.com/SigNoz/signoz.git
synced 2026-07-08 15:40:40 +01:00
Compare commits
22 Commits
main
...
nv/functio
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4dd134494c | ||
|
|
0aff9ef889 | ||
|
|
2d653d1a6e | ||
|
|
0261b172f1 | ||
|
|
36aa3dcf0b | ||
|
|
6016f3d0b2 | ||
|
|
c31768f5fc | ||
|
|
84d75c637f | ||
|
|
226c3a978c | ||
|
|
ab9d78d314 | ||
|
|
2efad1c634 | ||
|
|
1b75857a6b | ||
|
|
531b637b6a | ||
|
|
9fdb62abbe | ||
|
|
fde7a5d818 | ||
|
|
83bbcd4b41 | ||
|
|
eafd71f205 | ||
|
|
e6a8736a1a | ||
|
|
0d744cf94c | ||
|
|
748dff9489 | ||
|
|
68da3b2beb | ||
|
|
5574e08ddc |
@@ -3,6 +3,7 @@ package postgressqlschema
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"log/slog"
|
||||
|
||||
"github.com/uptrace/bun"
|
||||
|
||||
@@ -214,6 +215,20 @@ WHERE
|
||||
}
|
||||
|
||||
func (provider *provider) GetIndices(ctx context.Context, name sqlschema.TableName) ([]sqlschema.Index, error) {
|
||||
// A key is one entry an index is built on: a column (plain) or an expression
|
||||
// like lower(x) (functional). Returns one row per key of each non-constraint-
|
||||
// backed index on the table, ordered by position. Keys are enumerated with
|
||||
// generate_series over indnkeyatts (count of key columns, excluding non-key
|
||||
// INCLUDE columns), so expression keys are covered too. Reconstruction uses:
|
||||
//
|
||||
// index_name | index the key belongs to
|
||||
// is_expression | true if the key is an expression, not a plain column
|
||||
// key_def | rendered key text, e.g. org_id or lower(key)
|
||||
// column_name | column name for plain keys; NULL for expression keys
|
||||
// predicate | partial-index WHERE; NULL if not partial (same every row)
|
||||
//
|
||||
// table_name/unique/primary/column_position are carried over from the prior
|
||||
// query (unique gates the loop, column_position orders the keys).
|
||||
rows, err := provider.
|
||||
sqlstore.
|
||||
BunDB().
|
||||
@@ -224,19 +239,21 @@ SELECT
|
||||
i.indisunique AS unique,
|
||||
i.indisprimary AS primary,
|
||||
a.attname AS column_name,
|
||||
array_position(i.indkey, a.attnum) AS column_position,
|
||||
n AS column_position,
|
||||
(i.indkey[n - 1] = 0) AS is_expression,
|
||||
pg_get_indexdef(i.indexrelid, n, true) AS key_def,
|
||||
pg_get_expr(i.indpred, i.indrelid) AS predicate
|
||||
FROM
|
||||
pg_index i
|
||||
LEFT JOIN pg_class ct ON ct.oid = i.indrelid
|
||||
LEFT JOIN pg_class ci ON ci.oid = i.indexrelid
|
||||
LEFT JOIN pg_attribute a ON a.attrelid = ct.oid
|
||||
LEFT JOIN pg_constraint con ON con.conindid = i.indexrelid
|
||||
CROSS JOIN generate_series(1, i.indnkeyatts) AS n
|
||||
LEFT JOIN pg_attribute a ON a.attrelid = ct.oid AND a.attnum = i.indkey[n - 1]
|
||||
WHERE
|
||||
a.attnum = ANY(i.indkey)
|
||||
AND con.oid IS NULL
|
||||
ct.relname = ?
|
||||
AND ct.relkind = 'r'
|
||||
AND ct.relname = ?
|
||||
AND con.oid IS NULL
|
||||
ORDER BY index_name, column_position`, string(name))
|
||||
if err != nil {
|
||||
return nil, provider.sqlstore.WrapNotFoundErrf(err, errors.CodeNotFound, "no indices for table (%s) found", name)
|
||||
@@ -251,6 +268,10 @@ ORDER BY index_name, column_position`, string(name))
|
||||
type indexEntry struct {
|
||||
columns []sqlschema.ColumnName
|
||||
predicate *string
|
||||
// keyDefs holds every key rendered by pg_get_indexdef, in key order; used
|
||||
// for functional indexes where a key may be an expression like lower(x).
|
||||
keyDefs []string
|
||||
hasExpression bool
|
||||
}
|
||||
|
||||
uniqueIndicesMap := make(map[string]*indexEntry)
|
||||
@@ -260,30 +281,45 @@ ORDER BY index_name, column_position`, string(name))
|
||||
indexName string
|
||||
unique bool
|
||||
primary bool
|
||||
columnName string
|
||||
// starts from 0 and is unused in this function, this is to ensure that the column names are in the correct order
|
||||
columnName *string
|
||||
// starts from 1 and is unused in this function, this is to ensure that the column names are in the correct order
|
||||
columnPosition int
|
||||
isExpression bool
|
||||
keyDef string
|
||||
predicate *string
|
||||
)
|
||||
|
||||
if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition, &predicate); err != nil {
|
||||
if err := rows.Scan(&tableName, &indexName, &unique, &primary, &columnName, &columnPosition, &isExpression, &keyDef, &predicate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if unique {
|
||||
if _, ok := uniqueIndicesMap[indexName]; !ok {
|
||||
uniqueIndicesMap[indexName] = &indexEntry{
|
||||
columns: []sqlschema.ColumnName{sqlschema.ColumnName(columnName)},
|
||||
predicate: predicate,
|
||||
}
|
||||
} else {
|
||||
uniqueIndicesMap[indexName].columns = append(uniqueIndicesMap[indexName].columns, sqlschema.ColumnName(columnName))
|
||||
}
|
||||
if !unique {
|
||||
continue
|
||||
}
|
||||
|
||||
entry, ok := uniqueIndicesMap[indexName]
|
||||
if !ok {
|
||||
entry = &indexEntry{predicate: predicate}
|
||||
uniqueIndicesMap[indexName] = entry
|
||||
}
|
||||
|
||||
entry.keyDefs = append(entry.keyDefs, keyDef)
|
||||
if isExpression {
|
||||
entry.hasExpression = true
|
||||
} else if columnName != nil {
|
||||
entry.columns = append(entry.columns, sqlschema.ColumnName(*columnName))
|
||||
}
|
||||
}
|
||||
|
||||
indices := make([]sqlschema.Index, 0)
|
||||
for indexName, entry := range uniqueIndicesMap {
|
||||
// functional partial indexes aren't representable (PartialUniqueIndex has no
|
||||
// expressions); skip rather than misrepresent.
|
||||
if entry.hasExpression && entry.predicate != nil {
|
||||
provider.settings.Logger().WarnContext(ctx, "skipping functional partial unique index; not representable by sqlschema", slog.String("index", indexName), slog.String("table", string(name)))
|
||||
continue
|
||||
}
|
||||
|
||||
if entry.predicate != nil {
|
||||
index := &sqlschema.PartialUniqueIndex{
|
||||
TableName: name,
|
||||
@@ -291,6 +327,17 @@ ORDER BY index_name, column_position`, string(name))
|
||||
Where: *entry.predicate,
|
||||
}
|
||||
|
||||
if index.Name() == indexName {
|
||||
indices = append(indices, index)
|
||||
} else {
|
||||
indices = append(indices, index.Named(indexName))
|
||||
}
|
||||
} else if entry.hasExpression {
|
||||
index := &sqlschema.UniqueIndex{
|
||||
TableName: name,
|
||||
Expressions: entry.keyDefs,
|
||||
}
|
||||
|
||||
if index.Name() == indexName {
|
||||
indices = append(indices, index)
|
||||
} else {
|
||||
|
||||
112
ee/sqlschema/postgressqlschema/roundtrip_test.go
Normal file
112
ee/sqlschema/postgressqlschema/roundtrip_test.go
Normal file
@@ -0,0 +1,112 @@
|
||||
//go:build integration
|
||||
|
||||
// Requires a running postgres (make devenv-postgres); excluded from the default
|
||||
// test run. Run with: go test -tags integration ./...
|
||||
|
||||
package postgressqlschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/ee/sqlstore/postgressqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestGetIndicesRoundTrip creates each index shape on a throwaway table in devenv
|
||||
// postgres and checks GetIndices reconstructs an equal index, exercising every
|
||||
// query branch: plain vs expression keys, multi-key ordering, partial predicate,
|
||||
// and the skipped functional-partial case. Skips if postgres is unreachable.
|
||||
func TestGetIndicesRoundTrip(t *testing.T) {
|
||||
dsn := os.Getenv("TEST_POSTGRES_DSN")
|
||||
if dsn == "" {
|
||||
dsn = devenvPostgresDSN
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
cfg := sqlstore.Config{
|
||||
Provider: "postgres",
|
||||
Postgres: sqlstore.PostgresConfig{DSN: dsn},
|
||||
Connection: sqlstore.ConnectionConfig{MaxOpenConns: 10, MaxConnLifetime: time.Minute},
|
||||
}
|
||||
|
||||
providerSettings := instrumentationtest.New().ToProviderSettings()
|
||||
store, err := postgressqlstore.New(ctx, providerSettings, cfg)
|
||||
if err != nil {
|
||||
t.Skipf("postgres unreachable at %s (run `make devenv-postgres`): %v", dsn, err)
|
||||
}
|
||||
|
||||
pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
if err := store.SQLDB().PingContext(pingCtx); err != nil {
|
||||
t.Skipf("postgres unreachable at %s (run `make devenv-postgres`): %v", dsn, err)
|
||||
}
|
||||
|
||||
schema, err := New(ctx, providerSettings, sqlschema.Config{}, store)
|
||||
require.NoError(t, err)
|
||||
|
||||
const table = "sqlschema_roundtrip"
|
||||
|
||||
exec := func(sql string) {
|
||||
_, err := store.BunDB().ExecContext(ctx, sql)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
reset := func() {
|
||||
exec(`DROP TABLE IF EXISTS ` + table)
|
||||
exec(`CREATE TABLE ` + table + ` (a text, b text, c text)`)
|
||||
}
|
||||
t.Cleanup(func() { _, _ = store.BunDB().ExecContext(ctx, `DROP TABLE IF EXISTS `+table) })
|
||||
|
||||
roundTrip := func(t *testing.T, declared sqlschema.Index) {
|
||||
reset()
|
||||
for _, sql := range schema.Operator().CreateIndex(declared) {
|
||||
exec(string(sql))
|
||||
}
|
||||
|
||||
indices, err := schema.GetIndices(ctx, table)
|
||||
require.NoError(t, err)
|
||||
|
||||
var got sqlschema.Index
|
||||
for _, index := range indices {
|
||||
if index.Name() == declared.Name() {
|
||||
got = index
|
||||
}
|
||||
}
|
||||
require.NotNil(t, got, "GetIndices did not return %q; returned %d indices", declared.Name(), len(indices))
|
||||
require.True(t, declared.Equals(got), "round-tripped index should equal the declared one; got %#v", got)
|
||||
}
|
||||
|
||||
t.Run("PlainSingleColumn", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, ColumnNames: []sqlschema.ColumnName{"a"}})
|
||||
})
|
||||
|
||||
t.Run("PlainMultiColumn", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, ColumnNames: []sqlschema.ColumnName{"a", "b"}})
|
||||
})
|
||||
|
||||
t.Run("FunctionalSingleExpression", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, Expressions: []string{"LOWER(a)"}})
|
||||
})
|
||||
|
||||
t.Run("FunctionalMixedKeys", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, Expressions: []string{"a", "LOWER(b)", "LOWER(c)"}})
|
||||
})
|
||||
|
||||
t.Run("PartialUnique", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.PartialUniqueIndex{TableName: table, ColumnNames: []sqlschema.ColumnName{"a"}, Where: "b IS NOT NULL"})
|
||||
})
|
||||
|
||||
t.Run("FunctionalPartialIsSkipped", func(t *testing.T) {
|
||||
reset()
|
||||
exec(`CREATE UNIQUE INDEX rt_functional_partial ON ` + table + ` (LOWER(a)) WHERE b IS NOT NULL`)
|
||||
|
||||
indices, err := schema.GetIndices(ctx, table)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, indices, "functional partial unique index must be skipped, not reconstructed")
|
||||
})
|
||||
}
|
||||
@@ -98,7 +98,7 @@ func (s *store) CreateOrGet(ctx context.Context, tags []*tagtypes.Tag) ([]*tagty
|
||||
BunDBCtx(ctx).
|
||||
NewInsert().
|
||||
Model(&tags).
|
||||
// On("CONFLICT (org_id, kind, (LOWER(key)), (LOWER(value))) DO UPDATE").
|
||||
On("CONFLICT (org_id, kind, (LOWER(key)), (LOWER(value))) DO UPDATE").
|
||||
Set("key = tag.key").
|
||||
Returning("*").
|
||||
Scan(ctx)
|
||||
|
||||
@@ -88,62 +88,60 @@ func TestStore_Create_PopulatesIDsOnFreshInsert(t *testing.T) {
|
||||
assert.Equal(t, preIDB, stored["team\x00blr"].ID)
|
||||
}
|
||||
|
||||
// todo (@namanverma): uncomment once unique index is there.
|
||||
//
|
||||
// func TestStore_Create_ConflictReturnsExistingRowID(t *testing.T) {
|
||||
// ctx := context.Background()
|
||||
// sqlstore := newTestStore(t)
|
||||
// s := NewStore(sqlstore)
|
||||
func TestStore_Create_ConflictReturnsExistingRowID(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
sqlstore := newTestStore(t)
|
||||
s := NewStore(sqlstore)
|
||||
|
||||
// orgID := valuer.GenerateUUID()
|
||||
orgID := valuer.GenerateUUID()
|
||||
|
||||
// // Simulate a concurrent insert: someone else has already inserted "tag:Database".
|
||||
// winner := tagtypes.NewTag(orgID, dashboardKind, "tag", "Database")
|
||||
// _, err := s.CreateOrGet(ctx, []*tagtypes.Tag{winner})
|
||||
// require.NoError(t, err)
|
||||
// winnerID := winner.ID
|
||||
// Simulate a concurrent insert: someone else has already inserted "tag:Database".
|
||||
winner := tagtypes.NewTag(orgID, dashboardKind, "tag", "Database")
|
||||
_, err := s.CreateOrGet(ctx, []*tagtypes.Tag{winner})
|
||||
require.NoError(t, err)
|
||||
winnerID := winner.ID
|
||||
|
||||
// // Now our request runs with a different pre-generated ID for the same
|
||||
// // (key, value) — case differs but the functional unique index collapses
|
||||
// // them. RETURNING should overwrite our stale ID with winner's ID.
|
||||
// loser := tagtypes.NewTag(orgID, dashboardKind, "TAG", "DATABASE")
|
||||
// loserPreID := loser.ID
|
||||
// require.NotEqual(t, winnerID, loserPreID, "pre-generated IDs must differ for this test to be meaningful")
|
||||
// Now our request runs with a different pre-generated ID for the same
|
||||
// (key, value) — case differs but the functional unique index collapses
|
||||
// them. RETURNING should overwrite our stale ID with winner's ID.
|
||||
loser := tagtypes.NewTag(orgID, dashboardKind, "TAG", "DATABASE")
|
||||
loserPreID := loser.ID
|
||||
require.NotEqual(t, winnerID, loserPreID, "pre-generated IDs must differ for this test to be meaningful")
|
||||
|
||||
// got, err := s.CreateOrGet(ctx, []*tagtypes.Tag{loser})
|
||||
// require.NoError(t, err)
|
||||
// require.Len(t, got, 1)
|
||||
got, err := s.CreateOrGet(ctx, []*tagtypes.Tag{loser})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 1)
|
||||
|
||||
// assert.Equal(t, winnerID, got[0].ID, "returned slice should carry the existing row's ID, not our stale one")
|
||||
// assert.Equal(t, winnerID, loser.ID, "input slice element is mutated in place")
|
||||
assert.Equal(t, winnerID, got[0].ID, "returned slice should carry the existing row's ID, not our stale one")
|
||||
assert.Equal(t, winnerID, loser.ID, "input slice element is mutated in place")
|
||||
|
||||
// // And the DB still has exactly one row for that (lower(key), lower(value)) — winner's, with winner's casing.
|
||||
// stored := tagsByLowerKeyValue(t, sqlstore.BunDB())
|
||||
// require.Len(t, stored, 1)
|
||||
// assert.Equal(t, winnerID, stored["tag\x00database"].ID)
|
||||
// assert.Equal(t, "tag", stored["tag\x00database"].Key, "winner's casing preserved in key")
|
||||
// assert.Equal(t, "Database", stored["tag\x00database"].Value, "winner's casing preserved in value")
|
||||
// }
|
||||
// And the DB still has exactly one row for that (lower(key), lower(value)) — winner's, with winner's casing.
|
||||
stored := tagsByLowerKeyValue(t, sqlstore.BunDB())
|
||||
require.Len(t, stored, 1)
|
||||
assert.Equal(t, winnerID, stored["tag\x00database"].ID)
|
||||
assert.Equal(t, "tag", stored["tag\x00database"].Key, "winner's casing preserved in key")
|
||||
assert.Equal(t, "Database", stored["tag\x00database"].Value, "winner's casing preserved in value")
|
||||
}
|
||||
|
||||
// func TestStore_Create_MixedFreshAndConflict(t *testing.T) {
|
||||
// ctx := context.Background()
|
||||
// sqlstore := newTestStore(t)
|
||||
// s := NewStore(sqlstore)
|
||||
func TestStore_Create_MixedFreshAndConflict(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
sqlstore := newTestStore(t)
|
||||
s := NewStore(sqlstore)
|
||||
|
||||
// orgID := valuer.GenerateUUID()
|
||||
// pre := tagtypes.NewTag(orgID, dashboardKind, "tag", "Database")
|
||||
// _, err := s.CreateOrGet(ctx, []*tagtypes.Tag{pre})
|
||||
// require.NoError(t, err)
|
||||
// preExistingID := pre.ID
|
||||
orgID := valuer.GenerateUUID()
|
||||
pre := tagtypes.NewTag(orgID, dashboardKind, "tag", "Database")
|
||||
_, err := s.CreateOrGet(ctx, []*tagtypes.Tag{pre})
|
||||
require.NoError(t, err)
|
||||
preExistingID := pre.ID
|
||||
|
||||
// conflict := tagtypes.NewTag(orgID, dashboardKind, "tag", "Database")
|
||||
// fresh := tagtypes.NewTag(orgID, dashboardKind, "team", "BLR")
|
||||
// freshPreID := fresh.ID
|
||||
conflict := tagtypes.NewTag(orgID, dashboardKind, "tag", "Database")
|
||||
fresh := tagtypes.NewTag(orgID, dashboardKind, "team", "BLR")
|
||||
freshPreID := fresh.ID
|
||||
|
||||
// got, err := s.CreateOrGet(ctx, []*tagtypes.Tag{conflict, fresh})
|
||||
// require.NoError(t, err)
|
||||
// require.Len(t, got, 2)
|
||||
got, err := s.CreateOrGet(ctx, []*tagtypes.Tag{conflict, fresh})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, got, 2)
|
||||
|
||||
// assert.Equal(t, preExistingID, got[0].ID, "conflicting row's ID overwritten with the existing row's")
|
||||
// assert.Equal(t, freshPreID, got[1].ID, "fresh row's pre-generated ID is preserved")
|
||||
// }
|
||||
assert.Equal(t, preExistingID, got[0].ID, "conflicting row's ID overwritten with the existing row's")
|
||||
assert.Equal(t, freshPreID, got[1].ID, "fresh row's pre-generated ID is preserved")
|
||||
}
|
||||
|
||||
@@ -218,6 +218,7 @@ func NewSQLMigrationProviderFactories(
|
||||
sqlmigration.NewMigrateSSORoleMappingNamesFactory(sqlstore),
|
||||
sqlmigration.NewAddMetricReductionRulesFactory(sqlstore, sqlschema),
|
||||
sqlmigration.NewRemoveOrganizationTuplesFactory(sqlstore),
|
||||
sqlmigration.NewAddTagUniqueIndexFactory(sqlstore, sqlschema),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
59
pkg/sqlmigration/099_add_tag_unique_index.go
Normal file
59
pkg/sqlmigration/099_add_tag_unique_index.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package sqlmigration
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/uptrace/bun"
|
||||
"github.com/uptrace/bun/migrate"
|
||||
)
|
||||
|
||||
type addTagUniqueIndex struct {
|
||||
sqlstore sqlstore.SQLStore
|
||||
sqlschema sqlschema.SQLSchema
|
||||
}
|
||||
|
||||
func NewAddTagUniqueIndexFactory(sqlstore sqlstore.SQLStore, sqlschema sqlschema.SQLSchema) factory.ProviderFactory[SQLMigration, Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("add_tag_unique_index"), func(ctx context.Context, ps factory.ProviderSettings, c Config) (SQLMigration, error) {
|
||||
return &addTagUniqueIndex{
|
||||
sqlstore: sqlstore,
|
||||
sqlschema: sqlschema,
|
||||
}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func (migration *addTagUniqueIndex) Register(migrations *migrate.Migrations) error {
|
||||
return migrations.Register(migration.Up, migration.Down)
|
||||
}
|
||||
|
||||
func (migration *addTagUniqueIndex) Up(ctx context.Context, db *bun.DB) error {
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = tx.Rollback()
|
||||
}()
|
||||
|
||||
sqls := migration.sqlschema.Operator().CreateIndex(
|
||||
&sqlschema.UniqueIndex{
|
||||
TableName: "tag",
|
||||
Expressions: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
|
||||
},
|
||||
)
|
||||
|
||||
for _, sql := range sqls {
|
||||
if _, err := tx.ExecContext(ctx, string(sql)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (migration *addTagUniqueIndex) Down(_ context.Context, _ *bun.DB) error {
|
||||
return nil
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package sqlschema
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
@@ -49,13 +51,29 @@ type Index interface {
|
||||
ToDropSQL(fmter SQLFormatter) []byte
|
||||
}
|
||||
|
||||
// UniqueIndex is either plain or functional: ColumnNames and Expressions are
|
||||
// mutually exclusive and setting both panics.
|
||||
//
|
||||
// - Plain: set only ColumnNames. Columns are identifier-quoted and the
|
||||
// auto-generated name is a readable join (uq_t_a_b ON t (a, b)).
|
||||
// - Functional (e.g. LOWER(col)): set only Expressions, one per key, emitted
|
||||
// verbatim (caller owns well-formedness); plain columns go in as bare
|
||||
// identifiers. The auto-name uses a hash suffix (uq_t_<hash>) since
|
||||
// expressions aren't valid identifier fragments. Use this only when at least
|
||||
// one key is a real expression; an all-identifier Expressions reconstructs as
|
||||
// ColumnNames on read-back and won't compare equal — use ColumnNames instead.
|
||||
type UniqueIndex struct {
|
||||
TableName TableName
|
||||
ColumnNames []ColumnName
|
||||
Expressions []string
|
||||
name string
|
||||
}
|
||||
|
||||
func (index *UniqueIndex) Name() string {
|
||||
if len(index.ColumnNames) > 0 && len(index.Expressions) > 0 {
|
||||
panic("sqlschema: UniqueIndex sets both ColumnNames and Expressions; they are mutually exclusive — for a functional index put every key (plain columns as bare identifiers) in Expressions and leave ColumnNames empty")
|
||||
}
|
||||
|
||||
if index.name != "" {
|
||||
return index.name
|
||||
}
|
||||
@@ -65,6 +83,14 @@ func (index *UniqueIndex) Name() string {
|
||||
b.WriteString("_")
|
||||
b.WriteString(string(index.TableName))
|
||||
b.WriteString("_")
|
||||
|
||||
if len(index.Expressions) > 0 {
|
||||
hasher := fnv.New32a()
|
||||
_, _ = hasher.Write([]byte(strings.Join(normalizeExpressions(index.Expressions), "\x00")))
|
||||
fmt.Fprintf(&b, "%08x", hasher.Sum32())
|
||||
return b.String()
|
||||
}
|
||||
|
||||
for i, column := range index.ColumnNames {
|
||||
if i > 0 {
|
||||
b.WriteString("_")
|
||||
@@ -77,10 +103,13 @@ func (index *UniqueIndex) Name() string {
|
||||
func (index *UniqueIndex) Named(name string) Index {
|
||||
copyOfColumnNames := make([]ColumnName, len(index.ColumnNames))
|
||||
copy(copyOfColumnNames, index.ColumnNames)
|
||||
copyOfExpressions := make([]string, len(index.Expressions))
|
||||
copy(copyOfExpressions, index.Expressions)
|
||||
|
||||
return &UniqueIndex{
|
||||
TableName: index.TableName,
|
||||
ColumnNames: copyOfColumnNames,
|
||||
Expressions: copyOfExpressions,
|
||||
name: name,
|
||||
}
|
||||
}
|
||||
@@ -101,10 +130,42 @@ func (index *UniqueIndex) Equals(other Index) bool {
|
||||
if other.Type() != IndexTypeUnique {
|
||||
return false
|
||||
}
|
||||
|
||||
otherUnique, ok := other.(*UniqueIndex)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
// Plain and functional indexes produce different SQL even if their column
|
||||
// sets overlap; require both shapes to match.
|
||||
if (len(index.Expressions) == 0) != (len(otherUnique.Expressions) == 0) {
|
||||
return false
|
||||
}
|
||||
// Expressions are compared normalized so a declared `LOWER(x)` matches the
|
||||
// `lower(x)` a backend renders on read-back.
|
||||
if len(index.Expressions) > 0 && !slices.Equal(normalizeExpressions(index.Expressions), normalizeExpressions(otherUnique.Expressions)) {
|
||||
return false
|
||||
}
|
||||
return index.Name() == other.Name() && slices.Equal(index.Columns(), other.Columns())
|
||||
}
|
||||
|
||||
// normalizeExpressions canonicalizes each Expressions entry (case, whitespace,
|
||||
// redundant parens, identifier quoting) so a declared LOWER(x) matches the
|
||||
// lower(x) a backend renders on read-back. Reuses expressionNormalizer: an index key
|
||||
// is the same SQL grammar as a WHERE predicate. It does not handle opclass,
|
||||
// COLLATE, or ASC/DESC/NULLS suffixes: postgres strips them on read-back
|
||||
// (pg_get_indexdef), while sqlite keeps them verbatim, so an index declared with
|
||||
// one round-trips on sqlite but not on postgres.
|
||||
func normalizeExpressions(expressions []string) []string {
|
||||
if len(expressions) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
normalized := make([]string, len(expressions))
|
||||
for i, expression := range expressions {
|
||||
normalized[i] = (&expressionNormalizer{input: expression, foldCase: true}).normalize()
|
||||
}
|
||||
return normalized
|
||||
}
|
||||
|
||||
func (index *UniqueIndex) ToCreateSQL(fmter SQLFormatter) []byte {
|
||||
sql := []byte{}
|
||||
|
||||
@@ -114,12 +175,20 @@ func (index *UniqueIndex) ToCreateSQL(fmter SQLFormatter) []byte {
|
||||
sql = fmter.AppendIdent(sql, string(index.TableName))
|
||||
sql = append(sql, " ("...)
|
||||
|
||||
for i, column := range index.ColumnNames {
|
||||
if i > 0 {
|
||||
sql = append(sql, ", "...)
|
||||
if len(index.Expressions) > 0 {
|
||||
for i, expr := range index.Expressions {
|
||||
if i > 0 {
|
||||
sql = append(sql, ", "...)
|
||||
}
|
||||
sql = append(sql, expr...)
|
||||
}
|
||||
} else {
|
||||
for i, column := range index.ColumnNames {
|
||||
if i > 0 {
|
||||
sql = append(sql, ", "...)
|
||||
}
|
||||
sql = fmter.AppendIdent(sql, string(column))
|
||||
}
|
||||
|
||||
sql = fmter.AppendIdent(sql, string(column))
|
||||
}
|
||||
|
||||
sql = append(sql, ")"...)
|
||||
@@ -160,7 +229,7 @@ func (index *PartialUniqueIndex) Name() string {
|
||||
b.WriteString(string(column))
|
||||
}
|
||||
b.WriteString("_")
|
||||
b.WriteString((&whereNormalizer{input: index.Where}).hash())
|
||||
b.WriteString((&expressionNormalizer{input: index.Where}).hash())
|
||||
return b.String()
|
||||
}
|
||||
|
||||
@@ -198,7 +267,7 @@ func (index *PartialUniqueIndex) Equals(other Index) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
return index.Name() == other.Name() && slices.Equal(index.Columns(), other.Columns()) && (&whereNormalizer{input: index.Where}).normalize() == (&whereNormalizer{input: otherPartial.Where}).normalize()
|
||||
return index.Name() == other.Name() && slices.Equal(index.Columns(), other.Columns()) && (&expressionNormalizer{input: index.Where}).normalize() == (&expressionNormalizer{input: otherPartial.Where}).normalize()
|
||||
}
|
||||
|
||||
func (index *PartialUniqueIndex) ToCreateSQL(fmter SQLFormatter) []byte {
|
||||
|
||||
@@ -38,6 +38,39 @@ func TestIndexToCreateSQL(t *testing.T) {
|
||||
},
|
||||
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "my_index" ON "users" ("id", "name", "email")`,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_SingleExpression",
|
||||
index: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
},
|
||||
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_users_72852951" ON "users" (LOWER(email))`,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_MixedColumnsAndExpressions",
|
||||
index: &UniqueIndex{
|
||||
TableName: "tag",
|
||||
Expressions: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
|
||||
},
|
||||
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_tag_a36c51df" ON "tag" (org_id, kind, LOWER(key), LOWER(value))`,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_ComplexExpression",
|
||||
index: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(TRIM(first_name) || ' ' || TRIM(last_name))"},
|
||||
},
|
||||
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_users_37e845f3" ON "users" (LOWER(TRIM(first_name) || ' ' || TRIM(last_name)))`,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_Named",
|
||||
index: &UniqueIndex{
|
||||
TableName: "tag",
|
||||
Expressions: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
|
||||
name: "uq_tag_org_kind_lower_key_lower_value",
|
||||
},
|
||||
sql: `CREATE UNIQUE INDEX IF NOT EXISTS "uq_tag_org_kind_lower_key_lower_value" ON "tag" (org_id, kind, LOWER(key), LOWER(value))`,
|
||||
},
|
||||
{
|
||||
name: "PartialUnique_1Column",
|
||||
index: &PartialUniqueIndex{
|
||||
@@ -229,6 +262,54 @@ func TestIndexEquals(t *testing.T) {
|
||||
},
|
||||
equals: false,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_Same",
|
||||
a: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
},
|
||||
b: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
},
|
||||
equals: true,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_CaseInsensitiveEqual",
|
||||
a: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
},
|
||||
b: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"lower(email)"},
|
||||
},
|
||||
equals: true,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_DifferentExpressions",
|
||||
a: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
},
|
||||
b: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"UPPER(email)"},
|
||||
},
|
||||
equals: false,
|
||||
},
|
||||
{
|
||||
name: "Unique_Functional_NotEqualToPlainSameColumns",
|
||||
a: &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
},
|
||||
b: &UniqueIndex{
|
||||
TableName: "users",
|
||||
ColumnNames: []ColumnName{"email"},
|
||||
},
|
||||
equals: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
@@ -238,6 +319,142 @@ func TestIndexEquals(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUniqueIndexFunctionalName(t *testing.T) {
|
||||
t.Run("autogen uses uq_<table>_<hash>", func(t *testing.T) {
|
||||
idx := &UniqueIndex{
|
||||
TableName: "tag",
|
||||
Expressions: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
|
||||
}
|
||||
assert.Equal(t, "uq_tag_a36c51df", idx.Name())
|
||||
})
|
||||
|
||||
t.Run("same expressions produce the same name", func(t *testing.T) {
|
||||
a := &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
}
|
||||
b := &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
}
|
||||
assert.Equal(t, a.Name(), b.Name())
|
||||
})
|
||||
|
||||
t.Run("different expressions produce different names", func(t *testing.T) {
|
||||
a := &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
}
|
||||
b := &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"UPPER(email)"},
|
||||
}
|
||||
assert.NotEqual(t, a.Name(), b.Name())
|
||||
})
|
||||
|
||||
t.Run("expressions in different order produce different names", func(t *testing.T) {
|
||||
a := &UniqueIndex{
|
||||
TableName: "tag",
|
||||
Expressions: []string{"org_id", "LOWER(key)"},
|
||||
}
|
||||
b := &UniqueIndex{
|
||||
TableName: "tag",
|
||||
Expressions: []string{"LOWER(key)", "org_id"},
|
||||
}
|
||||
assert.NotEqual(t, a.Name(), b.Name())
|
||||
})
|
||||
|
||||
t.Run("functional autogen differs from plain autogen for same columns", func(t *testing.T) {
|
||||
plain := &UniqueIndex{
|
||||
TableName: "users",
|
||||
ColumnNames: []ColumnName{"email"},
|
||||
}
|
||||
functional := &UniqueIndex{
|
||||
TableName: "users",
|
||||
Expressions: []string{"LOWER(email)"},
|
||||
}
|
||||
assert.Equal(t, "uq_users_email", plain.Name())
|
||||
assert.NotEqual(t, plain.Name(), functional.Name())
|
||||
})
|
||||
|
||||
t.Run("Named() override wins over hash", func(t *testing.T) {
|
||||
idx := (&UniqueIndex{
|
||||
TableName: "tag",
|
||||
Expressions: []string{"org_id", "LOWER(key)"},
|
||||
}).Named("my_functional_index")
|
||||
assert.Equal(t, "my_functional_index", idx.Name())
|
||||
})
|
||||
|
||||
t.Run("setting both ColumnNames and Expressions panics", func(t *testing.T) {
|
||||
idx := &UniqueIndex{
|
||||
TableName: "tag",
|
||||
ColumnNames: []ColumnName{"org_id"},
|
||||
Expressions: []string{"LOWER(key)"},
|
||||
}
|
||||
assert.Panics(t, func() { _ = idx.Name() })
|
||||
})
|
||||
}
|
||||
|
||||
func TestNormalizeExpressions(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
expressions []string
|
||||
output []string
|
||||
}{
|
||||
{
|
||||
name: "Empty",
|
||||
expressions: nil,
|
||||
output: nil,
|
||||
},
|
||||
{
|
||||
name: "PlainColumnsUnchanged",
|
||||
expressions: []string{"org_id", "kind"},
|
||||
output: []string{"org_id", "kind"},
|
||||
},
|
||||
{
|
||||
name: "FunctionNameCaseFolded",
|
||||
expressions: []string{"LOWER(key)", "LOWER(value)"},
|
||||
output: []string{"lower(key)", "lower(value)"},
|
||||
},
|
||||
{
|
||||
name: "PostgresRenderedMatchesDeclared",
|
||||
expressions: []string{"lower(key)"},
|
||||
output: []string{"lower(key)"},
|
||||
},
|
||||
{
|
||||
name: "WhitespaceCollapsedPerExpression",
|
||||
expressions: []string{"LOWER( key )", "org_id"},
|
||||
output: []string{"lower( key )", "org_id"},
|
||||
},
|
||||
{
|
||||
name: "RedundantOuterParenthesesStripped",
|
||||
expressions: []string{"(LOWER(key))"},
|
||||
output: []string{"lower(key)"},
|
||||
},
|
||||
{
|
||||
name: "QuotedIdentifierCaseByIdentifierPreserved",
|
||||
expressions: []string{`LOWER("Key")`},
|
||||
output: []string{`lower("Key")`},
|
||||
},
|
||||
{
|
||||
name: "StringLiteralCasePreserved",
|
||||
expressions: []string{"COALESCE(status, 'Deleted')"},
|
||||
output: []string{"coalesce(status, 'Deleted')"},
|
||||
},
|
||||
{
|
||||
name: "NormalizedIndependentlyPerElement",
|
||||
expressions: []string{"UPPER(a)", "b", "LOWER(c)"},
|
||||
output: []string{"upper(a)", "b", "lower(c)"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
assert.Equal(t, testCase.output, normalizeExpressions(testCase.expressions))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartialUniqueIndexName(t *testing.T) {
|
||||
a := &PartialUniqueIndex{
|
||||
TableName: "users",
|
||||
|
||||
@@ -6,25 +6,26 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
type whereNormalizer struct {
|
||||
input string
|
||||
type expressionNormalizer struct {
|
||||
input string
|
||||
foldCase bool
|
||||
}
|
||||
|
||||
func (n *whereNormalizer) hash() string {
|
||||
func (n *expressionNormalizer) hash() string {
|
||||
hasher := fnv.New32a()
|
||||
_, _ = hasher.Write([]byte(n.normalize()))
|
||||
return fmt.Sprintf("%08x", hasher.Sum32())
|
||||
}
|
||||
|
||||
func (n *whereNormalizer) normalize() string {
|
||||
where := strings.TrimSpace(n.input)
|
||||
where = n.stripOuterParentheses(where)
|
||||
func (n *expressionNormalizer) normalize() string {
|
||||
expr := strings.TrimSpace(n.input)
|
||||
expr = n.stripOuterParentheses(expr)
|
||||
|
||||
var output strings.Builder
|
||||
output.Grow(len(where))
|
||||
output.Grow(len(expr))
|
||||
|
||||
for i := 0; i < len(where); i++ {
|
||||
switch where[i] {
|
||||
for i := 0; i < len(expr); i++ {
|
||||
switch expr[i] {
|
||||
case ' ', '\t', '\n', '\r':
|
||||
if output.Len() > 0 {
|
||||
last := output.String()[output.Len()-1]
|
||||
@@ -33,21 +34,25 @@ func (n *whereNormalizer) normalize() string {
|
||||
}
|
||||
}
|
||||
case '\'':
|
||||
end := n.consumeSingleQuotedLiteral(where, i, &output)
|
||||
end := n.consumeSingleQuotedLiteral(expr, i, &output)
|
||||
i = end
|
||||
case '"':
|
||||
token, end := n.consumeDoubleQuotedToken(where, i)
|
||||
token, end := n.consumeDoubleQuotedToken(expr, i)
|
||||
output.WriteString(token)
|
||||
i = end
|
||||
default:
|
||||
output.WriteByte(where[i])
|
||||
c := expr[i]
|
||||
if n.foldCase && c >= 'A' && c <= 'Z' {
|
||||
c += 'a' - 'A'
|
||||
}
|
||||
output.WriteByte(c)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(output.String())
|
||||
}
|
||||
|
||||
func (n *whereNormalizer) stripOuterParentheses(s string) string {
|
||||
func (n *expressionNormalizer) stripOuterParentheses(s string) string {
|
||||
for {
|
||||
s = strings.TrimSpace(s)
|
||||
if len(s) < 2 || s[0] != '(' || s[len(s)-1] != ')' || !n.hasWrappingParentheses(s) {
|
||||
@@ -57,7 +62,7 @@ func (n *whereNormalizer) stripOuterParentheses(s string) string {
|
||||
}
|
||||
}
|
||||
|
||||
func (n *whereNormalizer) hasWrappingParentheses(s string) bool {
|
||||
func (n *expressionNormalizer) hasWrappingParentheses(s string) bool {
|
||||
depth := 0
|
||||
inSingleQuotedLiteral := false
|
||||
inDoubleQuotedToken := false
|
||||
@@ -101,7 +106,7 @@ func (n *whereNormalizer) hasWrappingParentheses(s string) bool {
|
||||
return depth == 0
|
||||
}
|
||||
|
||||
func (n *whereNormalizer) consumeSingleQuotedLiteral(s string, start int, output *strings.Builder) int {
|
||||
func (n *expressionNormalizer) consumeSingleQuotedLiteral(s string, start int, output *strings.Builder) int {
|
||||
output.WriteByte(s[start])
|
||||
for i := start + 1; i < len(s); i++ {
|
||||
output.WriteByte(s[i])
|
||||
@@ -118,7 +123,7 @@ func (n *whereNormalizer) consumeSingleQuotedLiteral(s string, start int, output
|
||||
return len(s) - 1
|
||||
}
|
||||
|
||||
func (n *whereNormalizer) consumeDoubleQuotedToken(s string, start int) (string, int) {
|
||||
func (n *expressionNormalizer) consumeDoubleQuotedToken(s string, start int) (string, int) {
|
||||
var ident strings.Builder
|
||||
|
||||
for i := start + 1; i < len(s); i++ {
|
||||
@@ -142,7 +147,7 @@ func (n *whereNormalizer) consumeDoubleQuotedToken(s string, start int) (string,
|
||||
return s[start:], len(s) - 1
|
||||
}
|
||||
|
||||
func (n *whereNormalizer) isSimpleUnquotedIdentifier(s string) bool {
|
||||
func (n *expressionNormalizer) isSimpleUnquotedIdentifier(s string) bool {
|
||||
if s == "" || strings.ToLower(s) != s {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestWhereNormalizerNormalize(t *testing.T) {
|
||||
func TestExpressionNormalizerNormalize(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
input string
|
||||
@@ -47,11 +47,16 @@ func TestWhereNormalizerNormalize(t *testing.T) {
|
||||
input: `NOT ("deleted_at" IS NOT NULL AND ("active" = false OR "status" = 'archived'))`,
|
||||
output: `NOT (deleted_at IS NOT NULL AND (active = false OR status = 'archived'))`,
|
||||
},
|
||||
{
|
||||
name: "CaseNotFoldedWithoutFlag",
|
||||
input: `LOWER("email") IS NOT NULL`,
|
||||
output: `LOWER(email) IS NOT NULL`,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
assert.Equal(t, testCase.output, (&whereNormalizer{input: testCase.input}).normalize())
|
||||
assert.Equal(t, testCase.output, (&expressionNormalizer{input: testCase.input}).normalize())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package sqlitesqlschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
@@ -92,7 +93,6 @@ func (provider *provider) GetIndices(ctx context.Context, tableName sqlschema.Ta
|
||||
unique bool
|
||||
origin string
|
||||
partial bool
|
||||
columns []sqlschema.ColumnName
|
||||
)
|
||||
if err := rows.Scan(&seq, &name, &unique, &origin, &partial); err != nil {
|
||||
return nil, err
|
||||
@@ -108,53 +108,103 @@ func (provider *provider) GetIndices(ctx context.Context, tableName sqlschema.Ta
|
||||
continue
|
||||
}
|
||||
|
||||
if err := provider.
|
||||
sqlstore.
|
||||
BunDB().
|
||||
NewRaw("SELECT name FROM PRAGMA_index_info(?)", string(name)).
|
||||
Scan(ctx, &columns); err != nil {
|
||||
if !unique {
|
||||
continue
|
||||
}
|
||||
|
||||
columns, hasExpression, err := provider.indexKeys(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if unique && partial {
|
||||
var indexSQL string
|
||||
// SQLite exposes expression keys and the predicate only in the DDL.
|
||||
var ddl string
|
||||
if hasExpression || partial {
|
||||
if err := provider.
|
||||
sqlstore.
|
||||
BunDB().
|
||||
NewRaw("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?", name).
|
||||
Scan(ctx, &indexSQL); err != nil {
|
||||
Scan(ctx, &ddl); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
where := extractWhereClause(indexSQL)
|
||||
index := &sqlschema.PartialUniqueIndex{
|
||||
// functional partial indexes aren't representable (PartialUniqueIndex has no
|
||||
// expressions); skip rather than misrepresent.
|
||||
if hasExpression && partial {
|
||||
provider.settings.Logger().WarnContext(ctx, "skipping functional partial unique index; not representable by sqlschema", slog.String("index", name), slog.String("table", string(tableName)))
|
||||
continue
|
||||
}
|
||||
|
||||
var index sqlschema.Index
|
||||
if partial {
|
||||
index = &sqlschema.PartialUniqueIndex{
|
||||
TableName: tableName,
|
||||
ColumnNames: columns,
|
||||
Where: where,
|
||||
Where: extractWhereClause(ddl),
|
||||
}
|
||||
|
||||
if index.Name() == name {
|
||||
indices = append(indices, index)
|
||||
} else {
|
||||
indices = append(indices, index.Named(name))
|
||||
} else if hasExpression {
|
||||
index = &sqlschema.UniqueIndex{
|
||||
TableName: tableName,
|
||||
Expressions: extractIndexColumns(ddl),
|
||||
}
|
||||
} else if unique {
|
||||
index := &sqlschema.UniqueIndex{
|
||||
} else {
|
||||
index = &sqlschema.UniqueIndex{
|
||||
TableName: tableName,
|
||||
ColumnNames: columns,
|
||||
}
|
||||
|
||||
if index.Name() == name {
|
||||
indices = append(indices, index)
|
||||
} else {
|
||||
indices = append(indices, index.Named(name))
|
||||
}
|
||||
}
|
||||
|
||||
if index.Name() != name {
|
||||
index = index.Named(name)
|
||||
}
|
||||
indices = append(indices, index)
|
||||
}
|
||||
|
||||
return indices, nil
|
||||
}
|
||||
|
||||
// indexKeys returns an index's plain key columns and whether any key is an
|
||||
// expression (SQLite marks expression keys with cid = -2 and a NULL name).
|
||||
func (provider *provider) indexKeys(ctx context.Context, name string) ([]sqlschema.ColumnName, bool, error) {
|
||||
rows, err := provider.
|
||||
sqlstore.
|
||||
BunDB().
|
||||
QueryContext(ctx, "SELECT cid, name FROM PRAGMA_index_info(?)", name)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer func() {
|
||||
if err := rows.Close(); err != nil {
|
||||
provider.settings.Logger().ErrorContext(ctx, "error closing rows", errors.Attr(err))
|
||||
}
|
||||
}()
|
||||
|
||||
var (
|
||||
columns []sqlschema.ColumnName
|
||||
hasExpression bool
|
||||
)
|
||||
for rows.Next() {
|
||||
var (
|
||||
cid int
|
||||
columnName *string
|
||||
)
|
||||
if err := rows.Scan(&cid, &columnName); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
if cid == -2 {
|
||||
hasExpression = true
|
||||
continue
|
||||
}
|
||||
if columnName != nil {
|
||||
columns = append(columns, sqlschema.ColumnName(*columnName))
|
||||
}
|
||||
}
|
||||
|
||||
return columns, hasExpression, nil
|
||||
}
|
||||
|
||||
func (provider *provider) ToggleFKEnforcement(ctx context.Context, db bun.IDB, on bool) error {
|
||||
_, err := db.ExecContext(ctx, "PRAGMA foreign_keys = ?", on)
|
||||
if err != nil {
|
||||
@@ -173,6 +223,92 @@ func (provider *provider) ToggleFKEnforcement(ctx context.Context, db bun.IDB, o
|
||||
return errors.NewInternalf(errors.CodeInternal, "foreign_keys(actual: %s, expected: %s), maybe a transaction is in progress?", strconv.FormatBool(val), strconv.FormatBool(on))
|
||||
}
|
||||
|
||||
// extractIndexColumns returns an index's key entries: the comma-separated items
|
||||
// in the first top-level parenthesised list after ON, each verbatim (trimmed).
|
||||
// Quotes and nested parens are respected so their commas don't split keys.
|
||||
// SQLite reports expression keys with a NULL name in PRAGMA_index_info, so their
|
||||
// text is only recoverable from the DDL.
|
||||
func extractIndexColumns(sql string) []string {
|
||||
var (
|
||||
inSingleQuotedLiteral bool
|
||||
inDoubleQuotedIdentifier bool
|
||||
inBacktickQuotedIdentifier bool
|
||||
inBracketQuotedIdentifier bool
|
||||
depth int
|
||||
started bool
|
||||
keyStart int
|
||||
columns []string
|
||||
)
|
||||
|
||||
for i := 0; i < len(sql); i++ {
|
||||
switch sql[i] {
|
||||
case '\'':
|
||||
if inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
|
||||
break
|
||||
}
|
||||
if inSingleQuotedLiteral && i+1 < len(sql) && sql[i+1] == '\'' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
inSingleQuotedLiteral = !inSingleQuotedLiteral
|
||||
continue
|
||||
case '"':
|
||||
if inSingleQuotedLiteral || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
|
||||
break
|
||||
}
|
||||
if inDoubleQuotedIdentifier && i+1 < len(sql) && sql[i+1] == '"' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
inDoubleQuotedIdentifier = !inDoubleQuotedIdentifier
|
||||
continue
|
||||
case '`':
|
||||
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBracketQuotedIdentifier {
|
||||
break
|
||||
}
|
||||
inBacktickQuotedIdentifier = !inBacktickQuotedIdentifier
|
||||
continue
|
||||
case '[':
|
||||
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
|
||||
break
|
||||
}
|
||||
inBracketQuotedIdentifier = true
|
||||
continue
|
||||
case ']':
|
||||
if inBracketQuotedIdentifier {
|
||||
inBracketQuotedIdentifier = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if inSingleQuotedLiteral || inDoubleQuotedIdentifier || inBacktickQuotedIdentifier || inBracketQuotedIdentifier {
|
||||
continue
|
||||
}
|
||||
|
||||
switch sql[i] {
|
||||
case '(':
|
||||
depth++
|
||||
if depth == 1 && !started {
|
||||
started = true
|
||||
keyStart = i + 1
|
||||
}
|
||||
case ')':
|
||||
depth--
|
||||
if depth == 0 && started {
|
||||
columns = append(columns, strings.TrimSpace(sql[keyStart:i]))
|
||||
return columns
|
||||
}
|
||||
case ',':
|
||||
if depth == 1 {
|
||||
columns = append(columns, strings.TrimSpace(sql[keyStart:i]))
|
||||
keyStart = i + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return columns
|
||||
}
|
||||
|
||||
func extractWhereClause(sql string) string {
|
||||
lastWhere := -1
|
||||
inSingleQuotedLiteral := false
|
||||
|
||||
@@ -50,3 +50,93 @@ func TestExtractWhereClause(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractIndexColumns(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
sql string
|
||||
columns []string
|
||||
}{
|
||||
{
|
||||
name: "SingleColumn",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "users" ("email")`,
|
||||
columns: []string{`"email"`},
|
||||
},
|
||||
{
|
||||
name: "SingleExpression",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "users" (LOWER(email))`,
|
||||
columns: []string{"LOWER(email)"},
|
||||
},
|
||||
{
|
||||
name: "MixedColumnsAndExpressions",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "tag" (org_id, kind, LOWER(key), LOWER(value))`,
|
||||
columns: []string{"org_id", "kind", "LOWER(key)", "LOWER(value)"},
|
||||
},
|
||||
{
|
||||
name: "CommaInsideStringLiteralNotSplit",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "users" (LOWER(TRIM(first_name) || ', ' || last_name))`,
|
||||
columns: []string{"LOWER(TRIM(first_name) || ', ' || last_name)"},
|
||||
},
|
||||
{
|
||||
name: "CommaInsideQuotedIdentifierNotSplit",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "t" ("weird,col", LOWER(x))`,
|
||||
columns: []string{`"weird,col"`, "LOWER(x)"},
|
||||
},
|
||||
{
|
||||
name: "WhereClauseAfterKeyListIgnored",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "tag" (LOWER(key)) WHERE deleted = 0`,
|
||||
columns: []string{"LOWER(key)"},
|
||||
},
|
||||
{
|
||||
name: "WhereClauseWithParensAfterKeyListIgnored",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "tag" (LOWER(key)) WHERE (deleted = 0 AND kind = 'a')`,
|
||||
columns: []string{"LOWER(key)"},
|
||||
},
|
||||
{
|
||||
name: "CommaInsideBacktickIdentifierNotSplit",
|
||||
sql: "CREATE UNIQUE INDEX `idx` ON `t` (`weird,col`, x)",
|
||||
columns: []string{"`weird,col`", "x"},
|
||||
},
|
||||
{
|
||||
name: "CommaInsideBracketIdentifierNotSplit",
|
||||
sql: `CREATE UNIQUE INDEX [idx] ON [t] ([weird,col], x)`,
|
||||
columns: []string{"[weird,col]", "x"},
|
||||
},
|
||||
{
|
||||
name: "EscapedDoubleQuoteInIdentifier",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "t" ("a""b", x)`,
|
||||
columns: []string{`"a""b"`, "x"},
|
||||
},
|
||||
{
|
||||
name: "EscapedSingleQuoteInLiteral",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "t" (COALESCE(x, 'it''s, ok'))`,
|
||||
columns: []string{"COALESCE(x, 'it''s, ok')"},
|
||||
},
|
||||
{
|
||||
name: "NestedFunctionCallCommasNotSplit",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "t" (COALESCE(a, b, c), d)`,
|
||||
columns: []string{"COALESCE(a, b, c)", "d"},
|
||||
},
|
||||
{
|
||||
name: "NewlinesAndTabsTrimmed",
|
||||
sql: "CREATE UNIQUE INDEX \"idx\" ON \"t\" (\n\tLOWER(key),\n\torg_id\n)",
|
||||
columns: []string{"LOWER(key)", "org_id"},
|
||||
},
|
||||
{
|
||||
name: "CollateAndOrderSuffixKeptVerbatim",
|
||||
sql: `CREATE UNIQUE INDEX "idx" ON "t" (LOWER(key) COLLATE NOCASE, value DESC)`,
|
||||
columns: []string{"LOWER(key) COLLATE NOCASE", "value DESC"},
|
||||
},
|
||||
{
|
||||
name: "NoKeyListReturnsNil",
|
||||
sql: `CREATE INDEX "idx" ON "t"`,
|
||||
columns: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
assert.Equal(t, testCase.columns, extractIndexColumns(testCase.sql))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
103
pkg/sqlschema/sqlitesqlschema/roundtrip_test.go
Normal file
103
pkg/sqlschema/sqlitesqlschema/roundtrip_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
//go:build integration
|
||||
|
||||
// Stands up a real sqlite store (temp file); excluded from the default test run.
|
||||
// Run with: go test -tags integration ./...
|
||||
|
||||
package sqlitesqlschema
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/sqlschema"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore/sqlitesqlstore"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestGetIndicesRoundTrip creates each index shape on a fresh sqlite database and
|
||||
// checks GetIndices reconstructs an equal index, exercising every branch: plain
|
||||
// vs expression keys parsed from the DDL, multi-key ordering, partial predicate,
|
||||
// and the skipped functional-partial case.
|
||||
func TestGetIndicesRoundTrip(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
cfg := sqlstore.Config{
|
||||
Provider: "sqlite",
|
||||
Sqlite: sqlstore.SqliteConfig{
|
||||
Path: filepath.Join(t.TempDir(), "roundtrip.db"),
|
||||
Mode: "wal",
|
||||
BusyTimeout: 10 * time.Second,
|
||||
TransactionMode: "deferred",
|
||||
},
|
||||
Connection: sqlstore.ConnectionConfig{MaxOpenConns: 10},
|
||||
}
|
||||
|
||||
providerSettings := instrumentationtest.New().ToProviderSettings()
|
||||
store, err := sqlitesqlstore.New(ctx, providerSettings, cfg)
|
||||
require.NoError(t, err)
|
||||
|
||||
schema, err := New(ctx, providerSettings, sqlschema.Config{}, store)
|
||||
require.NoError(t, err)
|
||||
|
||||
const table = "sqlschema_roundtrip"
|
||||
|
||||
exec := func(sql string) {
|
||||
_, err := store.BunDB().ExecContext(ctx, sql)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
reset := func() {
|
||||
exec(`DROP TABLE IF EXISTS ` + table)
|
||||
exec(`CREATE TABLE ` + table + ` (a text, b text, c text)`)
|
||||
}
|
||||
|
||||
roundTrip := func(t *testing.T, declared sqlschema.Index) {
|
||||
reset()
|
||||
for _, sql := range schema.Operator().CreateIndex(declared) {
|
||||
exec(string(sql))
|
||||
}
|
||||
|
||||
indices, err := schema.GetIndices(ctx, table)
|
||||
require.NoError(t, err)
|
||||
|
||||
var got sqlschema.Index
|
||||
for _, index := range indices {
|
||||
if index.Name() == declared.Name() {
|
||||
got = index
|
||||
}
|
||||
}
|
||||
require.NotNil(t, got, "GetIndices did not return %q; returned %d indices", declared.Name(), len(indices))
|
||||
require.True(t, declared.Equals(got), "round-tripped index should equal the declared one; got %#v", got)
|
||||
}
|
||||
|
||||
t.Run("PlainSingleColumn", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, ColumnNames: []sqlschema.ColumnName{"a"}})
|
||||
})
|
||||
|
||||
t.Run("PlainMultiColumn", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, ColumnNames: []sqlschema.ColumnName{"a", "b"}})
|
||||
})
|
||||
|
||||
t.Run("FunctionalSingleExpression", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, Expressions: []string{"LOWER(a)"}})
|
||||
})
|
||||
|
||||
t.Run("FunctionalMixedKeys", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.UniqueIndex{TableName: table, Expressions: []string{"a", "LOWER(b)", "LOWER(c)"}})
|
||||
})
|
||||
|
||||
t.Run("PartialUnique", func(t *testing.T) {
|
||||
roundTrip(t, &sqlschema.PartialUniqueIndex{TableName: table, ColumnNames: []sqlschema.ColumnName{"a"}, Where: "b IS NOT NULL"})
|
||||
})
|
||||
|
||||
t.Run("FunctionalPartialIsSkipped", func(t *testing.T) {
|
||||
reset()
|
||||
exec(`CREATE UNIQUE INDEX rt_functional_partial ON ` + table + ` (LOWER(a)) WHERE b IS NOT NULL`)
|
||||
|
||||
indices, err := schema.GetIndices(ctx, table)
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, indices, "functional partial unique index must be skipped, not reconstructed")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user