mirror of
https://github.com/SigNoz/signoz.git
synced 2026-05-06 10:30:31 +01:00
Compare commits
55 Commits
postproces
...
feat/billi
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b847a6605d | ||
|
|
acdaef6c2e | ||
|
|
a7690bdaa2 | ||
|
|
a7fde606ca | ||
|
|
28e19b85f5 | ||
|
|
28d37122a4 | ||
|
|
860d14f5fa | ||
|
|
0aaf556137 | ||
|
|
582ba1c677 | ||
|
|
3d8cddf84e | ||
|
|
01678e422b | ||
|
|
8a7028a75d | ||
|
|
9fcdb613cb | ||
|
|
3616022049 | ||
|
|
5d270b716b | ||
|
|
19d9d26051 | ||
|
|
522148362b | ||
|
|
781158ecab | ||
|
|
0603bd6b27 | ||
|
|
37c57e6c05 | ||
|
|
12a2e63a31 | ||
|
|
453bcc06c4 | ||
|
|
fac5fe6b9e | ||
|
|
0ad412b844 | ||
|
|
d1957b5eac | ||
|
|
dba9cfd455 | ||
|
|
ed2011a7bb | ||
|
|
68385478c7 | ||
|
|
eb661b7ac7 | ||
|
|
afd6868423 | ||
|
|
8ddf0a13c1 | ||
|
|
16f0d2aa38 | ||
|
|
3af912c586 | ||
|
|
ad7715802b | ||
|
|
b579bdbd7b | ||
|
|
aa64cf7bbf | ||
|
|
2d33b1a743 | ||
|
|
4fbf7de8e1 | ||
|
|
7528b19fd4 | ||
|
|
42e4196aad | ||
|
|
22cdb03702 | ||
|
|
6eca3dc06e | ||
|
|
0631189417 | ||
|
|
ec552b94cc | ||
|
|
ee8d99f1d0 | ||
|
|
bf77e26a86 | ||
|
|
9cd3cf23d7 | ||
|
|
4a44802ebc | ||
|
|
f2aed0d834 | ||
|
|
527d8c0459 | ||
|
|
8fdc91260e | ||
|
|
218c4524b1 | ||
|
|
02dec846eb | ||
|
|
99dadb7247 | ||
|
|
44b41c40de |
1
.github/CODEOWNERS
vendored
1
.github/CODEOWNERS
vendored
@@ -107,6 +107,7 @@ go.mod @therealpandey
|
||||
/pkg/modules/organization/ @therealpandey
|
||||
/pkg/modules/authdomain/ @therealpandey
|
||||
/pkg/modules/role/ @therealpandey
|
||||
/pkg/types/coretypes/ @therealpandey @vikrantgupta25
|
||||
|
||||
# IdentN Owners
|
||||
|
||||
|
||||
17
.github/workflows/goci.yaml
vendored
17
.github/workflows/goci.yaml
vendored
@@ -102,3 +102,20 @@ jobs:
|
||||
run: |
|
||||
go run cmd/enterprise/*.go generate openapi
|
||||
git diff --compact-summary --exit-code || (echo; echo "Unexpected difference in openapi spec. Run go run cmd/enterprise/*.go generate openapi locally and commit."; exit 1)
|
||||
authz:
|
||||
if: |
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
|
||||
(github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: self-checkout
|
||||
uses: actions/checkout@v4
|
||||
- name: go-install
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.24"
|
||||
- name: generate-authz
|
||||
run: |
|
||||
go run cmd/enterprise/*.go generate authz
|
||||
git diff --compact-summary --exit-code || (echo; echo "Unexpected difference in authz permissions. Run go run cmd/enterprise/*.go generate authz locally and commit."; exit 1)
|
||||
|
||||
40
.github/workflows/jsci.yaml
vendored
40
.github/workflows/jsci.yaml
vendored
@@ -63,46 +63,6 @@ jobs:
|
||||
uses: actions/checkout@v4
|
||||
- name: run
|
||||
run: bash frontend/scripts/validate-md-languages.sh
|
||||
authz:
|
||||
if: |
|
||||
github.event_name == 'merge_group' ||
|
||||
(github.event_name == 'pull_request' && ! github.event.pull_request.head.repo.fork && github.event.pull_request.user.login != 'dependabot[bot]' && ! contains(github.event.pull_request.labels.*.name, 'safe-to-test')) ||
|
||||
(github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe-to-test'))
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: self-checkout
|
||||
uses: actions/checkout@v5
|
||||
- name: node-install
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: "22"
|
||||
- name: deps-install
|
||||
working-directory: ./frontend
|
||||
run: |
|
||||
yarn install
|
||||
- name: uv-install
|
||||
uses: astral-sh/setup-uv@v5
|
||||
- name: uv-deps
|
||||
working-directory: ./tests/integration
|
||||
run: |
|
||||
uv sync
|
||||
- name: setup-test
|
||||
run: |
|
||||
make py-test-setup
|
||||
- name: generate
|
||||
working-directory: ./frontend
|
||||
run: |
|
||||
yarn generate:permissions-type
|
||||
- name: teardown-test
|
||||
if: always()
|
||||
run: |
|
||||
make py-test-teardown
|
||||
- name: validate
|
||||
run: |
|
||||
if ! git diff --exit-code frontend/src/hooks/useAuthZ/permissions.type.ts; then
|
||||
echo "::error::frontend/src/hooks/useAuthZ/permissions.type.ts is out of date. Please run the generator locally and commit the changes: npm run generate:permissions-type (from the frontend directory)"
|
||||
exit 1
|
||||
fi
|
||||
openapi:
|
||||
if: |
|
||||
github.event_name == 'merge_group' ||
|
||||
|
||||
117
cmd/authz.go
Normal file
117
cmd/authz.go
Normal file
@@ -0,0 +1,117 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"sort"
|
||||
"text/template"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
const permissionsTypePath = "frontend/src/hooks/useAuthZ/permissions.type.ts"
|
||||
|
||||
var permissionsTypeTemplate = template.Must(template.New("permissions").Parse(
|
||||
`// AUTO GENERATED FILE - DO NOT EDIT - GENERATED BY cmd/enterprise/*.go generate authz
|
||||
export default {
|
||||
status: 'success',
|
||||
data: {
|
||||
resources: [
|
||||
{{- range .Resources }}
|
||||
{
|
||||
kind: '{{ .Kind }}',
|
||||
type: '{{ .Type }}',
|
||||
},
|
||||
{{- end }}
|
||||
],
|
||||
relations: {
|
||||
{{- range .Relations }}
|
||||
{{ .Verb }}: [{{ range $i, $t := .Types }}{{ if $i }}, {{ end }}'{{ $t }}'{{ end }}],
|
||||
{{- end }}
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
`))
|
||||
|
||||
type permissionsTypeRelation struct {
|
||||
Verb string
|
||||
Types []string
|
||||
}
|
||||
|
||||
type permissionsTypeResource struct {
|
||||
Kind string
|
||||
Type string
|
||||
}
|
||||
|
||||
type permissionsTypeData struct {
|
||||
Resources []permissionsTypeResource
|
||||
Relations []permissionsTypeRelation
|
||||
}
|
||||
|
||||
func registerGenerateAuthz(parentCmd *cobra.Command) {
|
||||
authzCmd := &cobra.Command{
|
||||
Use: "authz",
|
||||
Short: "Generate authz permissions for the frontend",
|
||||
RunE: func(currCmd *cobra.Command, args []string) error {
|
||||
return runGenerateAuthz(currCmd.Context())
|
||||
},
|
||||
}
|
||||
|
||||
parentCmd.AddCommand(authzCmd)
|
||||
}
|
||||
|
||||
func runGenerateAuthz(_ context.Context) error {
|
||||
registry := coretypes.NewRegistry()
|
||||
|
||||
allowedResources := map[string]bool{
|
||||
coretypes.NewResourceRef(coretypes.ResourceServiceAccount).String(): true,
|
||||
coretypes.NewResourceRef(coretypes.ResourceRole).String(): true,
|
||||
coretypes.NewResourceRef(coretypes.ResourceMetaResourcesRole).String(): true,
|
||||
}
|
||||
|
||||
allowedTypes := map[string]bool{}
|
||||
|
||||
refs := registry.ResourceRefs()
|
||||
resources := make([]permissionsTypeResource, 0, len(refs))
|
||||
for _, ref := range refs {
|
||||
if !allowedResources[ref.String()] {
|
||||
continue
|
||||
}
|
||||
allowedTypes[ref.Type.StringValue()] = true
|
||||
resources = append(resources, permissionsTypeResource{
|
||||
Kind: ref.Kind.String(),
|
||||
Type: ref.Type.StringValue(),
|
||||
})
|
||||
}
|
||||
|
||||
typesByVerb := registry.TypesByVerb()
|
||||
verbs := make([]coretypes.Verb, 0, len(typesByVerb))
|
||||
for verb := range typesByVerb {
|
||||
verbs = append(verbs, verb)
|
||||
}
|
||||
sort.Slice(verbs, func(i, j int) bool { return verbs[i].StringValue() < verbs[j].StringValue() })
|
||||
|
||||
relations := make([]permissionsTypeRelation, 0, len(verbs))
|
||||
for _, verb := range verbs {
|
||||
types := make([]string, 0, len(typesByVerb[verb]))
|
||||
for _, t := range typesByVerb[verb] {
|
||||
if !allowedTypes[t.StringValue()] {
|
||||
continue
|
||||
}
|
||||
types = append(types, t.StringValue())
|
||||
}
|
||||
relations = append(relations, permissionsTypeRelation{
|
||||
Verb: verb.StringValue(),
|
||||
Types: types,
|
||||
})
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := permissionsTypeTemplate.Execute(&buf, permissionsTypeData{Resources: resources, Relations: relations}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return os.WriteFile(permissionsTypePath, buf.Bytes(), 0o600)
|
||||
}
|
||||
@@ -18,16 +18,19 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/gateway"
|
||||
"github.com/SigNoz/signoz/pkg/gateway/noopgateway"
|
||||
"github.com/SigNoz/signoz/pkg/global"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/licensing/nooplicensing"
|
||||
"github.com/SigNoz/signoz/pkg/meterreporter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rulestatehistory"
|
||||
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
@@ -92,13 +95,13 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
func(ctx context.Context, providerSettings factory.ProviderSettings, store authtypes.AuthNStore, licensing licensing.Licensing) (map[authtypes.AuthNProvider]authn.AuthN, error) {
|
||||
return signoz.NewAuthNs(ctx, providerSettings, store, licensing)
|
||||
},
|
||||
func(ctx context.Context, sqlstore sqlstore.SQLStore, _ licensing.Licensing, _ []authz.OnBeforeRoleDelete, _ dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
|
||||
openfgaDataStore, err := openfgaserver.NewSQLStore(sqlstore)
|
||||
func(ctx context.Context, sqlstore sqlstore.SQLStore, config authz.Config, _ licensing.Licensing, _ []authz.OnBeforeRoleDelete) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
|
||||
openfgaDataStore, err := openfgaserver.NewSQLStore(sqlstore, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore), nil
|
||||
return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, authtypes.NewRegistry()), nil
|
||||
},
|
||||
func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, _ querier.Querier, _ licensing.Licensing) dashboard.Module {
|
||||
return impldashboard.NewModule(impldashboard.NewStore(store), settings, analytics, orgGetter, queryParser)
|
||||
@@ -109,6 +112,9 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
func(_ licensing.Licensing) factory.NamedMap[factory.ProviderFactory[auditor.Auditor, auditor.Config]] {
|
||||
return signoz.NewAuditorProviderFactories()
|
||||
},
|
||||
func(_ context.Context, _ flagger.Flagger, _ licensing.Licensing, _ telemetrystore.TelemetryStore, _ retention.Getter, _ organization.Getter, _ zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string) {
|
||||
return signoz.NewMeterReporterProviderFactories(), "noop"
|
||||
},
|
||||
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
|
||||
return querier.NewHandler(ps, q, a)
|
||||
},
|
||||
|
||||
@@ -17,6 +17,14 @@ import (
|
||||
"github.com/SigNoz/signoz/ee/gateway/httpgateway"
|
||||
enterpriselicensing "github.com/SigNoz/signoz/ee/licensing"
|
||||
"github.com/SigNoz/signoz/ee/licensing/httplicensing"
|
||||
"github.com/SigNoz/signoz/ee/metercollector/baseplatformfeemetercollector"
|
||||
"github.com/SigNoz/signoz/ee/metercollector/datapointcountmetercollector"
|
||||
"github.com/SigNoz/signoz/ee/metercollector/datapointsizemetercollector"
|
||||
"github.com/SigNoz/signoz/ee/metercollector/logcountmetercollector"
|
||||
"github.com/SigNoz/signoz/ee/metercollector/logsizemetercollector"
|
||||
"github.com/SigNoz/signoz/ee/metercollector/spancountmetercollector"
|
||||
"github.com/SigNoz/signoz/ee/metercollector/spansizemetercollector"
|
||||
"github.com/SigNoz/signoz/ee/meterreporter/httpmeterreporter"
|
||||
"github.com/SigNoz/signoz/ee/modules/cloudintegration/implcloudintegration"
|
||||
"github.com/SigNoz/signoz/ee/modules/cloudintegration/implcloudintegration/implcloudprovider"
|
||||
"github.com/SigNoz/signoz/ee/modules/dashboard/impldashboard"
|
||||
@@ -35,14 +43,18 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/cache"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
pkgflagger "github.com/SigNoz/signoz/pkg/flagger"
|
||||
"github.com/SigNoz/signoz/pkg/gateway"
|
||||
"github.com/SigNoz/signoz/pkg/global"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/meterreporter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/cloudintegration"
|
||||
pkgcloudintegration "github.com/SigNoz/signoz/pkg/modules/cloudintegration/implcloudintegration"
|
||||
"github.com/SigNoz/signoz/pkg/modules/dashboard"
|
||||
pkgimpldashboard "github.com/SigNoz/signoz/pkg/modules/dashboard/impldashboard"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/modules/rulestatehistory"
|
||||
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
@@ -57,7 +69,10 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/cloudintegrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/featuretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/SigNoz/signoz/pkg/version"
|
||||
"github.com/SigNoz/signoz/pkg/zeus"
|
||||
)
|
||||
@@ -137,12 +152,12 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
|
||||
return authNs, nil
|
||||
},
|
||||
func(ctx context.Context, sqlstore sqlstore.SQLStore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, dashboardModule dashboard.Module) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
|
||||
openfgaDataStore, err := openfgaserver.NewSQLStore(sqlstore)
|
||||
func(ctx context.Context, sqlstore sqlstore.SQLStore, config authz.Config, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete) (factory.ProviderFactory[authz.AuthZ, authz.Config], error) {
|
||||
openfgaDataStore, err := openfgaserver.NewSQLStore(sqlstore, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, licensing, onBeforeRoleDelete, dashboardModule), nil
|
||||
return openfgaauthz.NewProviderFactory(sqlstore, openfgaschema.NewSchema().Get(ctx), openfgaDataStore, licensing, onBeforeRoleDelete, authtypes.NewRegistry()), nil
|
||||
},
|
||||
func(store sqlstore.SQLStore, settings factory.ProviderSettings, analytics analytics.Analytics, orgGetter organization.Getter, queryParser queryparser.QueryParser, querier querier.Querier, licensing licensing.Licensing) dashboard.Module {
|
||||
return impldashboard.NewModule(pkgimpldashboard.NewStore(store), settings, analytics, orgGetter, queryParser, querier, licensing)
|
||||
@@ -157,6 +172,19 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
}
|
||||
return factories
|
||||
},
|
||||
func(ctx context.Context, flagger pkgflagger.Flagger, licensing licensing.Licensing, telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter, orgGetter organization.Getter, zeus zeus.Zeus) (factory.NamedMap[factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config]], string) {
|
||||
factories := signoz.NewMeterReporterProviderFactories()
|
||||
if err := factories.Add(httpmeterreporter.NewFactory(newMeterCollectors(licensing, telemetryStore, retentionGetter), licensing, telemetryStore, orgGetter, zeus)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
evalCtx := featuretypes.NewFlaggerEvaluationContext(valuer.UUID{})
|
||||
if flagger.BooleanOrEmpty(ctx, pkgflagger.FeatureUseMeterReporter, evalCtx) {
|
||||
return factories, "http"
|
||||
}
|
||||
|
||||
return factories, "noop"
|
||||
},
|
||||
func(ps factory.ProviderSettings, q querier.Querier, a analytics.Analytics) querier.Handler {
|
||||
communityHandler := querier.NewHandler(ps, q, a)
|
||||
return eequerier.NewHandler(ps, q, communityHandler)
|
||||
@@ -216,3 +244,15 @@ func runServer(ctx context.Context, config signoz.Config, logger *slog.Logger) e
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func newMeterCollectors(licensing licensing.Licensing, telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter) map[zeustypes.MeterName]metercollector.MeterCollector {
|
||||
return map[zeustypes.MeterName]metercollector.MeterCollector{
|
||||
baseplatformfeemetercollector.MeterName: baseplatformfeemetercollector.New(licensing),
|
||||
logcountmetercollector.MeterName: logcountmetercollector.New(telemetryStore, retentionGetter),
|
||||
logsizemetercollector.MeterName: logsizemetercollector.New(telemetryStore, retentionGetter),
|
||||
datapointcountmetercollector.MeterName: datapointcountmetercollector.New(telemetryStore, retentionGetter),
|
||||
datapointsizemetercollector.MeterName: datapointsizemetercollector.New(telemetryStore, retentionGetter),
|
||||
spancountmetercollector.MeterName: spancountmetercollector.New(telemetryStore, retentionGetter),
|
||||
spansizemetercollector.MeterName: spansizemetercollector.New(telemetryStore, retentionGetter),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ func RegisterGenerate(parentCmd *cobra.Command, logger *slog.Logger) {
|
||||
}
|
||||
|
||||
registerGenerateOpenAPI(generateCmd)
|
||||
registerGenerateAuthz(generateCmd)
|
||||
|
||||
parentCmd.AddCommand(generateCmd)
|
||||
}
|
||||
|
||||
@@ -428,4 +428,11 @@ authz:
|
||||
provider: openfga
|
||||
openfga:
|
||||
# maximum tuples allowed per openfga write operation.
|
||||
max_tuples_per_write: 100
|
||||
max_tuples_per_write: 300
|
||||
|
||||
##################### Meter Reporter #####################
|
||||
meterreporter:
|
||||
# The interval between collection ticks. Minimum 5m.
|
||||
interval: 6h
|
||||
# The per-tick timeout that bounds collect-and-ship work. Minimum 3m and must be less than interval.
|
||||
timeout: 5m
|
||||
|
||||
@@ -321,35 +321,6 @@ components:
|
||||
required:
|
||||
- id
|
||||
type: object
|
||||
AuthtypesGettableObjects:
|
||||
properties:
|
||||
resource:
|
||||
$ref: '#/components/schemas/AuthtypesResource'
|
||||
selectors:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
required:
|
||||
- resource
|
||||
- selectors
|
||||
type: object
|
||||
AuthtypesGettableResources:
|
||||
properties:
|
||||
relations:
|
||||
additionalProperties:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
nullable: true
|
||||
type: object
|
||||
resources:
|
||||
items:
|
||||
$ref: '#/components/schemas/AuthtypesResource'
|
||||
type: array
|
||||
required:
|
||||
- resources
|
||||
- relations
|
||||
type: object
|
||||
AuthtypesGettableToken:
|
||||
properties:
|
||||
accessToken:
|
||||
@@ -366,9 +337,9 @@ components:
|
||||
authorized:
|
||||
type: boolean
|
||||
object:
|
||||
$ref: '#/components/schemas/AuthtypesObject'
|
||||
$ref: '#/components/schemas/CoretypesObject'
|
||||
relation:
|
||||
type: string
|
||||
$ref: '#/components/schemas/AuthtypesRelation'
|
||||
required:
|
||||
- relation
|
||||
- object
|
||||
@@ -416,16 +387,6 @@ components:
|
||||
issuerAlias:
|
||||
type: string
|
||||
type: object
|
||||
AuthtypesObject:
|
||||
properties:
|
||||
resource:
|
||||
$ref: '#/components/schemas/AuthtypesResource'
|
||||
selector:
|
||||
type: string
|
||||
required:
|
||||
- resource
|
||||
- selector
|
||||
type: object
|
||||
AuthtypesOrgSessionContext:
|
||||
properties:
|
||||
authNSupport:
|
||||
@@ -442,22 +403,6 @@ components:
|
||||
provider:
|
||||
$ref: '#/components/schemas/AuthtypesAuthNProvider'
|
||||
type: object
|
||||
AuthtypesPatchableObjects:
|
||||
properties:
|
||||
additions:
|
||||
items:
|
||||
$ref: '#/components/schemas/AuthtypesGettableObjects'
|
||||
nullable: true
|
||||
type: array
|
||||
deletions:
|
||||
items:
|
||||
$ref: '#/components/schemas/AuthtypesGettableObjects'
|
||||
nullable: true
|
||||
type: array
|
||||
required:
|
||||
- additions
|
||||
- deletions
|
||||
type: object
|
||||
AuthtypesPatchableRole:
|
||||
properties:
|
||||
description:
|
||||
@@ -495,16 +440,15 @@ components:
|
||||
refreshToken:
|
||||
type: string
|
||||
type: object
|
||||
AuthtypesResource:
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
type:
|
||||
type: string
|
||||
required:
|
||||
- name
|
||||
- type
|
||||
type: object
|
||||
AuthtypesRelation:
|
||||
enum:
|
||||
- create
|
||||
- read
|
||||
- update
|
||||
- delete
|
||||
- list
|
||||
- assignee
|
||||
type: string
|
||||
AuthtypesRole:
|
||||
properties:
|
||||
createdAt:
|
||||
@@ -568,9 +512,9 @@ components:
|
||||
AuthtypesTransaction:
|
||||
properties:
|
||||
object:
|
||||
$ref: '#/components/schemas/AuthtypesObject'
|
||||
$ref: '#/components/schemas/CoretypesObject'
|
||||
relation:
|
||||
type: string
|
||||
$ref: '#/components/schemas/AuthtypesRelation'
|
||||
required:
|
||||
- relation
|
||||
- object
|
||||
@@ -2205,6 +2149,64 @@ components:
|
||||
to_user:
|
||||
type: string
|
||||
type: object
|
||||
CoretypesObject:
|
||||
properties:
|
||||
resource:
|
||||
$ref: '#/components/schemas/CoretypesResourceRef'
|
||||
selector:
|
||||
type: string
|
||||
required:
|
||||
- resource
|
||||
- selector
|
||||
type: object
|
||||
CoretypesObjectGroup:
|
||||
properties:
|
||||
resource:
|
||||
$ref: '#/components/schemas/CoretypesResourceRef'
|
||||
selectors:
|
||||
items:
|
||||
type: string
|
||||
type: array
|
||||
required:
|
||||
- resource
|
||||
- selectors
|
||||
type: object
|
||||
CoretypesPatchableObjects:
|
||||
properties:
|
||||
additions:
|
||||
items:
|
||||
$ref: '#/components/schemas/CoretypesObjectGroup'
|
||||
nullable: true
|
||||
type: array
|
||||
deletions:
|
||||
items:
|
||||
$ref: '#/components/schemas/CoretypesObjectGroup'
|
||||
nullable: true
|
||||
type: array
|
||||
required:
|
||||
- additions
|
||||
- deletions
|
||||
type: object
|
||||
CoretypesResourceRef:
|
||||
properties:
|
||||
kind:
|
||||
type: string
|
||||
type:
|
||||
$ref: '#/components/schemas/CoretypesType'
|
||||
required:
|
||||
- type
|
||||
- kind
|
||||
type: object
|
||||
CoretypesType:
|
||||
enum:
|
||||
- user
|
||||
- serviceaccount
|
||||
- anonymous
|
||||
- role
|
||||
- organization
|
||||
- metaresource
|
||||
- metaresources
|
||||
type: string
|
||||
DashboardtypesDashboard:
|
||||
properties:
|
||||
createdAt:
|
||||
@@ -5704,35 +5706,6 @@ paths:
|
||||
summary: Check permissions
|
||||
tags:
|
||||
- authz
|
||||
/api/v1/authz/resources:
|
||||
get:
|
||||
deprecated: false
|
||||
description: Gets all the available resources
|
||||
operationId: AuthzResources
|
||||
responses:
|
||||
"200":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
properties:
|
||||
data:
|
||||
$ref: '#/components/schemas/AuthtypesGettableResources'
|
||||
status:
|
||||
type: string
|
||||
required:
|
||||
- status
|
||||
- data
|
||||
type: object
|
||||
description: OK
|
||||
"500":
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/RenderErrorResponse'
|
||||
description: Internal Server Error
|
||||
summary: Get resources
|
||||
tags:
|
||||
- authz
|
||||
/api/v1/channels:
|
||||
get:
|
||||
deprecated: false
|
||||
@@ -8956,7 +8929,7 @@ paths:
|
||||
properties:
|
||||
data:
|
||||
items:
|
||||
$ref: '#/components/schemas/AuthtypesGettableObjects'
|
||||
$ref: '#/components/schemas/CoretypesObjectGroup'
|
||||
type: array
|
||||
status:
|
||||
type: string
|
||||
@@ -9029,7 +9002,7 @@ paths:
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '#/components/schemas/AuthtypesPatchableObjects'
|
||||
$ref: '#/components/schemas/CoretypesPatchableObjects'
|
||||
responses:
|
||||
"204":
|
||||
content:
|
||||
|
||||
@@ -2,7 +2,6 @@ package openfgaauthz
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
|
||||
"github.com/SigNoz/signoz/ee/authz/openfgaserver"
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
@@ -13,6 +12,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
openfgav1 "github.com/openfga/api/proto/openfga/v1"
|
||||
openfgapkgtransformer "github.com/openfga/language/pkg/go/transformer"
|
||||
@@ -25,19 +25,19 @@ type provider struct {
|
||||
openfgaServer *openfgaserver.Server
|
||||
licensing licensing.Licensing
|
||||
store authtypes.RoleStore
|
||||
registry []authz.RegisterTypeable
|
||||
registry *authtypes.Registry
|
||||
settings factory.ScopedProviderSettings
|
||||
onBeforeRoleDelete []authz.OnBeforeRoleDelete
|
||||
}
|
||||
|
||||
func NewProviderFactory(sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, registry ...authz.RegisterTypeable) factory.ProviderFactory[authz.AuthZ, authz.Config] {
|
||||
func NewProviderFactory(sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, registry *authtypes.Registry) factory.ProviderFactory[authz.AuthZ, authz.Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("openfga"), func(ctx context.Context, ps factory.ProviderSettings, config authz.Config) (authz.AuthZ, error) {
|
||||
return newOpenfgaProvider(ctx, ps, config, sqlstore, openfgaSchema, openfgaDataStore, licensing, onBeforeRoleDelete, registry)
|
||||
})
|
||||
}
|
||||
|
||||
func newOpenfgaProvider(ctx context.Context, settings factory.ProviderSettings, config authz.Config, sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, registry []authz.RegisterTypeable) (authz.AuthZ, error) {
|
||||
pkgOpenfgaAuthzProvider := pkgopenfgaauthz.NewProviderFactory(sqlstore, openfgaSchema, openfgaDataStore)
|
||||
func newOpenfgaProvider(ctx context.Context, settings factory.ProviderSettings, config authz.Config, sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, licensing licensing.Licensing, onBeforeRoleDelete []authz.OnBeforeRoleDelete, registry *authtypes.Registry) (authz.AuthZ, error) {
|
||||
pkgOpenfgaAuthzProvider := pkgopenfgaauthz.NewProviderFactory(sqlstore, openfgaSchema, openfgaDataStore, registry)
|
||||
pkgAuthzService, err := pkgOpenfgaAuthzProvider.New(ctx, settings, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -74,11 +74,11 @@ func (provider *provider) Stop(ctx context.Context) error {
|
||||
return provider.openfgaServer.Stop(ctx)
|
||||
}
|
||||
|
||||
func (provider *provider) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, relation authtypes.Relation, typeable authtypes.Typeable, selectors []authtypes.Selector, roleSelectors []authtypes.Selector) error {
|
||||
func (provider *provider) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, relation authtypes.Relation, typeable coretypes.Resource, selectors []coretypes.Selector, roleSelectors []coretypes.Selector) error {
|
||||
return provider.openfgaServer.CheckWithTupleCreation(ctx, claims, orgID, relation, typeable, selectors, roleSelectors)
|
||||
}
|
||||
|
||||
func (provider *provider) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, relation authtypes.Relation, typeable authtypes.Typeable, selectors []authtypes.Selector, roleSelectors []authtypes.Selector) error {
|
||||
func (provider *provider) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, relation authtypes.Relation, typeable coretypes.Resource, selectors []coretypes.Selector, roleSelectors []coretypes.Selector) error {
|
||||
return provider.openfgaServer.CheckWithTupleCreationWithoutClaims(ctx, orgID, relation, typeable, selectors, roleSelectors)
|
||||
}
|
||||
|
||||
@@ -108,7 +108,7 @@ func (provider *provider) CheckTransactions(ctx context.Context, subject string,
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
|
||||
func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType coretypes.Type) ([]*coretypes.Object, error) {
|
||||
return provider.openfgaServer.ListObjects(ctx, subject, relation, objectType)
|
||||
}
|
||||
|
||||
@@ -159,16 +159,10 @@ func (provider *provider) CreateManagedRoles(ctx context.Context, orgID valuer.U
|
||||
func (provider *provider) CreateManagedUserRoleTransactions(ctx context.Context, orgID valuer.UUID, userID valuer.UUID) error {
|
||||
tuples := make([]*openfgav1.TupleKey, 0)
|
||||
|
||||
grantTuples, err := provider.getManagedRoleGrantTuples(orgID, userID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
grantTuples := provider.getManagedRoleGrantTuples(orgID, userID)
|
||||
tuples = append(tuples, grantTuples...)
|
||||
|
||||
managedRoleTuples, err := provider.getManagedRoleTransactionTuples(orgID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
managedRoleTuples := provider.getManagedRoleTransactionTuples(orgID)
|
||||
tuples = append(tuples, managedRoleTuples...)
|
||||
|
||||
return provider.Write(ctx, tuples, nil)
|
||||
@@ -208,21 +202,7 @@ func (provider *provider) GetOrCreate(ctx context.Context, orgID valuer.UUID, ro
|
||||
return role, nil
|
||||
}
|
||||
|
||||
func (provider *provider) GetResources(_ context.Context) []*authtypes.Resource {
|
||||
resources := make([]*authtypes.Resource, 0)
|
||||
for _, register := range provider.registry {
|
||||
for _, typeable := range register.MustGetTypeables() {
|
||||
resources = append(resources, &authtypes.Resource{Name: typeable.Name(), Type: typeable.Type()})
|
||||
}
|
||||
}
|
||||
for _, typeable := range provider.MustGetTypeables() {
|
||||
resources = append(resources, &authtypes.Resource{Name: typeable.Name(), Type: typeable.Type()})
|
||||
}
|
||||
|
||||
return resources
|
||||
}
|
||||
|
||||
func (provider *provider) GetObjects(ctx context.Context, orgID valuer.UUID, id valuer.UUID, relation authtypes.Relation) ([]*authtypes.Object, error) {
|
||||
func (provider *provider) GetObjects(ctx context.Context, orgID valuer.UUID, id valuer.UUID, relation authtypes.Relation) ([]*coretypes.Object, error) {
|
||||
_, err := provider.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
@@ -233,16 +213,16 @@ func (provider *provider) GetObjects(ctx context.Context, orgID valuer.UUID, id
|
||||
return nil, err
|
||||
}
|
||||
|
||||
objects := make([]*authtypes.Object, 0)
|
||||
for _, objectType := range provider.getUniqueTypes() {
|
||||
if !slices.Contains(authtypes.TypeableRelations[objectType], relation) {
|
||||
objects := make([]*coretypes.Object, 0)
|
||||
for _, objectType := range provider.registry.Types() {
|
||||
if coretypes.ErrIfVerbNotValidForType(relation.Verb, objectType) != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
resourceObjects, err := provider.
|
||||
ListObjects(
|
||||
ctx,
|
||||
authtypes.MustNewSubject(authtypes.TypeableRole, storableRole.Name, orgID, &authtypes.RelationAssignee),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceRole(), storableRole.Name, orgID, &coretypes.VerbAssignee),
|
||||
relation,
|
||||
objectType,
|
||||
)
|
||||
@@ -265,7 +245,7 @@ func (provider *provider) Patch(ctx context.Context, orgID valuer.UUID, role *au
|
||||
return provider.store.Update(ctx, orgID, role)
|
||||
}
|
||||
|
||||
func (provider *provider) PatchObjects(ctx context.Context, orgID valuer.UUID, name string, relation authtypes.Relation, additions, deletions []*authtypes.Object) error {
|
||||
func (provider *provider) PatchObjects(ctx context.Context, orgID valuer.UUID, name string, relation authtypes.Relation, additions, deletions []*coretypes.Object) error {
|
||||
_, err := provider.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return errors.New(errors.TypeLicenseUnavailable, errors.CodeLicenseUnavailable, "a valid license is not available").WithAdditional("this feature requires a valid license").WithAdditional(err.Error())
|
||||
@@ -318,84 +298,63 @@ func (provider *provider) Delete(ctx context.Context, orgID valuer.UUID, id valu
|
||||
return provider.store.Delete(ctx, orgID, id)
|
||||
}
|
||||
|
||||
func (provider *provider) MustGetTypeables() []authtypes.Typeable {
|
||||
return []authtypes.Typeable{authtypes.TypeableRole, authtypes.TypeableResourcesRoles}
|
||||
}
|
||||
|
||||
func (provider *provider) getManagedRoleGrantTuples(orgID valuer.UUID, userID valuer.UUID) ([]*openfgav1.TupleKey, error) {
|
||||
func (provider *provider) getManagedRoleGrantTuples(orgID valuer.UUID, userID valuer.UUID) []*openfgav1.TupleKey {
|
||||
tuples := []*openfgav1.TupleKey{}
|
||||
|
||||
// Grant the admin role to the user
|
||||
adminSubject := authtypes.MustNewSubject(authtypes.TypeableUser, userID.String(), orgID, nil)
|
||||
adminTuple, err := authtypes.TypeableRole.Tuples(
|
||||
adminSubject := authtypes.MustNewSubject(coretypes.NewResourceUser(), userID.String(), orgID, nil)
|
||||
adminTuple := authtypes.NewTuples(
|
||||
coretypes.NewResourceRole(),
|
||||
adminSubject,
|
||||
authtypes.RelationAssignee,
|
||||
[]authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAdminRoleName),
|
||||
},
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
[]coretypes.Selector{coretypes.TypeRole.MustSelector(authtypes.SigNozAdminRoleName)},
|
||||
orgID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tuples = append(tuples, adminTuple...)
|
||||
|
||||
// Grant the admin role to the anonymous user
|
||||
anonymousSubject := authtypes.MustNewSubject(authtypes.TypeableAnonymous, authtypes.AnonymousUser.String(), orgID, nil)
|
||||
anonymousTuple, err := authtypes.TypeableRole.Tuples(
|
||||
anonymousSubject := authtypes.MustNewSubject(coretypes.NewResourceAnonymous(), coretypes.AnonymousUser.String(), orgID, nil)
|
||||
anonymousTuple := authtypes.NewTuples(
|
||||
coretypes.NewResourceRole(),
|
||||
anonymousSubject,
|
||||
authtypes.RelationAssignee,
|
||||
[]authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAnonymousRoleName),
|
||||
},
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
[]coretypes.Selector{coretypes.TypeRole.MustSelector(authtypes.SigNozAnonymousRoleName)},
|
||||
orgID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tuples = append(tuples, anonymousTuple...)
|
||||
|
||||
return tuples, nil
|
||||
return tuples
|
||||
}
|
||||
|
||||
func (provider *provider) getManagedRoleTransactionTuples(orgID valuer.UUID) ([]*openfgav1.TupleKey, error) {
|
||||
transactionsByRole := make(map[string][]*authtypes.Transaction)
|
||||
for _, register := range provider.registry {
|
||||
for roleName, txns := range register.MustGetManagedRoleTransactions() {
|
||||
transactionsByRole[roleName] = append(transactionsByRole[roleName], txns...)
|
||||
}
|
||||
}
|
||||
|
||||
func (provider *provider) getManagedRoleTransactionTuples(orgID valuer.UUID) []*openfgav1.TupleKey {
|
||||
tuples := make([]*openfgav1.TupleKey, 0)
|
||||
for roleName, transactions := range transactionsByRole {
|
||||
for roleName, transactions := range provider.registry.ManagedRoleTransactions() {
|
||||
for _, txn := range transactions {
|
||||
typeable := authtypes.MustNewTypeableFromType(txn.Object.Resource.Type, txn.Object.Resource.Name)
|
||||
txnTuples, err := typeable.Tuples(
|
||||
resource := coretypes.MustNewResourceFromTypeAndKind(txn.Object.Resource.Type, txn.Object.Resource.Kind)
|
||||
txnTuples := authtypes.NewTuples(
|
||||
resource,
|
||||
authtypes.MustNewSubject(
|
||||
authtypes.TypeableRole,
|
||||
coretypes.NewResourceRole(),
|
||||
roleName,
|
||||
orgID,
|
||||
&authtypes.RelationAssignee,
|
||||
&coretypes.VerbAssignee,
|
||||
),
|
||||
txn.Relation,
|
||||
[]authtypes.Selector{txn.Object.Selector},
|
||||
[]coretypes.Selector{txn.Object.Selector},
|
||||
orgID,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tuples = append(tuples, txnTuples...)
|
||||
}
|
||||
}
|
||||
|
||||
return tuples, nil
|
||||
return tuples
|
||||
}
|
||||
|
||||
func (provider *provider) deleteTuples(ctx context.Context, roleName string, orgID valuer.UUID) error {
|
||||
subject := authtypes.MustNewSubject(authtypes.TypeableRole, roleName, orgID, &authtypes.RelationAssignee)
|
||||
subject := authtypes.MustNewSubject(coretypes.NewResourceRole(), roleName, orgID, &coretypes.VerbAssignee)
|
||||
|
||||
tuples := make([]*openfgav1.TupleKey, 0)
|
||||
for _, objectType := range provider.getUniqueTypes() {
|
||||
for _, objectType := range provider.registry.Types() {
|
||||
typeTuples, err := provider.ReadTuples(ctx, &openfgav1.ReadRequestTupleKey{
|
||||
User: subject,
|
||||
Object: objectType.StringValue() + ":",
|
||||
@@ -424,28 +383,3 @@ func (provider *provider) deleteTuples(ctx context.Context, roleName string, org
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (provider *provider) getUniqueTypes() []authtypes.Type {
|
||||
seen := make(map[string]struct{})
|
||||
uniqueTypes := make([]authtypes.Type, 0)
|
||||
for _, register := range provider.registry {
|
||||
for _, typeable := range register.MustGetTypeables() {
|
||||
typeKey := typeable.Type().StringValue()
|
||||
if _, ok := seen[typeKey]; ok {
|
||||
continue
|
||||
}
|
||||
seen[typeKey] = struct{}{}
|
||||
uniqueTypes = append(uniqueTypes, typeable.Type())
|
||||
}
|
||||
}
|
||||
for _, typeable := range provider.MustGetTypeables() {
|
||||
typeKey := typeable.Type().StringValue()
|
||||
if _, ok := seen[typeKey]; ok {
|
||||
continue
|
||||
}
|
||||
seen[typeKey] = struct{}{}
|
||||
uniqueTypes = append(uniqueTypes, typeable.Type())
|
||||
}
|
||||
|
||||
return uniqueTypes
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module base
|
||||
|
||||
type organisation
|
||||
type organization
|
||||
relations
|
||||
define read: [user, serviceaccount, role#assignee]
|
||||
define update: [user, serviceaccount, role#assignee]
|
||||
@@ -10,12 +10,14 @@ type user
|
||||
define read: [user, serviceaccount, role#assignee]
|
||||
define update: [user, serviceaccount, role#assignee]
|
||||
define delete: [user, serviceaccount, role#assignee]
|
||||
define attach: [user, serviceaccount, role#assignee]
|
||||
|
||||
type serviceaccount
|
||||
type serviceaccount
|
||||
relations
|
||||
define read: [user, serviceaccount, role#assignee]
|
||||
define update: [user, serviceaccount, role#assignee]
|
||||
define delete: [user, serviceaccount, role#assignee]
|
||||
define delete: [user, serviceaccount, role#assignee]
|
||||
define attach: [user, serviceaccount, role#assignee]
|
||||
|
||||
type anonymous
|
||||
|
||||
@@ -26,6 +28,7 @@ type role
|
||||
define read: [user, serviceaccount, role#assignee]
|
||||
define update: [user, serviceaccount, role#assignee]
|
||||
define delete: [user, serviceaccount, role#assignee]
|
||||
define attach: [user, serviceaccount, role#assignee]
|
||||
|
||||
type metaresources
|
||||
relations
|
||||
@@ -43,4 +46,4 @@ type metaresource
|
||||
|
||||
type telemetryresource
|
||||
relations
|
||||
define read: [user, serviceaccount, role#assignee]
|
||||
define read: [user, serviceaccount, role#assignee]
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
openfgav1 "github.com/openfga/api/proto/openfga/v1"
|
||||
)
|
||||
@@ -33,18 +34,18 @@ func (server *Server) Stop(ctx context.Context) error {
|
||||
return server.pkgAuthzService.Stop(ctx)
|
||||
}
|
||||
|
||||
func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, relation authtypes.Relation, typeable authtypes.Typeable, selectors []authtypes.Selector, _ []authtypes.Selector) error {
|
||||
func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, relation authtypes.Relation, typeable coretypes.Resource, selectors []coretypes.Selector, _ []coretypes.Selector) error {
|
||||
subject := ""
|
||||
switch claims.Principal {
|
||||
case authtypes.PrincipalUser:
|
||||
user, err := authtypes.NewSubject(authtypes.TypeableUser, claims.UserID, orgID, nil)
|
||||
user, err := authtypes.NewSubject(coretypes.NewResourceUser(), claims.UserID, orgID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
subject = user
|
||||
case authtypes.PrincipalServiceAccount:
|
||||
serviceAccount, err := authtypes.NewSubject(authtypes.TypeableServiceAccount, claims.ServiceAccountID, orgID, nil)
|
||||
serviceAccount, err := authtypes.NewSubject(coretypes.NewResourceServiceAccount(), claims.ServiceAccountID, orgID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -52,10 +53,7 @@ func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtyp
|
||||
subject = serviceAccount
|
||||
}
|
||||
|
||||
tupleSlice, err := typeable.Tuples(subject, relation, selectors, orgID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tupleSlice := authtypes.NewTuples(typeable, subject, relation, selectors, orgID)
|
||||
|
||||
tuples := make(map[string]*openfgav1.TupleKey, len(tupleSlice))
|
||||
for idx, tuple := range tupleSlice {
|
||||
@@ -76,16 +74,13 @@ func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtyp
|
||||
return errors.Newf(errors.TypeForbidden, authtypes.ErrCodeAuthZForbidden, "subjects are not authorized for requested access")
|
||||
}
|
||||
|
||||
func (server *Server) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, relation authtypes.Relation, typeable authtypes.Typeable, selectors []authtypes.Selector, _ []authtypes.Selector) error {
|
||||
subject, err := authtypes.NewSubject(authtypes.TypeableAnonymous, authtypes.AnonymousUser.String(), orgID, nil)
|
||||
func (server *Server) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, relation authtypes.Relation, typeable coretypes.Resource, selectors []coretypes.Selector, _ []coretypes.Selector) error {
|
||||
subject, err := authtypes.NewSubject(coretypes.NewResourceAnonymous(), coretypes.AnonymousUser.String(), orgID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tupleSlice, err := typeable.Tuples(subject, relation, selectors, orgID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tupleSlice := authtypes.NewTuples(typeable, subject, relation, selectors, orgID)
|
||||
|
||||
tuples := make(map[string]*openfgav1.TupleKey, len(tupleSlice))
|
||||
for idx, tuple := range tupleSlice {
|
||||
@@ -110,7 +105,7 @@ func (server *Server) BatchCheck(ctx context.Context, tupleReq map[string]*openf
|
||||
return server.pkgAuthzService.BatchCheck(ctx, tupleReq)
|
||||
}
|
||||
|
||||
func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
|
||||
func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType coretypes.Type) ([]*coretypes.Object, error) {
|
||||
return server.pkgAuthzService.ListObjects(ctx, subject, relation, objectType)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package openfgaserver
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/ee/sqlstore/postgressqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/openfga/openfga/pkg/storage"
|
||||
@@ -10,11 +11,11 @@ import (
|
||||
"github.com/openfga/openfga/pkg/storage/sqlite"
|
||||
)
|
||||
|
||||
func NewSQLStore(store sqlstore.SQLStore) (storage.OpenFGADatastore, error) {
|
||||
func NewSQLStore(store sqlstore.SQLStore, config authz.Config) (storage.OpenFGADatastore, error) {
|
||||
switch store.BunDB().Dialect().Name().String() {
|
||||
case "sqlite":
|
||||
return sqlite.NewWithDB(store.SQLDB(), &sqlcommon.Config{
|
||||
MaxTuplesPerWriteField: 100,
|
||||
MaxTuplesPerWriteField: config.OpenFGA.MaxTuplesPerWrite,
|
||||
MaxTypesPerModelField: 100,
|
||||
})
|
||||
case "pg":
|
||||
@@ -24,7 +25,7 @@ func NewSQLStore(store sqlstore.SQLStore) (storage.OpenFGADatastore, error) {
|
||||
}
|
||||
|
||||
return postgres.NewWithDB(pgStore.Pool(), nil, &sqlcommon.Config{
|
||||
MaxTuplesPerWriteField: 100,
|
||||
MaxTuplesPerWriteField: config.OpenFGA.MaxTuplesPerWrite,
|
||||
MaxTypesPerModelField: 100,
|
||||
})
|
||||
}
|
||||
|
||||
53
ee/metercollector/baseplatformfeemetercollector/provider.go
Normal file
53
ee/metercollector/baseplatformfeemetercollector/provider.go
Normal file
@@ -0,0 +1,53 @@
|
||||
// Package baseplatformfeemetercollector collects the license-derived base platform fee meter.
|
||||
package baseplatformfeemetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterName is the typed registry key for this collector.
|
||||
var (
|
||||
MeterName = zeustypes.MustNewMeterName("signoz.meter.base.platform.fee")
|
||||
meterUnit = zeustypes.MeterUnitCount
|
||||
meterAggregation = zeustypes.MeterAggregationMax
|
||||
)
|
||||
|
||||
var _ metercollector.MeterCollector = (*Provider)(nil)
|
||||
|
||||
// Provider collects base platform fee meters.
|
||||
type Provider struct {
|
||||
licensing licensing.Licensing
|
||||
}
|
||||
|
||||
func New(licensing licensing.Licensing) *Provider {
|
||||
return &Provider{licensing: licensing}
|
||||
}
|
||||
|
||||
func (p *Provider) Name() zeustypes.MeterName { return MeterName }
|
||||
func (p *Provider) Unit() zeustypes.MeterUnit { return meterUnit }
|
||||
func (p *Provider) Aggregation() zeustypes.MeterAggregation {
|
||||
return meterAggregation
|
||||
}
|
||||
|
||||
// Collect emits value 1 when the org has an active license.
|
||||
func (p *Provider) Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
license, err := p.licensing.GetActive(ctx, orgID)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "fetch active license for base platform fee meter")
|
||||
}
|
||||
if license == nil || license.Key == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return []zeustypes.Meter{
|
||||
zeustypes.NewMeter(MeterName, 1, meterUnit, meterAggregation, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
}),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package baseplatformfeemetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/types/licensetypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestCollectEmitsBasePlatformFeeMeterForValidLicense(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
window := completedWindow()
|
||||
provider := New(&fakeLicensing{
|
||||
license: &licensetypes.License{Key: "license-key"},
|
||||
})
|
||||
|
||||
readings, err := provider.Collect(context.Background(), orgID, window)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []zeustypes.Meter{
|
||||
zeustypes.NewMeter(MeterName, 1, zeustypes.MeterUnitCount, zeustypes.MeterAggregationMax, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
}),
|
||||
}, readings)
|
||||
}
|
||||
|
||||
func TestCollectSkipsNilLicense(t *testing.T) {
|
||||
readings, err := New(&fakeLicensing{}).Collect(context.Background(), valuer.GenerateUUID(), completedWindow())
|
||||
require.NoError(t, err)
|
||||
require.Empty(t, readings)
|
||||
}
|
||||
|
||||
func TestProviderMetadata(t *testing.T) {
|
||||
provider := New(&fakeLicensing{})
|
||||
|
||||
require.Equal(t, "signoz.meter.base.platform.fee", provider.Name().String())
|
||||
require.Equal(t, zeustypes.MeterUnitCount, provider.Unit())
|
||||
require.Equal(t, zeustypes.MeterAggregationMax, provider.Aggregation())
|
||||
}
|
||||
|
||||
func completedWindow() zeustypes.MeterWindow {
|
||||
start := time.Date(2026, 5, 4, 0, 0, 0, 0, time.UTC)
|
||||
return zeustypes.MustNewMeterWindow(start.UnixMilli(), start.AddDate(0, 0, 1).UnixMilli(), true)
|
||||
}
|
||||
|
||||
var _ licensing.Licensing = (*fakeLicensing)(nil)
|
||||
|
||||
type fakeLicensing struct {
|
||||
license *licensetypes.License
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Start(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Stop(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Validate(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Activate(context.Context, valuer.UUID, string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) GetActive(context.Context, valuer.UUID) (*licensetypes.License, error) {
|
||||
return f.license, f.err
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Refresh(context.Context, valuer.UUID) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Checkout(context.Context, valuer.UUID, *licensetypes.PostableSubscription) (*licensetypes.GettableSubscription, error) {
|
||||
return &licensetypes.GettableSubscription{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Portal(context.Context, valuer.UUID, *licensetypes.PostableSubscription) (*licensetypes.GettableSubscription, error) {
|
||||
return &licensetypes.GettableSubscription{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) GetFeatureFlags(context.Context, valuer.UUID) ([]*licensetypes.Feature, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeLicensing) Collect(context.Context, valuer.UUID) (map[string]any, error) {
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
271
ee/metercollector/datapointcountmetercollector/provider.go
Normal file
271
ee/metercollector/datapointcountmetercollector/provider.go
Normal file
@@ -0,0 +1,271 @@
|
||||
// Package datapointcountmetercollector collects metric datapoint count meters
|
||||
// by workspace and retention. Keep the query local to this meter.
|
||||
package datapointcountmetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterName is the typed registry key for this collector.
|
||||
var (
|
||||
MeterName = zeustypes.MustNewMeterName("signoz.meter.metric.datapoint.count")
|
||||
meterUnit = zeustypes.MeterUnitCount
|
||||
meterAggregation = zeustypes.MeterAggregationSum
|
||||
)
|
||||
|
||||
var _ metercollector.MeterCollector = (*Provider)(nil)
|
||||
|
||||
// Provider collects datapoint count meters.
|
||||
type Provider struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
retentionGetter retention.Getter
|
||||
}
|
||||
|
||||
func New(telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter) *Provider {
|
||||
return &Provider{
|
||||
telemetryStore: telemetryStore,
|
||||
retentionGetter: retentionGetter,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) Name() zeustypes.MeterName { return MeterName }
|
||||
func (p *Provider) Unit() zeustypes.MeterUnit { return meterUnit }
|
||||
func (p *Provider) Aggregation() zeustypes.MeterAggregation {
|
||||
return meterAggregation
|
||||
}
|
||||
|
||||
// Collect aggregates datapoint count for the window and emits an empty-day sentinel.
|
||||
func (p *Provider) Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
meterName := MeterName.String()
|
||||
|
||||
slices, err := p.retentionGetter.ActiveSlices(
|
||||
ctx,
|
||||
orgID,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.SamplesV4LocalTableName,
|
||||
retentiontypes.DefaultMetricsRetentionDays,
|
||||
window.StartUnixMilli,
|
||||
window.EndUnixMilli,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "load retention slices for meter %q", meterName)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
dimensions map[string]string
|
||||
value float64
|
||||
}
|
||||
accumulator := make(map[string]*bucket)
|
||||
|
||||
for _, slice := range slices {
|
||||
query, args, dimensionColumns, err := buildQuery(meterName, slice, p.retentionGetter)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build retention query for meter %q", meterName)
|
||||
}
|
||||
|
||||
rows, err := p.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "query meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
if err := func() error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
dimensionValues := make([]string, len(dimensionColumns))
|
||||
var retentionDays int32
|
||||
var retentionRuleIndex int32
|
||||
var value float64
|
||||
|
||||
scanDest := make([]any, 0, len(dimensionValues)+3)
|
||||
for i := range dimensionValues {
|
||||
scanDest = append(scanDest, &dimensionValues[i])
|
||||
}
|
||||
scanDest = append(scanDest, &retentionDays, &retentionRuleIndex, &value)
|
||||
|
||||
if err := rows.Scan(scanDest...); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "scan meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, int(retentionDays), int(retentionRuleIndex), dimensionColumns, dimensionValues, slice.Rules)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build dimensions for meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
key := bucketKey(dimensions)
|
||||
b, ok := accumulator[key]
|
||||
if !ok {
|
||||
b = &bucket{dimensions: dimensions}
|
||||
accumulator[key] = b
|
||||
}
|
||||
b.value += value
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "iterate meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
meters := make([]zeustypes.Meter, 0, len(accumulator))
|
||||
for _, b := range accumulator {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, b.value, meterUnit, meterAggregation, window, b.dimensions))
|
||||
}
|
||||
|
||||
// Empty windows still emit a sentinel so checkpoints can advance.
|
||||
if len(meters) == 0 && len(slices) > 0 {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, 0, meterUnit, meterAggregation, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(slices[len(slices)-1].DefaultDays),
|
||||
}))
|
||||
}
|
||||
|
||||
return meters, nil
|
||||
}
|
||||
|
||||
// buildQuery stays local because each meter owns its billing query.
|
||||
func buildQuery(meterName string, slice retentiontypes.Slice, retentionGetter retention.Getter) (string, []any, []dimensionColumn, error) {
|
||||
retentionExpr, err := retentionGetter.BuildMultiIfSQL(slice.Rules, slice.DefaultDays)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
retentionRuleIndexExpr, err := retentionGetter.BuildRuleIndexSQL(slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
columns, err := dimensionColumnsFor(retentionGetter, slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
selects := make([]string, 0, len(columns)+3)
|
||||
groupBy := make([]string, 0, len(columns)+2)
|
||||
for _, column := range columns {
|
||||
selects = append(selects, fmt.Sprintf("JSONExtractString(labels, '%s') AS %s", column.key, column.alias))
|
||||
groupBy = append(groupBy, column.alias)
|
||||
}
|
||||
selects = append(selects,
|
||||
retentionExpr+" AS retention_days",
|
||||
retentionRuleIndexExpr+" AS retention_rule_index",
|
||||
"ifNull(sum(value), 0) AS value",
|
||||
)
|
||||
groupBy = append(groupBy, "retention_days", "retention_rule_index")
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", slice.StartMs),
|
||||
sb.LT("unix_milli", slice.EndMs),
|
||||
)
|
||||
sb.GroupBy(groupBy...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, columns, nil
|
||||
}
|
||||
|
||||
type dimensionColumn struct {
|
||||
key string
|
||||
alias string
|
||||
}
|
||||
|
||||
func dimensionColumnsFor(retentionGetter retention.Getter, rules []retentiontypes.CustomRetentionRule) ([]dimensionColumn, error) {
|
||||
dimensionKeys, err := retentionGetter.RuleDimensionKeys(rules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(dimensionKeys)+1)
|
||||
keys = append(keys, zeustypes.MeterDimensionWorkspaceKeyID)
|
||||
for _, key := range dimensionKeys {
|
||||
if key == zeustypes.MeterDimensionWorkspaceKeyID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
columns := make([]dimensionColumn, len(keys))
|
||||
for i, key := range keys {
|
||||
columns[i] = dimensionColumn{key: key, alias: fmt.Sprintf("dim_%d", i)}
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func buildDimensions(
|
||||
orgID valuer.UUID,
|
||||
retentionDays int,
|
||||
retentionRuleIndex int,
|
||||
columns []dimensionColumn,
|
||||
values []string,
|
||||
rules []retentiontypes.CustomRetentionRule,
|
||||
) (map[string]string, error) {
|
||||
if len(columns) != len(values) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "dimension column/value count mismatch: %d columns, %d values", len(columns), len(values))
|
||||
}
|
||||
|
||||
valuesByKey := make(map[string]string, len(columns))
|
||||
for i, column := range columns {
|
||||
valuesByKey[column.key] = values[i]
|
||||
}
|
||||
|
||||
dimensions := map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(retentionDays),
|
||||
}
|
||||
addNonEmpty(dimensions, zeustypes.MeterDimensionWorkspaceKeyID, valuesByKey[zeustypes.MeterDimensionWorkspaceKeyID])
|
||||
|
||||
if retentionRuleIndex < 0 {
|
||||
return dimensions, nil
|
||||
}
|
||||
if retentionRuleIndex >= len(rules) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "retention rule index %d out of range for %d rules", retentionRuleIndex, len(rules))
|
||||
}
|
||||
for _, filter := range rules[retentionRuleIndex].Filters {
|
||||
addNonEmpty(dimensions, filter.Key, valuesByKey[filter.Key])
|
||||
}
|
||||
return dimensions, nil
|
||||
}
|
||||
|
||||
func addNonEmpty(dimensions map[string]string, key, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
dimensions[key] = value
|
||||
}
|
||||
|
||||
func bucketKey(dimensions map[string]string) string {
|
||||
keys := make([]string, 0, len(dimensions))
|
||||
for key := range dimensions {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var b strings.Builder
|
||||
for _, key := range keys {
|
||||
value := dimensions[key]
|
||||
b.WriteString(strconv.Itoa(len(key)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(key)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(strconv.Itoa(len(value)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(value)
|
||||
b.WriteByte(';')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package datapointcountmetercollector
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDimensions(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
rules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service.name",
|
||||
Values: []string{"api"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
columns := []dimensionColumn{
|
||||
{key: zeustypes.MeterDimensionWorkspaceKeyID, alias: "dim_0"},
|
||||
{key: "service.name", alias: "dim_1"},
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, 30, 0, columns, []string{"workspace-1", "api"}, rules)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
}, dimensions)
|
||||
}
|
||||
|
||||
func TestProviderMetadata(t *testing.T) {
|
||||
provider := New(nil, nil)
|
||||
|
||||
require.Equal(t, "signoz.meter.metric.datapoint.count", provider.Name().String())
|
||||
require.Equal(t, zeustypes.MeterUnitCount, provider.Unit())
|
||||
require.Equal(t, zeustypes.MeterAggregationSum, provider.Aggregation())
|
||||
}
|
||||
|
||||
func TestBucketKeyIsStable(t *testing.T) {
|
||||
first := bucketKey(map[string]string{
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
})
|
||||
second := bucketKey(map[string]string{
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
})
|
||||
|
||||
require.Equal(t, first, second)
|
||||
require.NotEmpty(t, first)
|
||||
}
|
||||
271
ee/metercollector/datapointsizemetercollector/provider.go
Normal file
271
ee/metercollector/datapointsizemetercollector/provider.go
Normal file
@@ -0,0 +1,271 @@
|
||||
// Package datapointsizemetercollector collects metric datapoint size meters
|
||||
// by workspace and retention. Keep the query local to this meter.
|
||||
package datapointsizemetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterName is the typed registry key for this collector.
|
||||
var (
|
||||
MeterName = zeustypes.MustNewMeterName("signoz.meter.metric.datapoint.size")
|
||||
meterUnit = zeustypes.MeterUnitBytes
|
||||
meterAggregation = zeustypes.MeterAggregationSum
|
||||
)
|
||||
|
||||
var _ metercollector.MeterCollector = (*Provider)(nil)
|
||||
|
||||
// Provider collects datapoint size meters.
|
||||
type Provider struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
retentionGetter retention.Getter
|
||||
}
|
||||
|
||||
func New(telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter) *Provider {
|
||||
return &Provider{
|
||||
telemetryStore: telemetryStore,
|
||||
retentionGetter: retentionGetter,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) Name() zeustypes.MeterName { return MeterName }
|
||||
func (p *Provider) Unit() zeustypes.MeterUnit { return meterUnit }
|
||||
func (p *Provider) Aggregation() zeustypes.MeterAggregation {
|
||||
return meterAggregation
|
||||
}
|
||||
|
||||
// Collect aggregates datapoint size for the window and emits an empty-day sentinel.
|
||||
func (p *Provider) Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
meterName := MeterName.String()
|
||||
|
||||
slices, err := p.retentionGetter.ActiveSlices(
|
||||
ctx,
|
||||
orgID,
|
||||
telemetrymetrics.DBName,
|
||||
telemetrymetrics.SamplesV4LocalTableName,
|
||||
retentiontypes.DefaultMetricsRetentionDays,
|
||||
window.StartUnixMilli,
|
||||
window.EndUnixMilli,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "load retention slices for meter %q", meterName)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
dimensions map[string]string
|
||||
value float64
|
||||
}
|
||||
accumulator := make(map[string]*bucket)
|
||||
|
||||
for _, slice := range slices {
|
||||
query, args, dimensionColumns, err := buildQuery(meterName, slice, p.retentionGetter)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build retention query for meter %q", meterName)
|
||||
}
|
||||
|
||||
rows, err := p.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "query meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
if err := func() error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
dimensionValues := make([]string, len(dimensionColumns))
|
||||
var retentionDays int32
|
||||
var retentionRuleIndex int32
|
||||
var value float64
|
||||
|
||||
scanDest := make([]any, 0, len(dimensionValues)+3)
|
||||
for i := range dimensionValues {
|
||||
scanDest = append(scanDest, &dimensionValues[i])
|
||||
}
|
||||
scanDest = append(scanDest, &retentionDays, &retentionRuleIndex, &value)
|
||||
|
||||
if err := rows.Scan(scanDest...); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "scan meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, int(retentionDays), int(retentionRuleIndex), dimensionColumns, dimensionValues, slice.Rules)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build dimensions for meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
key := bucketKey(dimensions)
|
||||
b, ok := accumulator[key]
|
||||
if !ok {
|
||||
b = &bucket{dimensions: dimensions}
|
||||
accumulator[key] = b
|
||||
}
|
||||
b.value += value
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "iterate meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
meters := make([]zeustypes.Meter, 0, len(accumulator))
|
||||
for _, b := range accumulator {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, b.value, meterUnit, meterAggregation, window, b.dimensions))
|
||||
}
|
||||
|
||||
// Empty windows still emit a sentinel so checkpoints can advance.
|
||||
if len(meters) == 0 && len(slices) > 0 {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, 0, meterUnit, meterAggregation, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(slices[len(slices)-1].DefaultDays),
|
||||
}))
|
||||
}
|
||||
|
||||
return meters, nil
|
||||
}
|
||||
|
||||
// buildQuery stays local because each meter owns its billing query.
|
||||
func buildQuery(meterName string, slice retentiontypes.Slice, retentionGetter retention.Getter) (string, []any, []dimensionColumn, error) {
|
||||
retentionExpr, err := retentionGetter.BuildMultiIfSQL(slice.Rules, slice.DefaultDays)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
retentionRuleIndexExpr, err := retentionGetter.BuildRuleIndexSQL(slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
columns, err := dimensionColumnsFor(retentionGetter, slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
selects := make([]string, 0, len(columns)+3)
|
||||
groupBy := make([]string, 0, len(columns)+2)
|
||||
for _, column := range columns {
|
||||
selects = append(selects, fmt.Sprintf("JSONExtractString(labels, '%s') AS %s", column.key, column.alias))
|
||||
groupBy = append(groupBy, column.alias)
|
||||
}
|
||||
selects = append(selects,
|
||||
retentionExpr+" AS retention_days",
|
||||
retentionRuleIndexExpr+" AS retention_rule_index",
|
||||
"ifNull(sum(value), 0) AS value",
|
||||
)
|
||||
groupBy = append(groupBy, "retention_days", "retention_rule_index")
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", slice.StartMs),
|
||||
sb.LT("unix_milli", slice.EndMs),
|
||||
)
|
||||
sb.GroupBy(groupBy...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, columns, nil
|
||||
}
|
||||
|
||||
type dimensionColumn struct {
|
||||
key string
|
||||
alias string
|
||||
}
|
||||
|
||||
func dimensionColumnsFor(retentionGetter retention.Getter, rules []retentiontypes.CustomRetentionRule) ([]dimensionColumn, error) {
|
||||
dimensionKeys, err := retentionGetter.RuleDimensionKeys(rules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(dimensionKeys)+1)
|
||||
keys = append(keys, zeustypes.MeterDimensionWorkspaceKeyID)
|
||||
for _, key := range dimensionKeys {
|
||||
if key == zeustypes.MeterDimensionWorkspaceKeyID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
columns := make([]dimensionColumn, len(keys))
|
||||
for i, key := range keys {
|
||||
columns[i] = dimensionColumn{key: key, alias: fmt.Sprintf("dim_%d", i)}
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func buildDimensions(
|
||||
orgID valuer.UUID,
|
||||
retentionDays int,
|
||||
retentionRuleIndex int,
|
||||
columns []dimensionColumn,
|
||||
values []string,
|
||||
rules []retentiontypes.CustomRetentionRule,
|
||||
) (map[string]string, error) {
|
||||
if len(columns) != len(values) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "dimension column/value count mismatch: %d columns, %d values", len(columns), len(values))
|
||||
}
|
||||
|
||||
valuesByKey := make(map[string]string, len(columns))
|
||||
for i, column := range columns {
|
||||
valuesByKey[column.key] = values[i]
|
||||
}
|
||||
|
||||
dimensions := map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(retentionDays),
|
||||
}
|
||||
addNonEmpty(dimensions, zeustypes.MeterDimensionWorkspaceKeyID, valuesByKey[zeustypes.MeterDimensionWorkspaceKeyID])
|
||||
|
||||
if retentionRuleIndex < 0 {
|
||||
return dimensions, nil
|
||||
}
|
||||
if retentionRuleIndex >= len(rules) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "retention rule index %d out of range for %d rules", retentionRuleIndex, len(rules))
|
||||
}
|
||||
for _, filter := range rules[retentionRuleIndex].Filters {
|
||||
addNonEmpty(dimensions, filter.Key, valuesByKey[filter.Key])
|
||||
}
|
||||
return dimensions, nil
|
||||
}
|
||||
|
||||
func addNonEmpty(dimensions map[string]string, key, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
dimensions[key] = value
|
||||
}
|
||||
|
||||
func bucketKey(dimensions map[string]string) string {
|
||||
keys := make([]string, 0, len(dimensions))
|
||||
for key := range dimensions {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var b strings.Builder
|
||||
for _, key := range keys {
|
||||
value := dimensions[key]
|
||||
b.WriteString(strconv.Itoa(len(key)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(key)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(strconv.Itoa(len(value)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(value)
|
||||
b.WriteByte(';')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package datapointsizemetercollector
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDimensions(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
rules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service.name",
|
||||
Values: []string{"api"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
columns := []dimensionColumn{
|
||||
{key: zeustypes.MeterDimensionWorkspaceKeyID, alias: "dim_0"},
|
||||
{key: "service.name", alias: "dim_1"},
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, 30, 0, columns, []string{"workspace-1", "api"}, rules)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
}, dimensions)
|
||||
}
|
||||
|
||||
func TestProviderMetadata(t *testing.T) {
|
||||
provider := New(nil, nil)
|
||||
|
||||
require.Equal(t, "signoz.meter.metric.datapoint.size", provider.Name().String())
|
||||
require.Equal(t, zeustypes.MeterUnitBytes, provider.Unit())
|
||||
require.Equal(t, zeustypes.MeterAggregationSum, provider.Aggregation())
|
||||
}
|
||||
|
||||
func TestBucketKeyIsStable(t *testing.T) {
|
||||
first := bucketKey(map[string]string{
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
})
|
||||
second := bucketKey(map[string]string{
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
})
|
||||
|
||||
require.Equal(t, first, second)
|
||||
require.NotEmpty(t, first)
|
||||
}
|
||||
271
ee/metercollector/logcountmetercollector/provider.go
Normal file
271
ee/metercollector/logcountmetercollector/provider.go
Normal file
@@ -0,0 +1,271 @@
|
||||
// Package logcountmetercollector collects log count meters by workspace and
|
||||
// retention. Keep the query local to this meter.
|
||||
package logcountmetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterName is the typed registry key for this collector.
|
||||
var (
|
||||
MeterName = zeustypes.MustNewMeterName("signoz.meter.log.count")
|
||||
meterUnit = zeustypes.MeterUnitCount
|
||||
meterAggregation = zeustypes.MeterAggregationSum
|
||||
)
|
||||
|
||||
var _ metercollector.MeterCollector = (*Provider)(nil)
|
||||
|
||||
// Provider collects log count meters.
|
||||
type Provider struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
retentionGetter retention.Getter
|
||||
}
|
||||
|
||||
func New(telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter) *Provider {
|
||||
return &Provider{
|
||||
telemetryStore: telemetryStore,
|
||||
retentionGetter: retentionGetter,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) Name() zeustypes.MeterName { return MeterName }
|
||||
func (p *Provider) Unit() zeustypes.MeterUnit { return meterUnit }
|
||||
func (p *Provider) Aggregation() zeustypes.MeterAggregation {
|
||||
return meterAggregation
|
||||
}
|
||||
|
||||
// Collect aggregates log count for the window and emits an empty-day sentinel.
|
||||
func (p *Provider) Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
meterName := MeterName.String()
|
||||
|
||||
slices, err := p.retentionGetter.ActiveSlices(
|
||||
ctx,
|
||||
orgID,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2LocalTableName,
|
||||
retentiontypes.DefaultLogsRetentionDays,
|
||||
window.StartUnixMilli,
|
||||
window.EndUnixMilli,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "load retention slices for meter %q", meterName)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
dimensions map[string]string
|
||||
value float64
|
||||
}
|
||||
accumulator := make(map[string]*bucket)
|
||||
|
||||
for _, slice := range slices {
|
||||
query, args, dimensionColumns, err := buildQuery(meterName, slice, p.retentionGetter)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build retention query for meter %q", meterName)
|
||||
}
|
||||
|
||||
rows, err := p.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "query meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
if err := func() error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
dimensionValues := make([]string, len(dimensionColumns))
|
||||
var retentionDays int32
|
||||
var retentionRuleIndex int32
|
||||
var value float64
|
||||
|
||||
scanDest := make([]any, 0, len(dimensionValues)+3)
|
||||
for i := range dimensionValues {
|
||||
scanDest = append(scanDest, &dimensionValues[i])
|
||||
}
|
||||
scanDest = append(scanDest, &retentionDays, &retentionRuleIndex, &value)
|
||||
|
||||
if err := rows.Scan(scanDest...); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "scan meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, int(retentionDays), int(retentionRuleIndex), dimensionColumns, dimensionValues, slice.Rules)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build dimensions for meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
key := bucketKey(dimensions)
|
||||
b, ok := accumulator[key]
|
||||
if !ok {
|
||||
b = &bucket{dimensions: dimensions}
|
||||
accumulator[key] = b
|
||||
}
|
||||
b.value += value
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "iterate meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
meters := make([]zeustypes.Meter, 0, len(accumulator))
|
||||
for _, b := range accumulator {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, b.value, meterUnit, meterAggregation, window, b.dimensions))
|
||||
}
|
||||
|
||||
// Empty windows still emit a sentinel so checkpoints can advance.
|
||||
if len(meters) == 0 && len(slices) > 0 {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, 0, meterUnit, meterAggregation, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(slices[len(slices)-1].DefaultDays),
|
||||
}))
|
||||
}
|
||||
|
||||
return meters, nil
|
||||
}
|
||||
|
||||
// buildQuery stays local because each meter owns its billing query.
|
||||
func buildQuery(meterName string, slice retentiontypes.Slice, retentionGetter retention.Getter) (string, []any, []dimensionColumn, error) {
|
||||
retentionExpr, err := retentionGetter.BuildMultiIfSQL(slice.Rules, slice.DefaultDays)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
retentionRuleIndexExpr, err := retentionGetter.BuildRuleIndexSQL(slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
columns, err := dimensionColumnsFor(retentionGetter, slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
selects := make([]string, 0, len(columns)+3)
|
||||
groupBy := make([]string, 0, len(columns)+2)
|
||||
for _, column := range columns {
|
||||
selects = append(selects, fmt.Sprintf("JSONExtractString(labels, '%s') AS %s", column.key, column.alias))
|
||||
groupBy = append(groupBy, column.alias)
|
||||
}
|
||||
selects = append(selects,
|
||||
retentionExpr+" AS retention_days",
|
||||
retentionRuleIndexExpr+" AS retention_rule_index",
|
||||
"ifNull(sum(value), 0) AS value",
|
||||
)
|
||||
groupBy = append(groupBy, "retention_days", "retention_rule_index")
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", slice.StartMs),
|
||||
sb.LT("unix_milli", slice.EndMs),
|
||||
)
|
||||
sb.GroupBy(groupBy...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, columns, nil
|
||||
}
|
||||
|
||||
type dimensionColumn struct {
|
||||
key string
|
||||
alias string
|
||||
}
|
||||
|
||||
func dimensionColumnsFor(retentionGetter retention.Getter, rules []retentiontypes.CustomRetentionRule) ([]dimensionColumn, error) {
|
||||
dimensionKeys, err := retentionGetter.RuleDimensionKeys(rules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(dimensionKeys)+1)
|
||||
keys = append(keys, zeustypes.MeterDimensionWorkspaceKeyID)
|
||||
for _, key := range dimensionKeys {
|
||||
if key == zeustypes.MeterDimensionWorkspaceKeyID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
columns := make([]dimensionColumn, len(keys))
|
||||
for i, key := range keys {
|
||||
columns[i] = dimensionColumn{key: key, alias: fmt.Sprintf("dim_%d", i)}
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func buildDimensions(
|
||||
orgID valuer.UUID,
|
||||
retentionDays int,
|
||||
retentionRuleIndex int,
|
||||
columns []dimensionColumn,
|
||||
values []string,
|
||||
rules []retentiontypes.CustomRetentionRule,
|
||||
) (map[string]string, error) {
|
||||
if len(columns) != len(values) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "dimension column/value count mismatch: %d columns, %d values", len(columns), len(values))
|
||||
}
|
||||
|
||||
valuesByKey := make(map[string]string, len(columns))
|
||||
for i, column := range columns {
|
||||
valuesByKey[column.key] = values[i]
|
||||
}
|
||||
|
||||
dimensions := map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(retentionDays),
|
||||
}
|
||||
addNonEmpty(dimensions, zeustypes.MeterDimensionWorkspaceKeyID, valuesByKey[zeustypes.MeterDimensionWorkspaceKeyID])
|
||||
|
||||
if retentionRuleIndex < 0 {
|
||||
return dimensions, nil
|
||||
}
|
||||
if retentionRuleIndex >= len(rules) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "retention rule index %d out of range for %d rules", retentionRuleIndex, len(rules))
|
||||
}
|
||||
for _, filter := range rules[retentionRuleIndex].Filters {
|
||||
addNonEmpty(dimensions, filter.Key, valuesByKey[filter.Key])
|
||||
}
|
||||
return dimensions, nil
|
||||
}
|
||||
|
||||
func addNonEmpty(dimensions map[string]string, key, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
dimensions[key] = value
|
||||
}
|
||||
|
||||
func bucketKey(dimensions map[string]string) string {
|
||||
keys := make([]string, 0, len(dimensions))
|
||||
for key := range dimensions {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var b strings.Builder
|
||||
for _, key := range keys {
|
||||
value := dimensions[key]
|
||||
b.WriteString(strconv.Itoa(len(key)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(key)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(strconv.Itoa(len(value)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(value)
|
||||
b.WriteByte(';')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
58
ee/metercollector/logcountmetercollector/provider_test.go
Normal file
58
ee/metercollector/logcountmetercollector/provider_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package logcountmetercollector
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDimensions(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
rules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service.name",
|
||||
Values: []string{"api"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
columns := []dimensionColumn{
|
||||
{key: zeustypes.MeterDimensionWorkspaceKeyID, alias: "dim_0"},
|
||||
{key: "service.name", alias: "dim_1"},
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, 30, 0, columns, []string{"workspace-1", "api"}, rules)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
}, dimensions)
|
||||
}
|
||||
|
||||
func TestProviderMetadata(t *testing.T) {
|
||||
provider := New(nil, nil)
|
||||
|
||||
require.Equal(t, "signoz.meter.log.count", provider.Name().String())
|
||||
require.Equal(t, zeustypes.MeterUnitCount, provider.Unit())
|
||||
require.Equal(t, zeustypes.MeterAggregationSum, provider.Aggregation())
|
||||
}
|
||||
|
||||
func TestBucketKeyIsStable(t *testing.T) {
|
||||
first := bucketKey(map[string]string{
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
})
|
||||
second := bucketKey(map[string]string{
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
})
|
||||
|
||||
require.Equal(t, first, second)
|
||||
require.NotEmpty(t, first)
|
||||
}
|
||||
271
ee/metercollector/logsizemetercollector/provider.go
Normal file
271
ee/metercollector/logsizemetercollector/provider.go
Normal file
@@ -0,0 +1,271 @@
|
||||
// Package logsizemetercollector collects log size meters by workspace and
|
||||
// retention. Keep the query local to this meter.
|
||||
package logsizemetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrylogs"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterName is the typed registry key for this collector.
|
||||
var (
|
||||
MeterName = zeustypes.MustNewMeterName("signoz.meter.log.size")
|
||||
meterUnit = zeustypes.MeterUnitBytes
|
||||
meterAggregation = zeustypes.MeterAggregationSum
|
||||
)
|
||||
|
||||
var _ metercollector.MeterCollector = (*Provider)(nil)
|
||||
|
||||
// Provider collects log size meters.
|
||||
type Provider struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
retentionGetter retention.Getter
|
||||
}
|
||||
|
||||
func New(telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter) *Provider {
|
||||
return &Provider{
|
||||
telemetryStore: telemetryStore,
|
||||
retentionGetter: retentionGetter,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) Name() zeustypes.MeterName { return MeterName }
|
||||
func (p *Provider) Unit() zeustypes.MeterUnit { return meterUnit }
|
||||
func (p *Provider) Aggregation() zeustypes.MeterAggregation {
|
||||
return meterAggregation
|
||||
}
|
||||
|
||||
// Collect aggregates log size for the window and emits an empty-day sentinel.
|
||||
func (p *Provider) Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
meterName := MeterName.String()
|
||||
|
||||
slices, err := p.retentionGetter.ActiveSlices(
|
||||
ctx,
|
||||
orgID,
|
||||
telemetrylogs.DBName,
|
||||
telemetrylogs.LogsV2LocalTableName,
|
||||
retentiontypes.DefaultLogsRetentionDays,
|
||||
window.StartUnixMilli,
|
||||
window.EndUnixMilli,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "load retention slices for meter %q", meterName)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
dimensions map[string]string
|
||||
value float64
|
||||
}
|
||||
accumulator := make(map[string]*bucket)
|
||||
|
||||
for _, slice := range slices {
|
||||
query, args, dimensionColumns, err := buildQuery(meterName, slice, p.retentionGetter)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build retention query for meter %q", meterName)
|
||||
}
|
||||
|
||||
rows, err := p.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "query meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
if err := func() error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
dimensionValues := make([]string, len(dimensionColumns))
|
||||
var retentionDays int32
|
||||
var retentionRuleIndex int32
|
||||
var value float64
|
||||
|
||||
scanDest := make([]any, 0, len(dimensionValues)+3)
|
||||
for i := range dimensionValues {
|
||||
scanDest = append(scanDest, &dimensionValues[i])
|
||||
}
|
||||
scanDest = append(scanDest, &retentionDays, &retentionRuleIndex, &value)
|
||||
|
||||
if err := rows.Scan(scanDest...); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "scan meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, int(retentionDays), int(retentionRuleIndex), dimensionColumns, dimensionValues, slice.Rules)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build dimensions for meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
key := bucketKey(dimensions)
|
||||
b, ok := accumulator[key]
|
||||
if !ok {
|
||||
b = &bucket{dimensions: dimensions}
|
||||
accumulator[key] = b
|
||||
}
|
||||
b.value += value
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "iterate meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
meters := make([]zeustypes.Meter, 0, len(accumulator))
|
||||
for _, b := range accumulator {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, b.value, meterUnit, meterAggregation, window, b.dimensions))
|
||||
}
|
||||
|
||||
// Empty windows still emit a sentinel so checkpoints can advance.
|
||||
if len(meters) == 0 && len(slices) > 0 {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, 0, meterUnit, meterAggregation, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(slices[len(slices)-1].DefaultDays),
|
||||
}))
|
||||
}
|
||||
|
||||
return meters, nil
|
||||
}
|
||||
|
||||
// buildQuery stays local because each meter owns its billing query.
|
||||
func buildQuery(meterName string, slice retentiontypes.Slice, retentionGetter retention.Getter) (string, []any, []dimensionColumn, error) {
|
||||
retentionExpr, err := retentionGetter.BuildMultiIfSQL(slice.Rules, slice.DefaultDays)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
retentionRuleIndexExpr, err := retentionGetter.BuildRuleIndexSQL(slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
columns, err := dimensionColumnsFor(retentionGetter, slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
selects := make([]string, 0, len(columns)+3)
|
||||
groupBy := make([]string, 0, len(columns)+2)
|
||||
for _, column := range columns {
|
||||
selects = append(selects, fmt.Sprintf("JSONExtractString(labels, '%s') AS %s", column.key, column.alias))
|
||||
groupBy = append(groupBy, column.alias)
|
||||
}
|
||||
selects = append(selects,
|
||||
retentionExpr+" AS retention_days",
|
||||
retentionRuleIndexExpr+" AS retention_rule_index",
|
||||
"ifNull(sum(value), 0) AS value",
|
||||
)
|
||||
groupBy = append(groupBy, "retention_days", "retention_rule_index")
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", slice.StartMs),
|
||||
sb.LT("unix_milli", slice.EndMs),
|
||||
)
|
||||
sb.GroupBy(groupBy...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, columns, nil
|
||||
}
|
||||
|
||||
type dimensionColumn struct {
|
||||
key string
|
||||
alias string
|
||||
}
|
||||
|
||||
func dimensionColumnsFor(retentionGetter retention.Getter, rules []retentiontypes.CustomRetentionRule) ([]dimensionColumn, error) {
|
||||
dimensionKeys, err := retentionGetter.RuleDimensionKeys(rules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(dimensionKeys)+1)
|
||||
keys = append(keys, zeustypes.MeterDimensionWorkspaceKeyID)
|
||||
for _, key := range dimensionKeys {
|
||||
if key == zeustypes.MeterDimensionWorkspaceKeyID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
columns := make([]dimensionColumn, len(keys))
|
||||
for i, key := range keys {
|
||||
columns[i] = dimensionColumn{key: key, alias: fmt.Sprintf("dim_%d", i)}
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func buildDimensions(
|
||||
orgID valuer.UUID,
|
||||
retentionDays int,
|
||||
retentionRuleIndex int,
|
||||
columns []dimensionColumn,
|
||||
values []string,
|
||||
rules []retentiontypes.CustomRetentionRule,
|
||||
) (map[string]string, error) {
|
||||
if len(columns) != len(values) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "dimension column/value count mismatch: %d columns, %d values", len(columns), len(values))
|
||||
}
|
||||
|
||||
valuesByKey := make(map[string]string, len(columns))
|
||||
for i, column := range columns {
|
||||
valuesByKey[column.key] = values[i]
|
||||
}
|
||||
|
||||
dimensions := map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(retentionDays),
|
||||
}
|
||||
addNonEmpty(dimensions, zeustypes.MeterDimensionWorkspaceKeyID, valuesByKey[zeustypes.MeterDimensionWorkspaceKeyID])
|
||||
|
||||
if retentionRuleIndex < 0 {
|
||||
return dimensions, nil
|
||||
}
|
||||
if retentionRuleIndex >= len(rules) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "retention rule index %d out of range for %d rules", retentionRuleIndex, len(rules))
|
||||
}
|
||||
for _, filter := range rules[retentionRuleIndex].Filters {
|
||||
addNonEmpty(dimensions, filter.Key, valuesByKey[filter.Key])
|
||||
}
|
||||
return dimensions, nil
|
||||
}
|
||||
|
||||
func addNonEmpty(dimensions map[string]string, key, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
dimensions[key] = value
|
||||
}
|
||||
|
||||
func bucketKey(dimensions map[string]string) string {
|
||||
keys := make([]string, 0, len(dimensions))
|
||||
for key := range dimensions {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var b strings.Builder
|
||||
for _, key := range keys {
|
||||
value := dimensions[key]
|
||||
b.WriteString(strconv.Itoa(len(key)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(key)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(strconv.Itoa(len(value)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(value)
|
||||
b.WriteByte(';')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
58
ee/metercollector/logsizemetercollector/provider_test.go
Normal file
58
ee/metercollector/logsizemetercollector/provider_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package logsizemetercollector
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDimensions(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
rules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service.name",
|
||||
Values: []string{"api"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
columns := []dimensionColumn{
|
||||
{key: zeustypes.MeterDimensionWorkspaceKeyID, alias: "dim_0"},
|
||||
{key: "service.name", alias: "dim_1"},
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, 30, 0, columns, []string{"workspace-1", "api"}, rules)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
}, dimensions)
|
||||
}
|
||||
|
||||
func TestProviderMetadata(t *testing.T) {
|
||||
provider := New(nil, nil)
|
||||
|
||||
require.Equal(t, "signoz.meter.log.size", provider.Name().String())
|
||||
require.Equal(t, zeustypes.MeterUnitBytes, provider.Unit())
|
||||
require.Equal(t, zeustypes.MeterAggregationSum, provider.Aggregation())
|
||||
}
|
||||
|
||||
func TestBucketKeyIsStable(t *testing.T) {
|
||||
first := bucketKey(map[string]string{
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
})
|
||||
second := bucketKey(map[string]string{
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
})
|
||||
|
||||
require.Equal(t, first, second)
|
||||
require.NotEmpty(t, first)
|
||||
}
|
||||
271
ee/metercollector/spancountmetercollector/provider.go
Normal file
271
ee/metercollector/spancountmetercollector/provider.go
Normal file
@@ -0,0 +1,271 @@
|
||||
// Package spancountmetercollector collects span count meters by workspace and
|
||||
// retention. Keep the query local to this meter.
|
||||
package spancountmetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterName is the typed registry key for this collector.
|
||||
var (
|
||||
MeterName = zeustypes.MustNewMeterName("signoz.meter.span.count")
|
||||
meterUnit = zeustypes.MeterUnitCount
|
||||
meterAggregation = zeustypes.MeterAggregationSum
|
||||
)
|
||||
|
||||
var _ metercollector.MeterCollector = (*Provider)(nil)
|
||||
|
||||
// Provider collects span count meters.
|
||||
type Provider struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
retentionGetter retention.Getter
|
||||
}
|
||||
|
||||
func New(telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter) *Provider {
|
||||
return &Provider{
|
||||
telemetryStore: telemetryStore,
|
||||
retentionGetter: retentionGetter,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) Name() zeustypes.MeterName { return MeterName }
|
||||
func (p *Provider) Unit() zeustypes.MeterUnit { return meterUnit }
|
||||
func (p *Provider) Aggregation() zeustypes.MeterAggregation {
|
||||
return meterAggregation
|
||||
}
|
||||
|
||||
// Collect aggregates span count for the window and emits an empty-day sentinel.
|
||||
func (p *Provider) Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
meterName := MeterName.String()
|
||||
|
||||
slices, err := p.retentionGetter.ActiveSlices(
|
||||
ctx,
|
||||
orgID,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.SpanIndexV3LocalTableName,
|
||||
retentiontypes.DefaultTracesRetentionDays,
|
||||
window.StartUnixMilli,
|
||||
window.EndUnixMilli,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "load retention slices for meter %q", meterName)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
dimensions map[string]string
|
||||
value float64
|
||||
}
|
||||
accumulator := make(map[string]*bucket)
|
||||
|
||||
for _, slice := range slices {
|
||||
query, args, dimensionColumns, err := buildQuery(meterName, slice, p.retentionGetter)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build retention query for meter %q", meterName)
|
||||
}
|
||||
|
||||
rows, err := p.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "query meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
if err := func() error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
dimensionValues := make([]string, len(dimensionColumns))
|
||||
var retentionDays int32
|
||||
var retentionRuleIndex int32
|
||||
var value float64
|
||||
|
||||
scanDest := make([]any, 0, len(dimensionValues)+3)
|
||||
for i := range dimensionValues {
|
||||
scanDest = append(scanDest, &dimensionValues[i])
|
||||
}
|
||||
scanDest = append(scanDest, &retentionDays, &retentionRuleIndex, &value)
|
||||
|
||||
if err := rows.Scan(scanDest...); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "scan meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, int(retentionDays), int(retentionRuleIndex), dimensionColumns, dimensionValues, slice.Rules)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build dimensions for meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
key := bucketKey(dimensions)
|
||||
b, ok := accumulator[key]
|
||||
if !ok {
|
||||
b = &bucket{dimensions: dimensions}
|
||||
accumulator[key] = b
|
||||
}
|
||||
b.value += value
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "iterate meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
meters := make([]zeustypes.Meter, 0, len(accumulator))
|
||||
for _, b := range accumulator {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, b.value, meterUnit, meterAggregation, window, b.dimensions))
|
||||
}
|
||||
|
||||
// Empty windows still emit a sentinel so checkpoints can advance.
|
||||
if len(meters) == 0 && len(slices) > 0 {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, 0, meterUnit, meterAggregation, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(slices[len(slices)-1].DefaultDays),
|
||||
}))
|
||||
}
|
||||
|
||||
return meters, nil
|
||||
}
|
||||
|
||||
// buildQuery stays local because each meter owns its billing query.
|
||||
func buildQuery(meterName string, slice retentiontypes.Slice, retentionGetter retention.Getter) (string, []any, []dimensionColumn, error) {
|
||||
retentionExpr, err := retentionGetter.BuildMultiIfSQL(slice.Rules, slice.DefaultDays)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
retentionRuleIndexExpr, err := retentionGetter.BuildRuleIndexSQL(slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
columns, err := dimensionColumnsFor(retentionGetter, slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
selects := make([]string, 0, len(columns)+3)
|
||||
groupBy := make([]string, 0, len(columns)+2)
|
||||
for _, column := range columns {
|
||||
selects = append(selects, fmt.Sprintf("JSONExtractString(labels, '%s') AS %s", column.key, column.alias))
|
||||
groupBy = append(groupBy, column.alias)
|
||||
}
|
||||
selects = append(selects,
|
||||
retentionExpr+" AS retention_days",
|
||||
retentionRuleIndexExpr+" AS retention_rule_index",
|
||||
"ifNull(sum(value), 0) AS value",
|
||||
)
|
||||
groupBy = append(groupBy, "retention_days", "retention_rule_index")
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", slice.StartMs),
|
||||
sb.LT("unix_milli", slice.EndMs),
|
||||
)
|
||||
sb.GroupBy(groupBy...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, columns, nil
|
||||
}
|
||||
|
||||
type dimensionColumn struct {
|
||||
key string
|
||||
alias string
|
||||
}
|
||||
|
||||
func dimensionColumnsFor(retentionGetter retention.Getter, rules []retentiontypes.CustomRetentionRule) ([]dimensionColumn, error) {
|
||||
dimensionKeys, err := retentionGetter.RuleDimensionKeys(rules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(dimensionKeys)+1)
|
||||
keys = append(keys, zeustypes.MeterDimensionWorkspaceKeyID)
|
||||
for _, key := range dimensionKeys {
|
||||
if key == zeustypes.MeterDimensionWorkspaceKeyID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
columns := make([]dimensionColumn, len(keys))
|
||||
for i, key := range keys {
|
||||
columns[i] = dimensionColumn{key: key, alias: fmt.Sprintf("dim_%d", i)}
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func buildDimensions(
|
||||
orgID valuer.UUID,
|
||||
retentionDays int,
|
||||
retentionRuleIndex int,
|
||||
columns []dimensionColumn,
|
||||
values []string,
|
||||
rules []retentiontypes.CustomRetentionRule,
|
||||
) (map[string]string, error) {
|
||||
if len(columns) != len(values) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "dimension column/value count mismatch: %d columns, %d values", len(columns), len(values))
|
||||
}
|
||||
|
||||
valuesByKey := make(map[string]string, len(columns))
|
||||
for i, column := range columns {
|
||||
valuesByKey[column.key] = values[i]
|
||||
}
|
||||
|
||||
dimensions := map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(retentionDays),
|
||||
}
|
||||
addNonEmpty(dimensions, zeustypes.MeterDimensionWorkspaceKeyID, valuesByKey[zeustypes.MeterDimensionWorkspaceKeyID])
|
||||
|
||||
if retentionRuleIndex < 0 {
|
||||
return dimensions, nil
|
||||
}
|
||||
if retentionRuleIndex >= len(rules) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "retention rule index %d out of range for %d rules", retentionRuleIndex, len(rules))
|
||||
}
|
||||
for _, filter := range rules[retentionRuleIndex].Filters {
|
||||
addNonEmpty(dimensions, filter.Key, valuesByKey[filter.Key])
|
||||
}
|
||||
return dimensions, nil
|
||||
}
|
||||
|
||||
func addNonEmpty(dimensions map[string]string, key, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
dimensions[key] = value
|
||||
}
|
||||
|
||||
func bucketKey(dimensions map[string]string) string {
|
||||
keys := make([]string, 0, len(dimensions))
|
||||
for key := range dimensions {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var b strings.Builder
|
||||
for _, key := range keys {
|
||||
value := dimensions[key]
|
||||
b.WriteString(strconv.Itoa(len(key)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(key)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(strconv.Itoa(len(value)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(value)
|
||||
b.WriteByte(';')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
58
ee/metercollector/spancountmetercollector/provider_test.go
Normal file
58
ee/metercollector/spancountmetercollector/provider_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package spancountmetercollector
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDimensions(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
rules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service.name",
|
||||
Values: []string{"api"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
columns := []dimensionColumn{
|
||||
{key: zeustypes.MeterDimensionWorkspaceKeyID, alias: "dim_0"},
|
||||
{key: "service.name", alias: "dim_1"},
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, 30, 0, columns, []string{"workspace-1", "api"}, rules)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
}, dimensions)
|
||||
}
|
||||
|
||||
func TestProviderMetadata(t *testing.T) {
|
||||
provider := New(nil, nil)
|
||||
|
||||
require.Equal(t, "signoz.meter.span.count", provider.Name().String())
|
||||
require.Equal(t, zeustypes.MeterUnitCount, provider.Unit())
|
||||
require.Equal(t, zeustypes.MeterAggregationSum, provider.Aggregation())
|
||||
}
|
||||
|
||||
func TestBucketKeyIsStable(t *testing.T) {
|
||||
first := bucketKey(map[string]string{
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
})
|
||||
second := bucketKey(map[string]string{
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
})
|
||||
|
||||
require.Equal(t, first, second)
|
||||
require.NotEmpty(t, first)
|
||||
}
|
||||
271
ee/metercollector/spansizemetercollector/provider.go
Normal file
271
ee/metercollector/spansizemetercollector/provider.go
Normal file
@@ -0,0 +1,271 @@
|
||||
// Package spansizemetercollector collects span size meters by workspace and
|
||||
// retention. Keep the query local to this meter.
|
||||
package spansizemetercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrytraces"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterName is the typed registry key for this collector.
|
||||
var (
|
||||
MeterName = zeustypes.MustNewMeterName("signoz.meter.span.size")
|
||||
meterUnit = zeustypes.MeterUnitBytes
|
||||
meterAggregation = zeustypes.MeterAggregationSum
|
||||
)
|
||||
|
||||
var _ metercollector.MeterCollector = (*Provider)(nil)
|
||||
|
||||
// Provider collects span size meters.
|
||||
type Provider struct {
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
retentionGetter retention.Getter
|
||||
}
|
||||
|
||||
func New(telemetryStore telemetrystore.TelemetryStore, retentionGetter retention.Getter) *Provider {
|
||||
return &Provider{
|
||||
telemetryStore: telemetryStore,
|
||||
retentionGetter: retentionGetter,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) Name() zeustypes.MeterName { return MeterName }
|
||||
func (p *Provider) Unit() zeustypes.MeterUnit { return meterUnit }
|
||||
func (p *Provider) Aggregation() zeustypes.MeterAggregation {
|
||||
return meterAggregation
|
||||
}
|
||||
|
||||
// Collect aggregates span size for the window and emits an empty-day sentinel.
|
||||
func (p *Provider) Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
meterName := MeterName.String()
|
||||
|
||||
slices, err := p.retentionGetter.ActiveSlices(
|
||||
ctx,
|
||||
orgID,
|
||||
telemetrytraces.DBName,
|
||||
telemetrytraces.SpanIndexV3LocalTableName,
|
||||
retentiontypes.DefaultTracesRetentionDays,
|
||||
window.StartUnixMilli,
|
||||
window.EndUnixMilli,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "load retention slices for meter %q", meterName)
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
dimensions map[string]string
|
||||
value float64
|
||||
}
|
||||
accumulator := make(map[string]*bucket)
|
||||
|
||||
for _, slice := range slices {
|
||||
query, args, dimensionColumns, err := buildQuery(meterName, slice, p.retentionGetter)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build retention query for meter %q", meterName)
|
||||
}
|
||||
|
||||
rows, err := p.telemetryStore.ClickhouseDB().Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "query meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
if err := func() error {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
dimensionValues := make([]string, len(dimensionColumns))
|
||||
var retentionDays int32
|
||||
var retentionRuleIndex int32
|
||||
var value float64
|
||||
|
||||
scanDest := make([]any, 0, len(dimensionValues)+3)
|
||||
for i := range dimensionValues {
|
||||
scanDest = append(scanDest, &dimensionValues[i])
|
||||
}
|
||||
scanDest = append(scanDest, &retentionDays, &retentionRuleIndex, &value)
|
||||
|
||||
if err := rows.Scan(scanDest...); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "scan meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, int(retentionDays), int(retentionRuleIndex), dimensionColumns, dimensionValues, slice.Rules)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "build dimensions for meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
|
||||
key := bucketKey(dimensions)
|
||||
b, ok := accumulator[key]
|
||||
if !ok {
|
||||
b = &bucket{dimensions: dimensions}
|
||||
accumulator[key] = b
|
||||
}
|
||||
b.value += value
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "iterate meter %q slice [%d, %d)", meterName, slice.StartMs, slice.EndMs)
|
||||
}
|
||||
return nil
|
||||
}(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
meters := make([]zeustypes.Meter, 0, len(accumulator))
|
||||
for _, b := range accumulator {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, b.value, meterUnit, meterAggregation, window, b.dimensions))
|
||||
}
|
||||
|
||||
// Empty windows still emit a sentinel so checkpoints can advance.
|
||||
if len(meters) == 0 && len(slices) > 0 {
|
||||
meters = append(meters, zeustypes.NewMeter(MeterName, 0, meterUnit, meterAggregation, window, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(slices[len(slices)-1].DefaultDays),
|
||||
}))
|
||||
}
|
||||
|
||||
return meters, nil
|
||||
}
|
||||
|
||||
// buildQuery stays local because each meter owns its billing query.
|
||||
func buildQuery(meterName string, slice retentiontypes.Slice, retentionGetter retention.Getter) (string, []any, []dimensionColumn, error) {
|
||||
retentionExpr, err := retentionGetter.BuildMultiIfSQL(slice.Rules, slice.DefaultDays)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
retentionRuleIndexExpr, err := retentionGetter.BuildRuleIndexSQL(slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
columns, err := dimensionColumnsFor(retentionGetter, slice.Rules)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
selects := make([]string, 0, len(columns)+3)
|
||||
groupBy := make([]string, 0, len(columns)+2)
|
||||
for _, column := range columns {
|
||||
selects = append(selects, fmt.Sprintf("JSONExtractString(labels, '%s') AS %s", column.key, column.alias))
|
||||
groupBy = append(groupBy, column.alias)
|
||||
}
|
||||
selects = append(selects,
|
||||
retentionExpr+" AS retention_days",
|
||||
retentionRuleIndexExpr+" AS retention_rule_index",
|
||||
"ifNull(sum(value), 0) AS value",
|
||||
)
|
||||
groupBy = append(groupBy, "retention_days", "retention_rule_index")
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select(selects...)
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
sb.Where(
|
||||
sb.Equal("metric_name", meterName),
|
||||
sb.GTE("unix_milli", slice.StartMs),
|
||||
sb.LT("unix_milli", slice.EndMs),
|
||||
)
|
||||
sb.GroupBy(groupBy...)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
return query, args, columns, nil
|
||||
}
|
||||
|
||||
type dimensionColumn struct {
|
||||
key string
|
||||
alias string
|
||||
}
|
||||
|
||||
func dimensionColumnsFor(retentionGetter retention.Getter, rules []retentiontypes.CustomRetentionRule) ([]dimensionColumn, error) {
|
||||
dimensionKeys, err := retentionGetter.RuleDimensionKeys(rules)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys := make([]string, 0, len(dimensionKeys)+1)
|
||||
keys = append(keys, zeustypes.MeterDimensionWorkspaceKeyID)
|
||||
for _, key := range dimensionKeys {
|
||||
if key == zeustypes.MeterDimensionWorkspaceKeyID {
|
||||
continue
|
||||
}
|
||||
keys = append(keys, key)
|
||||
}
|
||||
columns := make([]dimensionColumn, len(keys))
|
||||
for i, key := range keys {
|
||||
columns[i] = dimensionColumn{key: key, alias: fmt.Sprintf("dim_%d", i)}
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
func buildDimensions(
|
||||
orgID valuer.UUID,
|
||||
retentionDays int,
|
||||
retentionRuleIndex int,
|
||||
columns []dimensionColumn,
|
||||
values []string,
|
||||
rules []retentiontypes.CustomRetentionRule,
|
||||
) (map[string]string, error) {
|
||||
if len(columns) != len(values) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "dimension column/value count mismatch: %d columns, %d values", len(columns), len(values))
|
||||
}
|
||||
|
||||
valuesByKey := make(map[string]string, len(columns))
|
||||
for i, column := range columns {
|
||||
valuesByKey[column.key] = values[i]
|
||||
}
|
||||
|
||||
dimensions := map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: strconv.Itoa(retentionDays),
|
||||
}
|
||||
addNonEmpty(dimensions, zeustypes.MeterDimensionWorkspaceKeyID, valuesByKey[zeustypes.MeterDimensionWorkspaceKeyID])
|
||||
|
||||
if retentionRuleIndex < 0 {
|
||||
return dimensions, nil
|
||||
}
|
||||
if retentionRuleIndex >= len(rules) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "retention rule index %d out of range for %d rules", retentionRuleIndex, len(rules))
|
||||
}
|
||||
for _, filter := range rules[retentionRuleIndex].Filters {
|
||||
addNonEmpty(dimensions, filter.Key, valuesByKey[filter.Key])
|
||||
}
|
||||
return dimensions, nil
|
||||
}
|
||||
|
||||
func addNonEmpty(dimensions map[string]string, key, value string) {
|
||||
if value == "" {
|
||||
return
|
||||
}
|
||||
dimensions[key] = value
|
||||
}
|
||||
|
||||
func bucketKey(dimensions map[string]string) string {
|
||||
keys := make([]string, 0, len(dimensions))
|
||||
for key := range dimensions {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var b strings.Builder
|
||||
for _, key := range keys {
|
||||
value := dimensions[key]
|
||||
b.WriteString(strconv.Itoa(len(key)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(key)
|
||||
b.WriteByte('=')
|
||||
b.WriteString(strconv.Itoa(len(value)))
|
||||
b.WriteByte(':')
|
||||
b.WriteString(value)
|
||||
b.WriteByte(';')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
58
ee/metercollector/spansizemetercollector/provider_test.go
Normal file
58
ee/metercollector/spansizemetercollector/provider_test.go
Normal file
@@ -0,0 +1,58 @@
|
||||
package spansizemetercollector
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDimensions(t *testing.T) {
|
||||
orgID := valuer.GenerateUUID()
|
||||
rules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service.name",
|
||||
Values: []string{"api"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
columns := []dimensionColumn{
|
||||
{key: zeustypes.MeterDimensionWorkspaceKeyID, alias: "dim_0"},
|
||||
{key: "service.name", alias: "dim_1"},
|
||||
}
|
||||
|
||||
dimensions, err := buildDimensions(orgID, 30, 0, columns, []string{"workspace-1", "api"}, rules)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, map[string]string{
|
||||
zeustypes.MeterDimensionOrganizationID: orgID.StringValue(),
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
}, dimensions)
|
||||
}
|
||||
|
||||
func TestProviderMetadata(t *testing.T) {
|
||||
provider := New(nil, nil)
|
||||
|
||||
require.Equal(t, "signoz.meter.span.size", provider.Name().String())
|
||||
require.Equal(t, zeustypes.MeterUnitBytes, provider.Unit())
|
||||
require.Equal(t, zeustypes.MeterAggregationSum, provider.Aggregation())
|
||||
}
|
||||
|
||||
func TestBucketKeyIsStable(t *testing.T) {
|
||||
first := bucketKey(map[string]string{
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
})
|
||||
second := bucketKey(map[string]string{
|
||||
zeustypes.MeterDimensionWorkspaceKeyID: "workspace-1",
|
||||
"service.name": "api",
|
||||
zeustypes.MeterDimensionRetentionDays: "30",
|
||||
})
|
||||
|
||||
require.Equal(t, first, second)
|
||||
require.NotEmpty(t, first)
|
||||
}
|
||||
629
ee/meterreporter/httpmeterreporter/provider.go
Normal file
629
ee/meterreporter/httpmeterreporter/provider.go
Normal file
@@ -0,0 +1,629 @@
|
||||
package httpmeterreporter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/licensing"
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/meterreporter"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrymeter"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/SigNoz/signoz/pkg/zeus"
|
||||
"github.com/huandu/go-sqlbuilder"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var _ factory.ServiceWithHealthy = (*Provider)(nil)
|
||||
|
||||
var errCodeReportFailed = errors.MustNewCode("meterreporter_report_failed")
|
||||
|
||||
const (
|
||||
phaseSealed = "sealed"
|
||||
phaseToday = "today"
|
||||
|
||||
attrPhase = "phase"
|
||||
attrResult = "result"
|
||||
attrMeterReporterProvider = "meterreporter.provider"
|
||||
attrOrgID = "meterreporter.org_id"
|
||||
attrOrgCount = "meterreporter.org_count"
|
||||
attrMeter = "meterreporter.meter"
|
||||
attrDate = "meterreporter.date"
|
||||
attrReadings = "meterreporter.readings"
|
||||
attrReadingsCollected = "meterreporter.readings_collected"
|
||||
attrReadingsDropped = "meterreporter.readings_dropped"
|
||||
attrWindowStartUnixMilli = "meterreporter.window_start_unix_milli"
|
||||
attrWindowEndUnixMilli = "meterreporter.window_end_unix_milli"
|
||||
attrWindowCompleted = "meterreporter.window_completed"
|
||||
attrCatchupStart = "meterreporter.catchup_start"
|
||||
attrCatchupEnd = "meterreporter.catchup_end"
|
||||
attrDurationMs = "meterreporter.duration_ms"
|
||||
attrDryRun = "meterreporter.dry_run"
|
||||
attrIdempotencyKey = "meterreporter.idempotency_key"
|
||||
|
||||
resultSuccess = "success"
|
||||
resultFailure = "failure"
|
||||
|
||||
providerName = "http"
|
||||
)
|
||||
|
||||
// Provider collects registered meters and ships them to Zeus.
|
||||
type Provider struct {
|
||||
settings factory.ScopedProviderSettings
|
||||
config meterreporter.Config
|
||||
collectors []metercollector.MeterCollector
|
||||
|
||||
licensing licensing.Licensing
|
||||
telemetryStore telemetrystore.TelemetryStore
|
||||
orgGetter organization.Getter
|
||||
zeus zeus.Zeus
|
||||
|
||||
healthyC chan struct{}
|
||||
stopC chan struct{}
|
||||
goroutinesWg sync.WaitGroup
|
||||
metrics *reporterMetrics
|
||||
}
|
||||
|
||||
// NewFactory registers the HTTP meter reporter.
|
||||
func NewFactory(
|
||||
collectors map[zeustypes.MeterName]metercollector.MeterCollector,
|
||||
licensing licensing.Licensing,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
orgGetter organization.Getter,
|
||||
zeus zeus.Zeus,
|
||||
) factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config] {
|
||||
return factory.NewProviderFactory(
|
||||
factory.MustNewName(providerName),
|
||||
func(ctx context.Context, providerSettings factory.ProviderSettings, config meterreporter.Config) (meterreporter.Reporter, error) {
|
||||
return newProvider(ctx, providerSettings, config, collectors, licensing, telemetryStore, orgGetter, zeus)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func newProvider(
|
||||
_ context.Context,
|
||||
providerSettings factory.ProviderSettings,
|
||||
config meterreporter.Config,
|
||||
collectors map[zeustypes.MeterName]metercollector.MeterCollector,
|
||||
licensing licensing.Licensing,
|
||||
telemetryStore telemetrystore.TelemetryStore,
|
||||
orgGetter organization.Getter,
|
||||
zeus zeus.Zeus,
|
||||
) (*Provider, error) {
|
||||
settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/ee/meterreporter/httpmeterreporter")
|
||||
|
||||
metrics, err := newReporterMetrics(settings.Meter())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
orderedCollectors, err := validateCollectors(collectors)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Provider{
|
||||
settings: settings,
|
||||
config: config,
|
||||
collectors: orderedCollectors,
|
||||
licensing: licensing,
|
||||
telemetryStore: telemetryStore,
|
||||
orgGetter: orgGetter,
|
||||
zeus: zeus,
|
||||
healthyC: make(chan struct{}),
|
||||
stopC: make(chan struct{}),
|
||||
metrics: metrics,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func validateCollectors(collectors map[zeustypes.MeterName]metercollector.MeterCollector) ([]metercollector.MeterCollector, error) {
|
||||
ordered := make([]metercollector.MeterCollector, 0, len(collectors))
|
||||
for name, collector := range collectors {
|
||||
if name.IsZero() {
|
||||
return nil, errors.New(errors.TypeInvalidInput, meterreporter.ErrCodeInvalidInput, "empty meter name in collector registry")
|
||||
}
|
||||
if collector == nil {
|
||||
return nil, errors.Newf(errors.TypeInvalidInput, meterreporter.ErrCodeInvalidInput, "nil collector for meter %q", name.String())
|
||||
}
|
||||
if collector.Name() != name {
|
||||
return nil, errors.Newf(errors.TypeInvalidInput, meterreporter.ErrCodeInvalidInput, "registry key %q does not match collector.Name() %q", name.String(), collector.Name().String())
|
||||
}
|
||||
if collector.Unit().IsZero() {
|
||||
return nil, errors.Newf(errors.TypeInvalidInput, meterreporter.ErrCodeInvalidInput, "meter %q has empty unit", name.String())
|
||||
}
|
||||
if collector.Aggregation().IsZero() {
|
||||
return nil, errors.Newf(errors.TypeInvalidInput, meterreporter.ErrCodeInvalidInput, "meter %q has empty aggregation", name.String())
|
||||
}
|
||||
ordered = append(ordered, collector)
|
||||
}
|
||||
|
||||
sort.Slice(ordered, func(i, j int) bool {
|
||||
return ordered[i].Name().String() < ordered[j].Name().String()
|
||||
})
|
||||
|
||||
return ordered, nil
|
||||
}
|
||||
|
||||
// Start runs an immediate tick, then repeats on Config.Interval.
|
||||
func (provider *Provider) Start(ctx context.Context) error {
|
||||
close(provider.healthyC)
|
||||
|
||||
provider.settings.Logger().InfoContext(ctx, "meter reporter started",
|
||||
slog.Duration("interval", provider.config.Interval),
|
||||
slog.Duration("timeout", provider.config.Timeout),
|
||||
slog.Int("catchup_max_days_per_tick", provider.config.CatchupMaxDaysPerTick),
|
||||
slog.Int("meters", len(provider.collectors)),
|
||||
)
|
||||
|
||||
provider.goroutinesWg.Add(1)
|
||||
go func() {
|
||||
defer provider.goroutinesWg.Done()
|
||||
|
||||
provider.runTick(ctx)
|
||||
|
||||
ticker := time.NewTicker(provider.config.Interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-provider.stopC:
|
||||
return
|
||||
case <-ticker.C:
|
||||
provider.runTick(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
provider.goroutinesWg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop signals the tick loop and waits for any in-flight tick.
|
||||
func (provider *Provider) Stop(ctx context.Context) error {
|
||||
<-provider.healthyC
|
||||
provider.settings.Logger().InfoContext(ctx, "meter reporter stopping")
|
||||
select {
|
||||
case <-provider.stopC:
|
||||
// already closed
|
||||
default:
|
||||
close(provider.stopC)
|
||||
}
|
||||
provider.goroutinesWg.Wait()
|
||||
provider.settings.Logger().InfoContext(ctx, "meter reporter stopped")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (provider *Provider) Healthy() <-chan struct{} {
|
||||
return provider.healthyC
|
||||
}
|
||||
|
||||
// runTick executes one collect-and-ship cycle under Config.Timeout.
|
||||
func (provider *Provider) runTick(parentCtx context.Context) {
|
||||
tickStart := time.Now()
|
||||
ctx, span := provider.settings.Tracer().Start(parentCtx, "meterreporter.Tick", trace.WithAttributes(
|
||||
attribute.String(attrMeterReporterProvider, providerName),
|
||||
attribute.Int("meterreporter.meters", len(provider.collectors)),
|
||||
attribute.Int("meterreporter.catchup_max_days_per_tick", provider.config.CatchupMaxDaysPerTick),
|
||||
))
|
||||
defer span.End()
|
||||
|
||||
provider.metrics.ticks.Add(ctx, 1)
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, provider.config.Timeout)
|
||||
defer cancel()
|
||||
|
||||
provider.settings.Logger().DebugContext(ctx, "meter reporter tick started",
|
||||
slog.Duration("timeout", provider.config.Timeout),
|
||||
slog.Int("meters", len(provider.collectors)),
|
||||
)
|
||||
|
||||
if err := provider.tick(ctx); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
span.SetAttributes(
|
||||
attribute.String(attrResult, resultFailure),
|
||||
attribute.Int64(attrDurationMs, time.Since(tickStart).Milliseconds()),
|
||||
)
|
||||
provider.settings.Logger().ErrorContext(ctx, "meter reporter tick failed",
|
||||
errors.Attr(err),
|
||||
slog.Duration("timeout", provider.config.Timeout),
|
||||
slog.Duration("duration", time.Since(tickStart)),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
span.SetAttributes(
|
||||
attribute.String(attrResult, resultSuccess),
|
||||
attribute.Int64(attrDurationMs, time.Since(tickStart).Milliseconds()),
|
||||
)
|
||||
provider.settings.Logger().DebugContext(ctx, "meter reporter tick completed", slog.Duration("duration", time.Since(tickStart)))
|
||||
}
|
||||
|
||||
// tick processes sealed catchup days, then today's partial window.
|
||||
func (provider *Provider) tick(ctx context.Context) error {
|
||||
now := time.Now().UTC()
|
||||
// Use one timestamp so a tick cannot straddle midnight.
|
||||
todayStart := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
||||
yesterday := todayStart.AddDate(0, 0, -1)
|
||||
|
||||
orgs, err := provider.orgGetter.ListByOwnedKeyRange(ctx)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, errCodeReportFailed, "failed to list organizations")
|
||||
}
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.Int(attrOrgCount, len(orgs)))
|
||||
if len(orgs) == 0 {
|
||||
provider.settings.Logger().InfoContext(ctx, "skipping meter reporter tick; no organizations found")
|
||||
return nil
|
||||
}
|
||||
org := orgs[0]
|
||||
if len(orgs) > 1 {
|
||||
// signoz_meter samples have no org marker.
|
||||
provider.settings.Logger().WarnContext(ctx, "multiple orgs on a single instance; reporting only the first",
|
||||
slog.Int("org_count", len(orgs)),
|
||||
slog.String("selected_org_id", org.ID.StringValue()),
|
||||
)
|
||||
}
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.String(attrOrgID, org.ID.StringValue()))
|
||||
|
||||
license, err := provider.licensing.GetActive(ctx, org.ID)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, errCodeReportFailed, "failed to fetch active license for org %q", org.ID.StringValue())
|
||||
}
|
||||
if license == nil || license.Key == "" {
|
||||
provider.settings.Logger().WarnContext(ctx, "skipping tick, nil/empty license for org", slog.String("org_id", org.ID.StringValue()))
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: re-enable once /v2/meters/checkpoints is live in staging. Until
|
||||
// then we run with an empty checkpoint map; bootstrap floors are taken
|
||||
// from data and dropCheckpointed becomes a no-op for the sealed window.
|
||||
// checkpoints, err := provider.zeus.GetMeterCheckpoints(ctx, license.Key)
|
||||
// if err != nil {
|
||||
// provider.metrics.checkpointErrors.Add(ctx, 1)
|
||||
// provider.settings.Logger().ErrorContext(ctx, "skipping tick: meter checkpoints call failed", errors.Attr(err))
|
||||
// return nil
|
||||
// }
|
||||
// checkpointsByMeter := make(map[string]time.Time, len(checkpoints))
|
||||
// for _, checkpoint := range checkpoints {
|
||||
// checkpointsByMeter[checkpoint.Name] = checkpoint.Checkpoint.UTC()
|
||||
// }
|
||||
checkpointsByMeter := make(map[string]time.Time)
|
||||
|
||||
floor := provider.dataFloor(ctx, todayStart)
|
||||
catchupStart := provider.catchupStart(floor, todayStart, checkpointsByMeter)
|
||||
end := catchupStart.AddDate(0, 0, provider.config.CatchupMaxDaysPerTick-1)
|
||||
if end.After(yesterday) {
|
||||
end = yesterday
|
||||
}
|
||||
trace.SpanFromContext(ctx).SetAttributes(
|
||||
attribute.String(attrCatchupStart, catchupStart.Format("2006-01-02")),
|
||||
attribute.String(attrCatchupEnd, end.Format("2006-01-02")),
|
||||
)
|
||||
provider.settings.Logger().DebugContext(ctx, "meter reporter catchup window selected",
|
||||
slog.String("org_id", org.ID.StringValue()),
|
||||
slog.Time("data_floor", floor),
|
||||
slog.Time("catchup_start", catchupStart),
|
||||
slog.Time("catchup_end", end),
|
||||
slog.Int("catchup_max_days_per_tick", provider.config.CatchupMaxDaysPerTick),
|
||||
)
|
||||
for day := catchupStart; !day.After(end); day = day.AddDate(0, 0, 1) {
|
||||
window, err := zeustypes.NewMeterWindow(day.UnixMilli(), day.AddDate(0, 0, 1).UnixMilli(), true)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, errCodeReportFailed, "build sealed meter window")
|
||||
}
|
||||
err = provider.runPhase(ctx, org.ID, license.Key, window, checkpointsByMeter)
|
||||
result := resultSuccess
|
||||
if err != nil {
|
||||
result = resultFailure
|
||||
}
|
||||
provider.metrics.catchupDaysProcessed.Add(ctx, 1, metric.WithAttributes(attribute.String(attrResult, result)))
|
||||
if err != nil {
|
||||
provider.settings.Logger().WarnContext(ctx, "stopping sealed catchup after failed day",
|
||||
errors.Attr(err),
|
||||
slog.String("date", day.Format("2006-01-02")),
|
||||
)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Today's partial window runs every tick.
|
||||
if now.UnixMilli() > todayStart.UnixMilli() {
|
||||
todayWindow, err := zeustypes.NewMeterWindow(todayStart.UnixMilli(), now.UnixMilli(), false)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, errors.TypeInternal, errCodeReportFailed, "build current-day meter window")
|
||||
}
|
||||
_ = provider.runPhase(ctx, org.ID, license.Key, todayWindow, checkpointsByMeter)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// runPhase collects all meters for one window and ships the batch.
|
||||
func (provider *Provider) runPhase(ctx context.Context, orgID valuer.UUID, licenseKey string, window zeustypes.MeterWindow, checkpointsByMeter map[string]time.Time) error {
|
||||
phaseLabel := phaseToday
|
||||
if window.IsCompleted {
|
||||
phaseLabel = phaseSealed
|
||||
}
|
||||
phaseAttr := metric.WithAttributes(attribute.String(attrPhase, phaseLabel))
|
||||
date := time.UnixMilli(window.StartUnixMilli).UTC().Format("2006-01-02")
|
||||
phaseStart := time.Now()
|
||||
ctx, span := provider.settings.Tracer().Start(ctx, "meterreporter.RunPhase", trace.WithAttributes(
|
||||
attribute.String(attrPhase, phaseLabel),
|
||||
attribute.String(attrOrgID, orgID.StringValue()),
|
||||
attribute.String(attrDate, date),
|
||||
attribute.Int64(attrWindowStartUnixMilli, window.StartUnixMilli),
|
||||
attribute.Int64(attrWindowEndUnixMilli, window.EndUnixMilli),
|
||||
attribute.Bool(attrWindowCompleted, window.IsCompleted),
|
||||
))
|
||||
defer span.End()
|
||||
|
||||
provider.settings.Logger().DebugContext(ctx, "meter reporter phase started",
|
||||
slog.String("org_id", orgID.StringValue()),
|
||||
slog.String("phase", phaseLabel),
|
||||
slog.String("date", date),
|
||||
slog.Int64("start_unix_milli", window.StartUnixMilli),
|
||||
slog.Int64("end_unix_milli", window.EndUnixMilli),
|
||||
slog.Int("meters", len(provider.collectors)),
|
||||
)
|
||||
|
||||
collectStart := time.Now()
|
||||
readings := make([]zeustypes.Meter, 0, len(provider.collectors))
|
||||
for _, collector := range provider.collectors {
|
||||
meterName := collector.Name().String()
|
||||
collectStart := time.Now()
|
||||
collectCtx, collectSpan := provider.settings.Tracer().Start(ctx, "meterreporter.CollectMeter", trace.WithAttributes(
|
||||
attribute.String(attrPhase, phaseLabel),
|
||||
attribute.String(attrOrgID, orgID.StringValue()),
|
||||
attribute.String(attrMeter, meterName),
|
||||
attribute.String(attrDate, date),
|
||||
attribute.Int64(attrWindowStartUnixMilli, window.StartUnixMilli),
|
||||
attribute.Int64(attrWindowEndUnixMilli, window.EndUnixMilli),
|
||||
attribute.Bool(attrWindowCompleted, window.IsCompleted),
|
||||
))
|
||||
collectedReadings, err := collector.Collect(collectCtx, orgID, window)
|
||||
if err != nil {
|
||||
collectSpan.RecordError(err)
|
||||
collectSpan.SetStatus(codes.Error, err.Error())
|
||||
collectSpan.SetAttributes(
|
||||
attribute.String(attrResult, resultFailure),
|
||||
attribute.Int64(attrDurationMs, time.Since(collectStart).Milliseconds()),
|
||||
)
|
||||
collectSpan.End()
|
||||
provider.metrics.collectErrors.Add(ctx, 1, phaseAttr)
|
||||
provider.settings.Logger().WarnContext(ctx, "meter collection failed",
|
||||
errors.Attr(err),
|
||||
slog.String("meter", meterName),
|
||||
slog.String("org_id", orgID.StringValue()),
|
||||
slog.String("phase", phaseLabel),
|
||||
slog.String("date", date),
|
||||
slog.Duration("duration", time.Since(collectStart)),
|
||||
)
|
||||
continue
|
||||
}
|
||||
collectSpan.SetAttributes(
|
||||
attribute.String(attrResult, resultSuccess),
|
||||
attribute.Int(attrReadings, len(collectedReadings)),
|
||||
attribute.Int64(attrDurationMs, time.Since(collectStart).Milliseconds()),
|
||||
)
|
||||
collectSpan.End()
|
||||
provider.settings.Logger().DebugContext(ctx, "meter collection completed",
|
||||
slog.String("meter", meterName),
|
||||
slog.String("org_id", orgID.StringValue()),
|
||||
slog.String("phase", phaseLabel),
|
||||
slog.String("date", date),
|
||||
slog.Int("readings", len(collectedReadings)),
|
||||
slog.Duration("duration", time.Since(collectStart)),
|
||||
)
|
||||
readings = append(readings, collectedReadings...)
|
||||
}
|
||||
collectDuration := time.Since(collectStart)
|
||||
provider.metrics.collectDuration.Add(ctx, collectDuration.Seconds(), phaseAttr)
|
||||
provider.metrics.collectOperations.Add(ctx, 1, phaseAttr)
|
||||
span.SetAttributes(attribute.Int(attrReadingsCollected, len(readings)))
|
||||
|
||||
if window.IsCompleted {
|
||||
beforeDrop := len(readings)
|
||||
readings = dropCheckpointed(readings, time.UnixMilli(window.StartUnixMilli).UTC(), checkpointsByMeter)
|
||||
dropped := beforeDrop - len(readings)
|
||||
span.SetAttributes(attribute.Int(attrReadingsDropped, dropped))
|
||||
if dropped > 0 {
|
||||
provider.settings.Logger().DebugContext(ctx, "dropped checkpointed meter readings",
|
||||
slog.String("org_id", orgID.StringValue()),
|
||||
slog.String("phase", phaseLabel),
|
||||
slog.String("date", date),
|
||||
slog.Int("dropped", dropped),
|
||||
slog.Int("remaining", len(readings)),
|
||||
)
|
||||
}
|
||||
}
|
||||
if len(readings) == 0 {
|
||||
span.SetAttributes(
|
||||
attribute.String(attrResult, resultSuccess),
|
||||
attribute.Int(attrReadings, 0),
|
||||
attribute.Int64(attrDurationMs, time.Since(phaseStart).Milliseconds()),
|
||||
)
|
||||
provider.settings.Logger().DebugContext(ctx, "meter reporter phase produced no readings",
|
||||
slog.String("org_id", orgID.StringValue()),
|
||||
slog.String("phase", phaseLabel),
|
||||
slog.String("date", date),
|
||||
slog.Duration("collect_duration", collectDuration),
|
||||
slog.Duration("duration", time.Since(phaseStart)),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
shipStart := time.Now()
|
||||
err := provider.shipReadings(ctx, licenseKey, date, readings)
|
||||
shipDuration := time.Since(shipStart)
|
||||
provider.metrics.shipDuration.Add(ctx, shipDuration.Seconds(), phaseAttr)
|
||||
provider.metrics.shipOperations.Add(ctx, 1, phaseAttr)
|
||||
if err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
span.SetAttributes(attribute.String(attrResult, resultFailure))
|
||||
provider.metrics.postErrors.Add(ctx, 1, phaseAttr)
|
||||
provider.settings.Logger().ErrorContext(ctx, "failed to ship meter readings",
|
||||
errors.Attr(err),
|
||||
slog.String("phase", phaseLabel),
|
||||
slog.String("date", date),
|
||||
slog.Int("readings", len(readings)),
|
||||
slog.Duration("ship_duration", shipDuration),
|
||||
)
|
||||
return err
|
||||
}
|
||||
provider.metrics.readingsEmitted.Add(ctx, int64(len(readings)), phaseAttr)
|
||||
span.SetAttributes(
|
||||
attribute.String(attrResult, resultSuccess),
|
||||
attribute.Int(attrReadings, len(readings)),
|
||||
attribute.Int64(attrDurationMs, time.Since(phaseStart).Milliseconds()),
|
||||
)
|
||||
provider.settings.Logger().InfoContext(ctx, "meter reporter phase shipped",
|
||||
slog.String("org_id", orgID.StringValue()),
|
||||
slog.String("phase", phaseLabel),
|
||||
slog.String("date", date),
|
||||
slog.Int("readings", len(readings)),
|
||||
slog.Duration("collect_duration", collectDuration),
|
||||
slog.Duration("ship_duration", shipDuration),
|
||||
slog.Duration("duration", time.Since(phaseStart)),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropCheckpointed removes readings already covered by meter checkpoints.
|
||||
func dropCheckpointed(readings []zeustypes.Meter, windowDay time.Time, checkpointsByMeter map[string]time.Time) []zeustypes.Meter {
|
||||
if len(checkpointsByMeter) == 0 {
|
||||
return readings
|
||||
}
|
||||
kept := readings[:0]
|
||||
for _, reading := range readings {
|
||||
checkpoint, ok := checkpointsByMeter[reading.MeterName]
|
||||
if !ok || checkpoint.Before(windowDay) {
|
||||
kept = append(kept, reading)
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// catchupStart returns the earliest UTC day that still needs sealed reporting.
|
||||
func (provider *Provider) catchupStart(floor time.Time, todayStart time.Time, checkpointsByMeter map[string]time.Time) time.Time {
|
||||
catchupStart := todayStart
|
||||
|
||||
for _, collector := range provider.collectors {
|
||||
next := floor
|
||||
if checkpoint, ok := checkpointsByMeter[collector.Name().String()]; ok {
|
||||
next = checkpoint.AddDate(0, 0, 1)
|
||||
if next.Before(floor) {
|
||||
next = floor
|
||||
}
|
||||
}
|
||||
if next.Before(catchupStart) {
|
||||
catchupStart = next
|
||||
}
|
||||
}
|
||||
|
||||
yesterday := todayStart.AddDate(0, 0, -1)
|
||||
if catchupStart.After(yesterday) {
|
||||
catchupStart = yesterday
|
||||
}
|
||||
|
||||
return catchupStart
|
||||
}
|
||||
|
||||
// dataFloor returns the earliest signoz_meter sample day, or today on failure.
|
||||
func (provider *Provider) dataFloor(ctx context.Context, todayStart time.Time) time.Time {
|
||||
ctx, span := provider.settings.Tracer().Start(ctx, "meterreporter.DataFloor")
|
||||
defer span.End()
|
||||
|
||||
if provider.telemetryStore == nil {
|
||||
span.SetAttributes(attribute.String(attrResult, resultSuccess))
|
||||
return todayStart
|
||||
}
|
||||
|
||||
sb := sqlbuilder.NewSelectBuilder()
|
||||
sb.Select("ifNull(min(unix_milli), 0)")
|
||||
sb.From(telemetrymeter.DBName + "." + telemetrymeter.SamplesTableName)
|
||||
query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
|
||||
|
||||
var minMs int64
|
||||
if err := provider.telemetryStore.ClickhouseDB().QueryRow(ctx, query, args...).Scan(&minMs); err != nil {
|
||||
span.RecordError(err)
|
||||
span.SetStatus(codes.Error, err.Error())
|
||||
span.SetAttributes(attribute.String(attrResult, resultFailure))
|
||||
provider.settings.Logger().WarnContext(ctx, "failed to read data floor; falling back to latest sealed day", errors.Attr(err))
|
||||
return todayStart
|
||||
}
|
||||
if minMs == 0 {
|
||||
span.SetAttributes(
|
||||
attribute.String(attrResult, resultSuccess),
|
||||
attribute.Int64("meterreporter.data_floor_unix_milli", 0),
|
||||
)
|
||||
return todayStart
|
||||
}
|
||||
|
||||
minDay := time.UnixMilli(minMs).UTC()
|
||||
floor := time.Date(minDay.Year(), minDay.Month(), minDay.Day(), 0, 0, 0, 0, time.UTC)
|
||||
span.SetAttributes(
|
||||
attribute.String(attrResult, resultSuccess),
|
||||
attribute.Int64("meterreporter.data_floor_unix_milli", floor.UnixMilli()),
|
||||
)
|
||||
provider.settings.Logger().DebugContext(ctx, "meter reporter data floor loaded", slog.Time("data_floor", floor))
|
||||
return floor
|
||||
}
|
||||
|
||||
// shipReadings sends one day's meter batch to Zeus.
|
||||
func (provider *Provider) shipReadings(ctx context.Context, licenseKey string, date string, readings []zeustypes.Meter) error {
|
||||
idempotencyKey := fmt.Sprintf("meter-cron:%s", date)
|
||||
ctx, span := provider.settings.Tracer().Start(ctx, "meterreporter.ShipReadings", trace.WithAttributes(
|
||||
attribute.String(attrDate, date),
|
||||
attribute.Int(attrReadings, len(readings)),
|
||||
attribute.String(attrIdempotencyKey, idempotencyKey),
|
||||
attribute.Bool(attrDryRun, true),
|
||||
))
|
||||
defer span.End()
|
||||
|
||||
provider.settings.Logger().InfoContext(ctx, "meter readings prepared for shipment",
|
||||
slog.String("date", date),
|
||||
slog.Int("readings", len(readings)),
|
||||
slog.String("idempotency_key", idempotencyKey),
|
||||
slog.Bool("dry_run", true),
|
||||
)
|
||||
|
||||
// Temporary visibility while /v2/meters is offline.
|
||||
for _, reading := range readings {
|
||||
provider.settings.Logger().InfoContext(ctx, "meter reading prepared for shipment",
|
||||
slog.String("meter", reading.MeterName),
|
||||
slog.Float64("value", reading.Value),
|
||||
slog.String("unit", reading.Unit.StringValue()),
|
||||
slog.String("aggregation", reading.Aggregation.StringValue()),
|
||||
slog.Int64("start_unix_milli", reading.StartUnixMilli),
|
||||
slog.Int64("end_unix_milli", reading.EndUnixMilli),
|
||||
slog.Bool("is_completed", reading.IsCompleted),
|
||||
slog.Any("dimensions", reading.Dimensions),
|
||||
slog.String("idempotency_key", idempotencyKey),
|
||||
)
|
||||
}
|
||||
|
||||
// TODO: re-enable once /v2/meters is live in staging.
|
||||
// body, err := json.Marshal(zeustypes.PostableMeters{Meters: readings})
|
||||
// if err != nil {
|
||||
// return errors.Wrapf(err, errors.TypeInternal, errCodeReportFailed, "marshal meter readings for %s", date)
|
||||
// }
|
||||
// if err := provider.zeus.PutMetersV3(ctx, licenseKey, idempotencyKey, body); err != nil {
|
||||
// return errors.Wrapf(err, errors.TypeInternal, errCodeReportFailed, "ship meter readings for %s", date)
|
||||
// }
|
||||
_ = licenseKey
|
||||
span.SetAttributes(attribute.String(attrResult, resultSuccess))
|
||||
return nil
|
||||
}
|
||||
103
ee/meterreporter/httpmeterreporter/provider_internal_test.go
Normal file
103
ee/meterreporter/httpmeterreporter/provider_internal_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package httpmeterreporter
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/metercollector"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestValidateCollectorsRejectsBadRegistry(t *testing.T) {
|
||||
meterA := zeustypes.MustNewMeterName("signoz.test.a")
|
||||
meterB := zeustypes.MustNewMeterName("signoz.test.b")
|
||||
|
||||
t.Run("key name mismatch", func(t *testing.T) {
|
||||
_, err := validateCollectors(map[zeustypes.MeterName]metercollector.MeterCollector{
|
||||
meterA: testCollector{name: meterB},
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("nil collector", func(t *testing.T) {
|
||||
_, err := validateCollectors(map[zeustypes.MeterName]metercollector.MeterCollector{
|
||||
meterA: nil,
|
||||
})
|
||||
require.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDropCheckpointed(t *testing.T) {
|
||||
meterA := zeustypes.MustNewMeterName("signoz.test.a")
|
||||
meterB := zeustypes.MustNewMeterName("signoz.test.b")
|
||||
meterC := zeustypes.MustNewMeterName("signoz.test.c")
|
||||
windowDay := time.Date(2026, 5, 4, 0, 0, 0, 0, time.UTC)
|
||||
window := zeustypes.MustNewMeterWindow(windowDay.UnixMilli(), windowDay.AddDate(0, 0, 1).UnixMilli(), true)
|
||||
readings := []zeustypes.Meter{
|
||||
zeustypes.NewMeter(meterA, 0, zeustypes.MeterUnitCount, zeustypes.MeterAggregationSum, window, nil),
|
||||
zeustypes.NewMeter(meterB, 0, zeustypes.MeterUnitCount, zeustypes.MeterAggregationSum, window, nil),
|
||||
zeustypes.NewMeter(meterC, 0, zeustypes.MeterUnitCount, zeustypes.MeterAggregationSum, window, nil),
|
||||
}
|
||||
|
||||
kept := dropCheckpointed(readings, windowDay, map[string]time.Time{
|
||||
meterA.String(): windowDay,
|
||||
meterB.String(): windowDay.AddDate(0, 0, -1),
|
||||
})
|
||||
|
||||
require.Equal(t, []zeustypes.Meter{
|
||||
zeustypes.NewMeter(meterB, 0, zeustypes.MeterUnitCount, zeustypes.MeterAggregationSum, window, nil),
|
||||
zeustypes.NewMeter(meterC, 0, zeustypes.MeterUnitCount, zeustypes.MeterAggregationSum, window, nil),
|
||||
}, kept)
|
||||
}
|
||||
|
||||
func TestCatchupStart(t *testing.T) {
|
||||
meterA := zeustypes.MustNewMeterName("signoz.test.a")
|
||||
floor := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC)
|
||||
todayStart := time.Date(2026, 5, 5, 0, 0, 0, 0, time.UTC)
|
||||
provider := &Provider{
|
||||
collectors: []metercollector.MeterCollector{
|
||||
testCollector{name: meterA},
|
||||
},
|
||||
}
|
||||
|
||||
t.Run("no checkpoint starts at floor", func(t *testing.T) {
|
||||
require.Equal(t, floor, provider.catchupStart(floor, todayStart, nil))
|
||||
})
|
||||
|
||||
t.Run("checkpoint advances by one day", func(t *testing.T) {
|
||||
require.Equal(t, floor.AddDate(0, 0, 2), provider.catchupStart(floor, todayStart, map[string]time.Time{
|
||||
meterA.String(): floor.AddDate(0, 0, 1),
|
||||
}))
|
||||
})
|
||||
}
|
||||
|
||||
type testCollector struct {
|
||||
name zeustypes.MeterName
|
||||
unit zeustypes.MeterUnit
|
||||
aggregation zeustypes.MeterAggregation
|
||||
}
|
||||
|
||||
func (c testCollector) Name() zeustypes.MeterName {
|
||||
return c.name
|
||||
}
|
||||
|
||||
func (c testCollector) Unit() zeustypes.MeterUnit {
|
||||
if c.unit.IsZero() {
|
||||
return zeustypes.MeterUnitCount
|
||||
}
|
||||
return c.unit
|
||||
}
|
||||
|
||||
func (c testCollector) Aggregation() zeustypes.MeterAggregation {
|
||||
if c.aggregation.IsZero() {
|
||||
return zeustypes.MeterAggregationSum
|
||||
}
|
||||
return c.aggregation
|
||||
}
|
||||
|
||||
func (c testCollector) Collect(context.Context, valuer.UUID, zeustypes.MeterWindow) ([]zeustypes.Meter, error) {
|
||||
return nil, nil
|
||||
}
|
||||
90
ee/meterreporter/httpmeterreporter/telemetry.go
Normal file
90
ee/meterreporter/httpmeterreporter/telemetry.go
Normal file
@@ -0,0 +1,90 @@
|
||||
package httpmeterreporter
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
type reporterMetrics struct {
|
||||
ticks metric.Int64Counter
|
||||
readingsEmitted metric.Int64Counter
|
||||
collectErrors metric.Int64Counter
|
||||
postErrors metric.Int64Counter
|
||||
checkpointErrors metric.Int64Counter
|
||||
catchupDaysProcessed metric.Int64Counter
|
||||
collectDuration metric.Float64Counter
|
||||
collectOperations metric.Int64Counter
|
||||
shipDuration metric.Float64Counter
|
||||
shipOperations metric.Int64Counter
|
||||
}
|
||||
|
||||
func newReporterMetrics(meter metric.Meter) (*reporterMetrics, error) {
|
||||
var errs error
|
||||
|
||||
ticks, err := meter.Int64Counter("signoz.meterreporter.ticks", metric.WithDescription("Meter reporter ticks."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
readingsEmitted, err := meter.Int64Counter("signoz.meterreporter.readings.emitted", metric.WithDescription("Meter readings shipped to Zeus."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
collectErrors, err := meter.Int64Counter("signoz.meterreporter.collect.errors", metric.WithDescription("Meter collection errors."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
postErrors, err := meter.Int64Counter("signoz.meterreporter.post.errors", metric.WithDescription("Zeus POST failures."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
checkpointErrors, err := meter.Int64Counter("signoz.meterreporter.checkpoint.errors", metric.WithDescription("Zeus checkpoint read failures."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
catchupDaysProcessed, err := meter.Int64Counter("signoz.meterreporter.catchup.days_processed", metric.WithDescription("Sealed catchup days processed."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
collectDuration, err := meter.Float64Counter("signoz.meterreporter.collect.duration.seconds", metric.WithDescription("Cumulative collection duration."), metric.WithUnit("s"))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
collectOperations, err := meter.Int64Counter("signoz.meterreporter.collect.operations", metric.WithDescription("Collection phases measured."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
shipDuration, err := meter.Float64Counter("signoz.meterreporter.ship.duration.seconds", metric.WithDescription("Cumulative ship duration."), metric.WithUnit("s"))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
shipOperations, err := meter.Int64Counter("signoz.meterreporter.ship.operations", metric.WithDescription("Ship phases measured."))
|
||||
if err != nil {
|
||||
errs = errors.Join(errs, err)
|
||||
}
|
||||
|
||||
if errs != nil {
|
||||
return nil, errs
|
||||
}
|
||||
|
||||
return &reporterMetrics{
|
||||
ticks: ticks,
|
||||
readingsEmitted: readingsEmitted,
|
||||
collectErrors: collectErrors,
|
||||
postErrors: postErrors,
|
||||
checkpointErrors: checkpointErrors,
|
||||
catchupDaysProcessed: catchupDaysProcessed,
|
||||
collectDuration: collectDuration,
|
||||
collectOperations: collectOperations,
|
||||
shipDuration: shipDuration,
|
||||
shipOperations: shipOperations,
|
||||
}, nil
|
||||
}
|
||||
@@ -14,7 +14,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/querier"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
@@ -88,7 +88,7 @@ func (module *module) GetDashboardByPublicID(ctx context.Context, id valuer.UUID
|
||||
return dashboardtypes.NewDashboardFromStorableDashboard(storableDashboard), nil
|
||||
}
|
||||
|
||||
func (module *module) GetPublicDashboardSelectorsAndOrg(ctx context.Context, id valuer.UUID, orgs []*types.Organization) ([]authtypes.Selector, valuer.UUID, error) {
|
||||
func (module *module) GetPublicDashboardSelectorsAndOrg(ctx context.Context, id valuer.UUID, orgs []*types.Organization) ([]coretypes.Selector, valuer.UUID, error) {
|
||||
orgIDs := make([]string, len(orgs))
|
||||
for idx, org := range orgs {
|
||||
orgIDs[idx] = org.ID.StringValue()
|
||||
@@ -99,9 +99,9 @@ func (module *module) GetPublicDashboardSelectorsAndOrg(ctx context.Context, id
|
||||
return nil, valuer.UUID{}, err
|
||||
}
|
||||
|
||||
return []authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeMetaResource, id.StringValue()),
|
||||
authtypes.MustNewSelector(authtypes.TypeMetaResource, authtypes.WildCardSelectorString),
|
||||
return []coretypes.Selector{
|
||||
coretypes.TypeMetaResource.MustSelector(id.StringValue()),
|
||||
coretypes.TypeMetaResource.MustSelector(coretypes.WildCardSelectorString),
|
||||
}, storableDashboard.OrgID, nil
|
||||
}
|
||||
|
||||
@@ -217,28 +217,6 @@ func (module *module) LockUnlock(ctx context.Context, orgID valuer.UUID, id valu
|
||||
return module.pkgDashboardModule.LockUnlock(ctx, orgID, id, updatedBy, isAdmin, lock)
|
||||
}
|
||||
|
||||
func (module *module) MustGetTypeables() []authtypes.Typeable {
|
||||
return module.pkgDashboardModule.MustGetTypeables()
|
||||
}
|
||||
|
||||
func (module *module) MustGetManagedRoleTransactions() map[string][]*authtypes.Transaction {
|
||||
return map[string][]*authtypes.Transaction{
|
||||
authtypes.SigNozAnonymousRoleName: {
|
||||
{
|
||||
ID: valuer.GenerateUUID(),
|
||||
Relation: authtypes.RelationRead,
|
||||
Object: *authtypes.MustNewObject(
|
||||
authtypes.Resource{
|
||||
Type: authtypes.TypeMetaResource,
|
||||
Name: dashboardtypes.TypeableMetaResourcePublicDashboard.Name(),
|
||||
},
|
||||
authtypes.MustNewSelector(authtypes.TypeMetaResource, "*"),
|
||||
),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (module *module) deletePublic(ctx context.Context, _ valuer.UUID, dashboardID valuer.UUID) error {
|
||||
return module.store.DeletePublic(ctx, dashboardID.StringValue())
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/ee/query-service/anomaly"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/http/render"
|
||||
@@ -15,7 +17,6 @@ import (
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
func (aH *APIHandler) queryRangeV4(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -137,4 +138,3 @@ func (aH *APIHandler) queryRangeV4(w http.ResponseWriter, r *http.Request) {
|
||||
aH.QueryRangeV4(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -42,8 +42,8 @@ import (
|
||||
|
||||
// Server runs HTTP, Mux and a grpc server
|
||||
type Server struct {
|
||||
config signoz.Config
|
||||
signoz *signoz.SigNoz
|
||||
config signoz.Config
|
||||
signoz *signoz.SigNoz
|
||||
|
||||
// public http router
|
||||
httpConn net.Listener
|
||||
@@ -148,7 +148,7 @@ func NewServer(config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
|
||||
|
||||
s := &Server{
|
||||
config: config,
|
||||
signoz: signoz,
|
||||
signoz: signoz,
|
||||
httpHostPort: baseconst.HTTPHostPort,
|
||||
unavailableChannel: make(chan healthcheck.Status),
|
||||
usageManager: usageManager,
|
||||
@@ -317,4 +317,3 @@ func (s *Server) Stop(ctx context.Context) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -150,6 +150,72 @@ func (provider *Provider) PutMetersV2(ctx context.Context, key string, data []by
|
||||
return err
|
||||
}
|
||||
|
||||
func (provider *Provider) PutMetersV3(ctx context.Context, key string, idempotencyKey string, data []byte) error {
|
||||
headers := http.Header{}
|
||||
if idempotencyKey != "" {
|
||||
headers.Set("X-Idempotency-Key", idempotencyKey)
|
||||
}
|
||||
|
||||
_, err := provider.doWithHeaders(
|
||||
ctx,
|
||||
provider.config.URL.JoinPath("/v2/meters"),
|
||||
http.MethodPost,
|
||||
key,
|
||||
data,
|
||||
headers,
|
||||
)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (provider *Provider) GetMeterCheckpoints(ctx context.Context, key string) ([]zeustypes.MeterCheckpoint, error) {
|
||||
response, err := provider.do(
|
||||
ctx,
|
||||
provider.config.URL.JoinPath("/v2/meters/checkpoints"),
|
||||
http.MethodGet,
|
||||
key,
|
||||
nil,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
checkpointValues := gjson.GetBytes(response, "data.checkpoints")
|
||||
if !checkpointValues.Exists() || checkpointValues.Type == gjson.Null {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeus.ErrCodeResponseMalformed, "meter checkpoints are required")
|
||||
}
|
||||
|
||||
if !checkpointValues.IsArray() {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeus.ErrCodeResponseMalformed, "meter checkpoints must be an array")
|
||||
}
|
||||
|
||||
checkpointResults := checkpointValues.Array()
|
||||
checkpoints := make([]zeustypes.MeterCheckpoint, 0, len(checkpointResults))
|
||||
for _, checkpointValue := range checkpointResults {
|
||||
name := checkpointValue.Get("name").String()
|
||||
if name == "" {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeus.ErrCodeResponseMalformed, "meter checkpoint name is required")
|
||||
}
|
||||
|
||||
checkpointString := checkpointValue.Get("checkpoint").String()
|
||||
if checkpointString == "" {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeus.ErrCodeResponseMalformed, "meter checkpoint is required for %q", name)
|
||||
}
|
||||
|
||||
checkpoint, err := time.Parse("2006-01-02", checkpointString)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeus.ErrCodeResponseMalformed, "parse meter checkpoint %q for %q", checkpointString, name)
|
||||
}
|
||||
|
||||
checkpoints = append(checkpoints, zeustypes.MeterCheckpoint{
|
||||
Name: name,
|
||||
Checkpoint: checkpoint,
|
||||
})
|
||||
}
|
||||
|
||||
return checkpoints, nil
|
||||
}
|
||||
|
||||
func (provider *Provider) PutProfile(ctx context.Context, key string, profile *zeustypes.PostableProfile) error {
|
||||
body, err := json.Marshal(profile)
|
||||
if err != nil {
|
||||
@@ -185,12 +251,21 @@ func (provider *Provider) PutHost(ctx context.Context, key string, host *zeustyp
|
||||
}
|
||||
|
||||
func (provider *Provider) do(ctx context.Context, url *url.URL, method string, key string, requestBody []byte) ([]byte, error) {
|
||||
return provider.doWithHeaders(ctx, url, method, key, requestBody, nil)
|
||||
}
|
||||
|
||||
func (provider *Provider) doWithHeaders(ctx context.Context, url *url.URL, method string, key string, requestBody []byte, extraHeaders http.Header) ([]byte, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, method, url.String(), bytes.NewBuffer(requestBody))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
request.Header.Set("X-Signoz-Cloud-Api-Key", key)
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
for k, vs := range extraHeaders {
|
||||
for _, v := range vs {
|
||||
request.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
response, err := provider.httpClient.Do(request)
|
||||
if err != nil {
|
||||
|
||||
@@ -23,6 +23,11 @@
|
||||
"**/*.md",
|
||||
"**/*.json",
|
||||
"src/parser/**",
|
||||
"src/TraceOperator/parser/**"
|
||||
"src/TraceOperator/parser/**",
|
||||
".claude",
|
||||
".opencode",
|
||||
"dist",
|
||||
"playwright-report",
|
||||
".temp_cache"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -23,8 +23,7 @@
|
||||
"commitlint": "commitlint --edit $1",
|
||||
"test": "jest",
|
||||
"test:changedsince": "jest --changedSince=main --coverage --silent",
|
||||
"generate:api": "orval --config ./orval.config.ts && sh scripts/post-types-generation.sh",
|
||||
"generate:permissions-type": "node scripts/generate-permissions-type.cjs"
|
||||
"generate:api": "orval --config ./orval.config.ts && sh scripts/post-types-generation.sh"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execSync } = require('child_process');
|
||||
const axios = require('axios');
|
||||
|
||||
const PERMISSIONS_TYPE_FILE = path.join(
|
||||
__dirname,
|
||||
'../src/hooks/useAuthZ/permissions.type.ts',
|
||||
);
|
||||
|
||||
const SIGNOZ_INTEGRATION_IMAGE = 'signoz:integration';
|
||||
const LOCAL_BACKEND_URL = 'http://localhost:8080';
|
||||
|
||||
function log(message) {
|
||||
console.log(`[generate-permissions-type] ${message}`);
|
||||
}
|
||||
|
||||
function getBackendUrlFromDocker() {
|
||||
try {
|
||||
const output = execSync(
|
||||
`docker ps --filter "ancestor=${SIGNOZ_INTEGRATION_IMAGE}" --format "{{.Ports}}"`,
|
||||
{ encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
).trim();
|
||||
|
||||
if (!output) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const portMatch = output.match(/0\.0\.0\.0:(\d+)->8080\/tcp/);
|
||||
if (portMatch) {
|
||||
return `http://localhost:${portMatch[1]}`;
|
||||
}
|
||||
|
||||
const ipv6Match = output.match(/:::(\d+)->8080\/tcp/);
|
||||
if (ipv6Match) {
|
||||
return `http://localhost:${ipv6Match[1]}`;
|
||||
}
|
||||
} catch (err) {
|
||||
log(`Warning: Could not get port from docker: ${err.message}`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function checkBackendHealth(url, maxAttempts = 3, delayMs = 1000) {
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
try {
|
||||
await axios.get(`${url}/api/v1/health`, {
|
||||
timeout: 5000,
|
||||
validateStatus: (status) => status === 200,
|
||||
});
|
||||
return true;
|
||||
} catch (err) {
|
||||
if (attempt < maxAttempts) {
|
||||
await new Promise((r) => setTimeout(r, delayMs));
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function discoverBackendUrl() {
|
||||
const dockerUrl = getBackendUrlFromDocker();
|
||||
if (dockerUrl) {
|
||||
log(`Found ${SIGNOZ_INTEGRATION_IMAGE} container, trying ${dockerUrl}...`);
|
||||
if (await checkBackendHealth(dockerUrl)) {
|
||||
log(`Backend found at ${dockerUrl} (from py-test-setup)`);
|
||||
return dockerUrl;
|
||||
}
|
||||
log(`Backend at ${dockerUrl} is not responding`);
|
||||
}
|
||||
|
||||
log(`Trying local backend at ${LOCAL_BACKEND_URL}...`);
|
||||
if (await checkBackendHealth(LOCAL_BACKEND_URL)) {
|
||||
log(`Backend found at ${LOCAL_BACKEND_URL}`);
|
||||
return LOCAL_BACKEND_URL;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchResources(backendUrl) {
|
||||
log('Fetching resources from API...');
|
||||
const resourcesUrl = `${backendUrl}/api/v1/authz/resources`;
|
||||
|
||||
const { data: response } = await axios.get(resourcesUrl);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
function transformResponse(apiResponse) {
|
||||
if (!apiResponse.data) {
|
||||
throw new Error('Invalid API response: missing data field');
|
||||
}
|
||||
|
||||
const { resources, relations } = apiResponse.data;
|
||||
|
||||
return {
|
||||
status: apiResponse.status || 'success',
|
||||
data: {
|
||||
resources: resources,
|
||||
relations: relations,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function generateTypeScriptFile(data) {
|
||||
const resourcesStr = data.data.resources
|
||||
.map(
|
||||
(r) =>
|
||||
`\t\t\t{\n\t\t\t\tname: '${r.name}',\n\t\t\t\ttype: '${r.type}',\n\t\t\t},`,
|
||||
)
|
||||
.join('\n');
|
||||
|
||||
const relationsStr = Object.entries(data.data.relations)
|
||||
.map(
|
||||
([type, relations]) =>
|
||||
`\t\t\t${type}: [${relations.map((r) => `'${r}'`).join(', ')}],`,
|
||||
)
|
||||
.join('\n');
|
||||
|
||||
return `// AUTO GENERATED FILE - DO NOT EDIT - GENERATED BY scripts/generate-permissions-type
|
||||
export default {
|
||||
\tstatus: '${data.status}',
|
||||
\tdata: {
|
||||
\t\tresources: [
|
||||
${resourcesStr}
|
||||
\t\t],
|
||||
\t\trelations: {
|
||||
${relationsStr}
|
||||
\t\t},
|
||||
\t},
|
||||
} as const;
|
||||
`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
try {
|
||||
log('Starting permissions type generation...');
|
||||
|
||||
const backendUrl = await discoverBackendUrl();
|
||||
|
||||
if (!backendUrl) {
|
||||
console.error('\n' + '='.repeat(80));
|
||||
console.error('ERROR: No running SigNoz backend found!');
|
||||
console.error('='.repeat(80));
|
||||
console.error(
|
||||
'\nThe permissions type generator requires a running SigNoz backend.',
|
||||
);
|
||||
console.error('\nFor local development, start the backend with:');
|
||||
console.error(' make go-run-enterprise');
|
||||
console.error(
|
||||
'\nFor CI or integration testing, start the test environment with:',
|
||||
);
|
||||
console.error(' make py-test-setup');
|
||||
console.error(
|
||||
'\nIf running in CI and seeing this error, check that the py-test-setup',
|
||||
);
|
||||
console.error('step completed successfully before this step runs.');
|
||||
console.error('='.repeat(80) + '\n');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
log('Fetching resources...');
|
||||
const apiResponse = await fetchResources(backendUrl);
|
||||
|
||||
log('Transforming response...');
|
||||
const transformed = transformResponse(apiResponse);
|
||||
|
||||
log('Generating TypeScript file...');
|
||||
const content = generateTypeScriptFile(transformed);
|
||||
|
||||
log(`Writing to ${PERMISSIONS_TYPE_FILE}...`);
|
||||
fs.writeFileSync(PERMISSIONS_TYPE_FILE, content, 'utf8');
|
||||
|
||||
const rootDir = path.join(__dirname, '../..');
|
||||
const relativePath = path.relative(
|
||||
path.join(rootDir, 'frontend'),
|
||||
PERMISSIONS_TYPE_FILE,
|
||||
);
|
||||
log('Linting generated file...');
|
||||
execSync(`cd frontend && yarn oxlint ${relativePath}`, {
|
||||
cwd: rootDir,
|
||||
stdio: 'inherit',
|
||||
});
|
||||
|
||||
log('Successfully generated permissions.type.ts');
|
||||
} catch (error) {
|
||||
log(`Error: ${error.message}`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
|
||||
module.exports = { main };
|
||||
@@ -4,23 +4,16 @@
|
||||
* * regenerate with 'yarn generate:api'
|
||||
* SigNoz
|
||||
*/
|
||||
import { useMutation, useQuery } from 'react-query';
|
||||
import { useMutation } from 'react-query';
|
||||
import type {
|
||||
InvalidateOptions,
|
||||
MutationFunction,
|
||||
QueryClient,
|
||||
QueryFunction,
|
||||
QueryKey,
|
||||
UseMutationOptions,
|
||||
UseMutationResult,
|
||||
UseQueryOptions,
|
||||
UseQueryResult,
|
||||
} from 'react-query';
|
||||
|
||||
import type {
|
||||
AuthtypesTransactionDTO,
|
||||
AuthzCheck200,
|
||||
AuthzResources200,
|
||||
RenderErrorResponseDTO,
|
||||
} from '../sigNoz.schemas';
|
||||
|
||||
@@ -110,88 +103,3 @@ export const useAuthzCheck = <
|
||||
|
||||
return useMutation(mutationOptions);
|
||||
};
|
||||
/**
|
||||
* Gets all the available resources
|
||||
* @summary Get resources
|
||||
*/
|
||||
export const authzResources = (signal?: AbortSignal) => {
|
||||
return GeneratedAPIInstance<AuthzResources200>({
|
||||
url: `/api/v1/authz/resources`,
|
||||
method: 'GET',
|
||||
signal,
|
||||
});
|
||||
};
|
||||
|
||||
export const getAuthzResourcesQueryKey = () => {
|
||||
return [`/api/v1/authz/resources`] as const;
|
||||
};
|
||||
|
||||
export const getAuthzResourcesQueryOptions = <
|
||||
TData = Awaited<ReturnType<typeof authzResources>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof authzResources>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
}) => {
|
||||
const { query: queryOptions } = options ?? {};
|
||||
|
||||
const queryKey = queryOptions?.queryKey ?? getAuthzResourcesQueryKey();
|
||||
|
||||
const queryFn: QueryFunction<Awaited<ReturnType<typeof authzResources>>> = ({
|
||||
signal,
|
||||
}) => authzResources(signal);
|
||||
|
||||
return { queryKey, queryFn, ...queryOptions } as UseQueryOptions<
|
||||
Awaited<ReturnType<typeof authzResources>>,
|
||||
TError,
|
||||
TData
|
||||
> & { queryKey: QueryKey };
|
||||
};
|
||||
|
||||
export type AuthzResourcesQueryResult = NonNullable<
|
||||
Awaited<ReturnType<typeof authzResources>>
|
||||
>;
|
||||
export type AuthzResourcesQueryError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
* @summary Get resources
|
||||
*/
|
||||
|
||||
export function useAuthzResources<
|
||||
TData = Awaited<ReturnType<typeof authzResources>>,
|
||||
TError = ErrorType<RenderErrorResponseDTO>,
|
||||
>(options?: {
|
||||
query?: UseQueryOptions<
|
||||
Awaited<ReturnType<typeof authzResources>>,
|
||||
TError,
|
||||
TData
|
||||
>;
|
||||
}): UseQueryResult<TData, TError> & { queryKey: QueryKey } {
|
||||
const queryOptions = getAuthzResourcesQueryOptions(options);
|
||||
|
||||
const query = useQuery(queryOptions) as UseQueryResult<TData, TError> & {
|
||||
queryKey: QueryKey;
|
||||
};
|
||||
|
||||
query.queryKey = queryOptions.queryKey;
|
||||
|
||||
return query;
|
||||
}
|
||||
|
||||
/**
|
||||
* @summary Get resources
|
||||
*/
|
||||
export const invalidateAuthzResources = async (
|
||||
queryClient: QueryClient,
|
||||
options?: InvalidateOptions,
|
||||
): Promise<QueryClient> => {
|
||||
await queryClient.invalidateQueries(
|
||||
{ queryKey: getAuthzResourcesQueryKey() },
|
||||
options,
|
||||
);
|
||||
|
||||
return queryClient;
|
||||
};
|
||||
|
||||
@@ -18,9 +18,9 @@ import type {
|
||||
} from 'react-query';
|
||||
|
||||
import type {
|
||||
AuthtypesPatchableObjectsDTO,
|
||||
AuthtypesPatchableRoleDTO,
|
||||
AuthtypesPostableRoleDTO,
|
||||
CoretypesPatchableObjectsDTO,
|
||||
CreateRole201,
|
||||
DeleteRolePathParameters,
|
||||
GetObjects200,
|
||||
@@ -571,13 +571,13 @@ export const invalidateGetObjects = async (
|
||||
*/
|
||||
export const patchObjects = (
|
||||
{ id, relation }: PatchObjectsPathParameters,
|
||||
authtypesPatchableObjectsDTO: BodyType<AuthtypesPatchableObjectsDTO>,
|
||||
coretypesPatchableObjectsDTO: BodyType<CoretypesPatchableObjectsDTO>,
|
||||
) => {
|
||||
return GeneratedAPIInstance<string>({
|
||||
url: `/api/v1/roles/${id}/relations/${relation}/objects`,
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
data: authtypesPatchableObjectsDTO,
|
||||
data: coretypesPatchableObjectsDTO,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -590,7 +590,7 @@ export const getPatchObjectsMutationOptions = <
|
||||
TError,
|
||||
{
|
||||
pathParams: PatchObjectsPathParameters;
|
||||
data: BodyType<AuthtypesPatchableObjectsDTO>;
|
||||
data: BodyType<CoretypesPatchableObjectsDTO>;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
@@ -599,7 +599,7 @@ export const getPatchObjectsMutationOptions = <
|
||||
TError,
|
||||
{
|
||||
pathParams: PatchObjectsPathParameters;
|
||||
data: BodyType<AuthtypesPatchableObjectsDTO>;
|
||||
data: BodyType<CoretypesPatchableObjectsDTO>;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
@@ -616,7 +616,7 @@ export const getPatchObjectsMutationOptions = <
|
||||
Awaited<ReturnType<typeof patchObjects>>,
|
||||
{
|
||||
pathParams: PatchObjectsPathParameters;
|
||||
data: BodyType<AuthtypesPatchableObjectsDTO>;
|
||||
data: BodyType<CoretypesPatchableObjectsDTO>;
|
||||
}
|
||||
> = (props) => {
|
||||
const { pathParams, data } = props ?? {};
|
||||
@@ -630,7 +630,7 @@ export const getPatchObjectsMutationOptions = <
|
||||
export type PatchObjectsMutationResult = NonNullable<
|
||||
Awaited<ReturnType<typeof patchObjects>>
|
||||
>;
|
||||
export type PatchObjectsMutationBody = BodyType<AuthtypesPatchableObjectsDTO>;
|
||||
export type PatchObjectsMutationBody = BodyType<CoretypesPatchableObjectsDTO>;
|
||||
export type PatchObjectsMutationError = ErrorType<RenderErrorResponseDTO>;
|
||||
|
||||
/**
|
||||
@@ -645,7 +645,7 @@ export const usePatchObjects = <
|
||||
TError,
|
||||
{
|
||||
pathParams: PatchObjectsPathParameters;
|
||||
data: BodyType<AuthtypesPatchableObjectsDTO>;
|
||||
data: BodyType<CoretypesPatchableObjectsDTO>;
|
||||
},
|
||||
TContext
|
||||
>;
|
||||
@@ -654,7 +654,7 @@ export const usePatchObjects = <
|
||||
TError,
|
||||
{
|
||||
pathParams: PatchObjectsPathParameters;
|
||||
data: BodyType<AuthtypesPatchableObjectsDTO>;
|
||||
data: BodyType<CoretypesPatchableObjectsDTO>;
|
||||
},
|
||||
TContext
|
||||
> => {
|
||||
|
||||
@@ -1668,33 +1668,6 @@ export interface AuthtypesGettableAuthDomainDTO {
|
||||
updatedAt?: Date;
|
||||
}
|
||||
|
||||
export interface AuthtypesGettableObjectsDTO {
|
||||
resource: AuthtypesResourceDTO;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
selectors: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* @nullable
|
||||
*/
|
||||
export type AuthtypesGettableResourcesDTORelations = {
|
||||
[key: string]: string[];
|
||||
} | null;
|
||||
|
||||
export interface AuthtypesGettableResourcesDTO {
|
||||
/**
|
||||
* @type object
|
||||
* @nullable true
|
||||
*/
|
||||
relations: AuthtypesGettableResourcesDTORelations;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
resources: AuthtypesResourceDTO[];
|
||||
}
|
||||
|
||||
export interface AuthtypesGettableTokenDTO {
|
||||
/**
|
||||
* @type string
|
||||
@@ -1719,11 +1692,8 @@ export interface AuthtypesGettableTransactionDTO {
|
||||
* @type boolean
|
||||
*/
|
||||
authorized: boolean;
|
||||
object: AuthtypesObjectDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
relation: string;
|
||||
object: CoretypesObjectDTO;
|
||||
relation: AuthtypesRelationDTO;
|
||||
}
|
||||
|
||||
export type AuthtypesGoogleConfigDTODomainToAdminEmail = {
|
||||
@@ -1797,14 +1767,6 @@ export interface AuthtypesOIDCConfigDTO {
|
||||
issuerAlias?: string;
|
||||
}
|
||||
|
||||
export interface AuthtypesObjectDTO {
|
||||
resource: AuthtypesResourceDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
selector: string;
|
||||
}
|
||||
|
||||
export interface AuthtypesOrgSessionContextDTO {
|
||||
authNSupport?: AuthtypesAuthNSupportDTO;
|
||||
/**
|
||||
@@ -1822,19 +1784,6 @@ export interface AuthtypesPasswordAuthNSupportDTO {
|
||||
provider?: AuthtypesAuthNProviderDTO;
|
||||
}
|
||||
|
||||
export interface AuthtypesPatchableObjectsDTO {
|
||||
/**
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
additions: AuthtypesGettableObjectsDTO[] | null;
|
||||
/**
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
deletions: AuthtypesGettableObjectsDTO[] | null;
|
||||
}
|
||||
|
||||
export interface AuthtypesPatchableRoleDTO {
|
||||
/**
|
||||
* @type string
|
||||
@@ -1883,17 +1832,14 @@ export interface AuthtypesPostableRotateTokenDTO {
|
||||
refreshToken?: string;
|
||||
}
|
||||
|
||||
export interface AuthtypesResourceDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
name: string;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
type: string;
|
||||
export enum AuthtypesRelationDTO {
|
||||
create = 'create',
|
||||
read = 'read',
|
||||
update = 'update',
|
||||
delete = 'delete',
|
||||
list = 'list',
|
||||
assignee = 'assignee',
|
||||
}
|
||||
|
||||
export interface AuthtypesRoleDTO {
|
||||
/**
|
||||
* @type string
|
||||
@@ -1983,11 +1929,8 @@ export interface AuthtypesSessionContextDTO {
|
||||
}
|
||||
|
||||
export interface AuthtypesTransactionDTO {
|
||||
object: AuthtypesObjectDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
relation: string;
|
||||
object: CoretypesObjectDTO;
|
||||
relation: AuthtypesRelationDTO;
|
||||
}
|
||||
|
||||
export interface AuthtypesUpdatableAuthDomainDTO {
|
||||
@@ -4173,6 +4116,52 @@ export interface ConfigWechatConfigDTO {
|
||||
to_user?: string;
|
||||
}
|
||||
|
||||
export interface CoretypesObjectDTO {
|
||||
resource: CoretypesResourceRefDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
selector: string;
|
||||
}
|
||||
|
||||
export interface CoretypesObjectGroupDTO {
|
||||
resource: CoretypesResourceRefDTO;
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
selectors: string[];
|
||||
}
|
||||
|
||||
export interface CoretypesPatchableObjectsDTO {
|
||||
/**
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
additions: CoretypesObjectGroupDTO[] | null;
|
||||
/**
|
||||
* @type array
|
||||
* @nullable true
|
||||
*/
|
||||
deletions: CoretypesObjectGroupDTO[] | null;
|
||||
}
|
||||
|
||||
export interface CoretypesResourceRefDTO {
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
kind: string;
|
||||
type: CoretypesTypeDTO;
|
||||
}
|
||||
|
||||
export enum CoretypesTypeDTO {
|
||||
user = 'user',
|
||||
serviceaccount = 'serviceaccount',
|
||||
anonymous = 'anonymous',
|
||||
role = 'role',
|
||||
organization = 'organization',
|
||||
metaresource = 'metaresource',
|
||||
metaresources = 'metaresources',
|
||||
}
|
||||
export interface DashboardtypesDashboardDTO {
|
||||
/**
|
||||
* @type string
|
||||
@@ -8110,14 +8099,6 @@ export type AuthzCheck200 = {
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type AuthzResources200 = {
|
||||
data: AuthtypesGettableResourcesDTO;
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type ListChannels200 = {
|
||||
/**
|
||||
* @type array
|
||||
@@ -8743,7 +8724,7 @@ export type GetObjects200 = {
|
||||
/**
|
||||
* @type array
|
||||
*/
|
||||
data: AuthtypesGettableObjectsDTO[];
|
||||
data: CoretypesObjectGroupDTO[];
|
||||
/**
|
||||
* @type string
|
||||
*/
|
||||
|
||||
@@ -55,7 +55,7 @@ describe('GuardAuthZ', () => {
|
||||
);
|
||||
|
||||
render(
|
||||
<GuardAuthZ relation="read" object="dashboard:*">
|
||||
<GuardAuthZ relation="read" object="role:*">
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
@@ -79,7 +79,7 @@ describe('GuardAuthZ', () => {
|
||||
render(
|
||||
<GuardAuthZ
|
||||
relation="read"
|
||||
object="dashboard:*"
|
||||
object="role:*"
|
||||
fallbackOnLoading={<LoadingFallback />}
|
||||
>
|
||||
<TestChild />
|
||||
@@ -102,7 +102,7 @@ describe('GuardAuthZ', () => {
|
||||
);
|
||||
|
||||
const { container } = render(
|
||||
<GuardAuthZ relation="read" object="dashboard:*">
|
||||
<GuardAuthZ relation="read" object="role:*">
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
@@ -121,11 +121,7 @@ describe('GuardAuthZ', () => {
|
||||
);
|
||||
|
||||
render(
|
||||
<GuardAuthZ
|
||||
relation="read"
|
||||
object="dashboard:*"
|
||||
fallbackOnError={ErrorFallback}
|
||||
>
|
||||
<GuardAuthZ relation="read" object="role:*" fallbackOnError={ErrorFallback}>
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
@@ -155,7 +151,7 @@ describe('GuardAuthZ', () => {
|
||||
render(
|
||||
<GuardAuthZ
|
||||
relation="read"
|
||||
object="dashboard:*"
|
||||
object="role:*"
|
||||
fallbackOnError={errorFallbackWithCapture}
|
||||
>
|
||||
<TestChild />
|
||||
@@ -178,7 +174,7 @@ describe('GuardAuthZ', () => {
|
||||
);
|
||||
|
||||
const { container } = render(
|
||||
<GuardAuthZ relation="read" object="dashboard:*">
|
||||
<GuardAuthZ relation="read" object="role:*">
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
@@ -201,7 +197,7 @@ describe('GuardAuthZ', () => {
|
||||
render(
|
||||
<GuardAuthZ
|
||||
relation="update"
|
||||
object="dashboard:123"
|
||||
object="role:123"
|
||||
fallbackOnNoPermissions={NoPermissionFallback}
|
||||
>
|
||||
<TestChild />
|
||||
@@ -224,7 +220,7 @@ describe('GuardAuthZ', () => {
|
||||
);
|
||||
|
||||
const { container } = render(
|
||||
<GuardAuthZ relation="update" object="dashboard:123">
|
||||
<GuardAuthZ relation="update" object="role:123">
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
@@ -244,7 +240,7 @@ describe('GuardAuthZ', () => {
|
||||
);
|
||||
|
||||
const { container } = render(
|
||||
<GuardAuthZ relation="read" object="dashboard:*">
|
||||
<GuardAuthZ relation="read" object="role:*">
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
@@ -257,7 +253,7 @@ describe('GuardAuthZ', () => {
|
||||
});
|
||||
|
||||
it('should pass requiredPermissionName to fallbackOnNoPermissions', async () => {
|
||||
const permission = buildPermission('update', 'dashboard:123');
|
||||
const permission = buildPermission('update', 'role:123');
|
||||
|
||||
server.use(
|
||||
rest.post(AUTHZ_CHECK_URL, async (req, res, ctx) => {
|
||||
@@ -269,7 +265,7 @@ describe('GuardAuthZ', () => {
|
||||
render(
|
||||
<GuardAuthZ
|
||||
relation="update"
|
||||
object="dashboard:123"
|
||||
object="role:123"
|
||||
fallbackOnNoPermissions={NoPermissionFallbackWithSuggestions}
|
||||
>
|
||||
<TestChild />
|
||||
@@ -299,7 +295,7 @@ describe('GuardAuthZ', () => {
|
||||
);
|
||||
|
||||
const { rerender } = render(
|
||||
<GuardAuthZ relation="read" object="dashboard:*">
|
||||
<GuardAuthZ relation="read" object="role:*">
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
@@ -309,7 +305,7 @@ describe('GuardAuthZ', () => {
|
||||
});
|
||||
|
||||
rerender(
|
||||
<GuardAuthZ relation="delete" object="dashboard:456">
|
||||
<GuardAuthZ relation="delete" object="role:456">
|
||||
<TestChild />
|
||||
</GuardAuthZ>,
|
||||
);
|
||||
|
||||
@@ -41,11 +41,7 @@ describe('createGuardedRoute', () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'read',
|
||||
'dashboard:*',
|
||||
);
|
||||
const GuardedComponent = createGuardedRoute(TestComponent, 'read', 'role:*');
|
||||
|
||||
const mockMatch = {
|
||||
params: {},
|
||||
@@ -79,7 +75,7 @@ describe('createGuardedRoute', () => {
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'read',
|
||||
'dashboard:{id}',
|
||||
'role:{id}',
|
||||
);
|
||||
|
||||
const mockMatch = {
|
||||
@@ -113,7 +109,7 @@ describe('createGuardedRoute', () => {
|
||||
relation: txn.relation,
|
||||
object: {
|
||||
resource: {
|
||||
name: txn.object.resource.name,
|
||||
kind: txn.object.resource.kind,
|
||||
type: txn.object.resource.type,
|
||||
},
|
||||
selector: '123:456',
|
||||
@@ -131,7 +127,7 @@ describe('createGuardedRoute', () => {
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'update',
|
||||
'dashboard:{id}:{version}',
|
||||
'role:{id}:{version}',
|
||||
);
|
||||
|
||||
const mockMatch = {
|
||||
@@ -166,7 +162,7 @@ describe('createGuardedRoute', () => {
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'read',
|
||||
'dashboard:{id}',
|
||||
'role:{id}',
|
||||
);
|
||||
|
||||
const mockMatch = {
|
||||
@@ -201,11 +197,7 @@ describe('createGuardedRoute', () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'read',
|
||||
'dashboard:*',
|
||||
);
|
||||
const GuardedComponent = createGuardedRoute(TestComponent, 'read', 'role:*');
|
||||
|
||||
const mockMatch = {
|
||||
params: {},
|
||||
@@ -236,11 +228,7 @@ describe('createGuardedRoute', () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'read',
|
||||
'dashboard:*',
|
||||
);
|
||||
const GuardedComponent = createGuardedRoute(TestComponent, 'read', 'role:*');
|
||||
|
||||
const mockMatch = {
|
||||
params: {},
|
||||
@@ -278,7 +266,7 @@ describe('createGuardedRoute', () => {
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'update',
|
||||
'dashboard:{id}',
|
||||
'role:{id}',
|
||||
);
|
||||
|
||||
const mockMatch = {
|
||||
@@ -304,7 +292,7 @@ describe('createGuardedRoute', () => {
|
||||
});
|
||||
|
||||
expect(screen.getByText('update')).toBeInTheDocument();
|
||||
expect(screen.getByText('dashboard:123')).toBeInTheDocument();
|
||||
expect(screen.getByText('role:123')).toBeInTheDocument();
|
||||
expect(
|
||||
screen.queryByText('Test Component: test-value'),
|
||||
).not.toBeInTheDocument();
|
||||
@@ -335,7 +323,7 @@ describe('createGuardedRoute', () => {
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
ComponentWithMultipleProps,
|
||||
'read',
|
||||
'dashboard:*',
|
||||
'role:*',
|
||||
);
|
||||
|
||||
const mockMatch = {
|
||||
@@ -370,10 +358,10 @@ describe('createGuardedRoute', () => {
|
||||
requestCount++;
|
||||
const payload = (await req.json()) as AuthtypesTransactionDTO[];
|
||||
const obj = payload[0]?.object;
|
||||
const name = obj?.resource?.name;
|
||||
const kind = obj?.resource?.kind;
|
||||
const selector = obj?.selector ?? '*';
|
||||
const objectStr =
|
||||
obj?.resource?.type === 'metaresources' ? name : `${name}:${selector}`;
|
||||
obj?.resource?.type === 'metaresources' ? kind : `${kind}:${selector}`;
|
||||
requestedObjects.push(objectStr ?? '');
|
||||
|
||||
return res(ctx.status(200), ctx.json(authzMockResponse(payload, [true])));
|
||||
@@ -383,7 +371,7 @@ describe('createGuardedRoute', () => {
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'read',
|
||||
'dashboard:{id}',
|
||||
'role:{id}',
|
||||
);
|
||||
|
||||
const mockMatch1 = {
|
||||
@@ -407,7 +395,7 @@ describe('createGuardedRoute', () => {
|
||||
});
|
||||
|
||||
expect(requestCount).toBe(1);
|
||||
expect(requestedObjects).toContain('dashboard:123');
|
||||
expect(requestedObjects).toContain('role:123');
|
||||
|
||||
unmount();
|
||||
|
||||
@@ -432,7 +420,7 @@ describe('createGuardedRoute', () => {
|
||||
});
|
||||
|
||||
expect(requestCount).toBe(2);
|
||||
expect(requestedObjects).toContain('dashboard:456');
|
||||
expect(requestedObjects).toContain('role:456');
|
||||
});
|
||||
|
||||
it('should handle different relation types', async () => {
|
||||
@@ -446,7 +434,7 @@ describe('createGuardedRoute', () => {
|
||||
const GuardedComponent = createGuardedRoute(
|
||||
TestComponent,
|
||||
'delete',
|
||||
'dashboard:{id}',
|
||||
'role:{id}',
|
||||
);
|
||||
|
||||
const mockMatch = {
|
||||
|
||||
@@ -24,6 +24,43 @@
|
||||
line-height: 20px;
|
||||
}
|
||||
|
||||
.crossPanelSyncSectionHeader {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
align-self: flex-start;
|
||||
}
|
||||
|
||||
.crossPanelSyncInfoIcon {
|
||||
cursor: help;
|
||||
color: var(--l3-foreground);
|
||||
}
|
||||
|
||||
.crossPanelSyncTooltipContent {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
max-width: 300px;
|
||||
}
|
||||
|
||||
.crossPanelSyncTooltipTitle {
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.crossPanelSyncTooltipDescription {
|
||||
font-size: 12px;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
.crossPanelSyncTooltipDocLink {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
color: var(--primary-background);
|
||||
font-size: 12px;
|
||||
margin-top: 4px;
|
||||
}
|
||||
|
||||
.crossPanelSyncRow {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Col, Input, Radio, Select, Space, Typography } from 'antd';
|
||||
import { Col, Input, Radio, Select, Space, Tooltip, Typography } from 'antd';
|
||||
import AddTags from 'container/DashboardContainer/DashboardSettings/General/AddTags';
|
||||
import { useDashboardCursorSyncMode } from 'hooks/dashboard/useDashboardCursorSyncMode';
|
||||
import { useSyncTooltipFilterMode } from 'hooks/dashboard/useSyncTooltipFilterMode';
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
SyncTooltipFilterMode,
|
||||
} from 'lib/uPlotV2/plugins/TooltipPlugin/types';
|
||||
import { isEqual } from 'lodash-es';
|
||||
import { Check, X } from 'lucide-react';
|
||||
import { Check, ExternalLink, Info, X } from '@signozhq/icons';
|
||||
import { useDashboardStore } from 'providers/Dashboard/store/useDashboardStore';
|
||||
|
||||
import styles from './GeneralSettings.module.scss';
|
||||
@@ -173,9 +173,36 @@ function GeneralDashboardSettings(): JSX.Element {
|
||||
</Space>
|
||||
</Col>
|
||||
<Col className={`${styles.overviewSettings} ${styles.crossPanelSyncGroup}`}>
|
||||
<Typography.Text className={styles.crossPanelSyncSectionTitle}>
|
||||
Cross-Panel Sync
|
||||
</Typography.Text>
|
||||
<div className={styles.crossPanelSyncSectionHeader}>
|
||||
<Typography.Text className={styles.crossPanelSyncSectionTitle}>
|
||||
Cross-Panel Sync
|
||||
</Typography.Text>
|
||||
<Tooltip
|
||||
title={
|
||||
<div className={styles.crossPanelSyncTooltipContent}>
|
||||
<strong className={styles.crossPanelSyncTooltipTitle}>
|
||||
Cross-Panel Sync
|
||||
</strong>
|
||||
<span className={styles.crossPanelSyncTooltipDescription}>
|
||||
Sync crosshair and tooltip across all the dashboard panels
|
||||
</span>
|
||||
<a
|
||||
href="https://signoz.io/docs/dashboards/interactivity/#cross-panel-sync"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className={styles.crossPanelSyncTooltipDocLink}
|
||||
>
|
||||
Learn more
|
||||
<ExternalLink size={12} />
|
||||
</a>
|
||||
</div>
|
||||
}
|
||||
placement="top"
|
||||
mouseEnterDelay={0.5}
|
||||
>
|
||||
<Info size={14} className={styles.crossPanelSyncInfoIcon} />
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className={styles.crossPanelSyncRow}>
|
||||
<div className={styles.crossPanelSyncInfo}>
|
||||
<Typography.Text className={styles.crossPanelSyncTitle}>
|
||||
|
||||
@@ -4,7 +4,6 @@ import { useHistory, useLocation } from 'react-router-dom';
|
||||
import { Table2, Trash2, Users } from '@signozhq/icons';
|
||||
import { Button, toast, ToggleGroup, ToggleGroupItem } from '@signozhq/ui';
|
||||
import { Skeleton } from 'antd';
|
||||
import { useAuthzResources } from 'api/generated/services/authz';
|
||||
import {
|
||||
getGetObjectsQueryKey,
|
||||
useDeleteRole,
|
||||
@@ -12,6 +11,9 @@ import {
|
||||
useGetRole,
|
||||
usePatchObjects,
|
||||
} from 'api/generated/services/role';
|
||||
import permissionsType from 'hooks/useAuthZ/permissions.type';
|
||||
|
||||
import type { AuthzResources } from '../utils';
|
||||
import ErrorInPlace from 'components/ErrorInPlace/ErrorInPlace';
|
||||
import ROUTES from 'constants/routes';
|
||||
import { capitalize } from 'lodash-es';
|
||||
@@ -52,10 +54,7 @@ function RoleDetailsPage(): JSX.Element {
|
||||
const queryClient = useQueryClient();
|
||||
const { showErrorModal } = useErrorModal();
|
||||
|
||||
const { data: authzResourcesResponse } = useAuthzResources({
|
||||
query: { enabled: true },
|
||||
});
|
||||
const authzResources = authzResourcesResponse?.data ?? null;
|
||||
const authzResources = permissionsType.data as unknown as AuthzResources;
|
||||
|
||||
// Extract channelId from URL pathname since useParams doesn't work in nested routing
|
||||
const roleIdMatch = pathname.match(ROLE_ID_REGEX);
|
||||
@@ -94,7 +93,7 @@ function RoleDetailsPage(): JSX.Element {
|
||||
|
||||
const initialConfig = useMemo(() => {
|
||||
if (!objectsData?.data || !activePermission) {
|
||||
return undefined;
|
||||
return;
|
||||
}
|
||||
return objectsToPermissionConfig(
|
||||
objectsData.data,
|
||||
|
||||
@@ -15,15 +15,6 @@ const CUSTOM_ROLE_ID = '019c24aa-3333-0001-aaaa-111111111111';
|
||||
const MANAGED_ROLE_ID = '019c24aa-2248-756f-9833-984f1ab63819';
|
||||
|
||||
const rolesApiBase = 'http://localhost/api/v1/roles';
|
||||
const authzResourcesUrl = 'http://localhost/api/v1/authz/resources';
|
||||
|
||||
const authzResourcesResponse = {
|
||||
status: 'success',
|
||||
data: {
|
||||
relations: { create: ['dashboard'], read: ['dashboard'] },
|
||||
resources: [{ name: 'dashboard', type: 'dashboard' }],
|
||||
},
|
||||
};
|
||||
|
||||
const emptyObjectsResponse = { status: 'success', data: [] };
|
||||
|
||||
@@ -45,9 +36,6 @@ function setupDefaultHandlers(roleId = CUSTOM_ROLE_ID): void {
|
||||
rest.get(`${rolesApiBase}/:id`, (_req, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(roleResponse)),
|
||||
),
|
||||
rest.get(authzResourcesUrl, (_req, res, ctx) =>
|
||||
res(ctx.status(200), ctx.json(authzResourcesResponse)),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
import type {
|
||||
AuthtypesGettableObjectsDTO,
|
||||
AuthtypesGettableResourcesDTO,
|
||||
CoretypesResourceRefDTO,
|
||||
CoretypesObjectGroupDTO,
|
||||
CoretypesTypeDTO,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
|
||||
import type {
|
||||
PermissionConfig,
|
||||
ResourceDefinition,
|
||||
} from '../PermissionSidePanel/PermissionSidePanel.types';
|
||||
|
||||
type AuthzResources = {
|
||||
resources: CoretypesResourceRefDTO[];
|
||||
relations: Record<string, string[]>;
|
||||
};
|
||||
import { PermissionScope } from '../PermissionSidePanel/PermissionSidePanel.types';
|
||||
import {
|
||||
buildConfig,
|
||||
@@ -33,17 +39,17 @@ jest.mock('../RoleDetails/constants', () => {
|
||||
};
|
||||
});
|
||||
|
||||
const dashboardResource: AuthtypesGettableResourcesDTO['resources'][number] = {
|
||||
name: 'dashboard',
|
||||
type: 'metaresource',
|
||||
const dashboardResource: AuthzResources['resources'][number] = {
|
||||
kind: 'dashboard',
|
||||
type: 'metaresource' as CoretypesTypeDTO,
|
||||
};
|
||||
|
||||
const alertResource: AuthtypesGettableResourcesDTO['resources'][number] = {
|
||||
name: 'alert',
|
||||
type: 'metaresource',
|
||||
const alertResource: AuthzResources['resources'][number] = {
|
||||
kind: 'alert',
|
||||
type: 'metaresource' as CoretypesTypeDTO,
|
||||
};
|
||||
|
||||
const baseAuthzResources: AuthtypesGettableResourcesDTO = {
|
||||
const baseAuthzResources: AuthzResources = {
|
||||
resources: [dashboardResource, alertResource],
|
||||
relations: {
|
||||
create: ['metaresource'],
|
||||
@@ -220,7 +226,7 @@ describe('buildPatchPayload', () => {
|
||||
|
||||
describe('objectsToPermissionConfig', () => {
|
||||
it('maps a wildcard selector to ALL scope', () => {
|
||||
const objects: AuthtypesGettableObjectsDTO[] = [
|
||||
const objects: CoretypesObjectGroupDTO[] = [
|
||||
{ resource: dashboardResource, selectors: ['*'] },
|
||||
];
|
||||
|
||||
@@ -233,7 +239,7 @@ describe('objectsToPermissionConfig', () => {
|
||||
});
|
||||
|
||||
it('maps specific selectors to ONLY_SELECTED scope with the IDs', () => {
|
||||
const objects: AuthtypesGettableObjectsDTO[] = [
|
||||
const objects: CoretypesObjectGroupDTO[] = [
|
||||
{ resource: dashboardResource, selectors: [ID_A, ID_B] },
|
||||
];
|
||||
|
||||
@@ -338,7 +344,7 @@ describe('buildConfig', () => {
|
||||
|
||||
describe('derivePermissionTypes', () => {
|
||||
it('derives one PermissionType per relation key with correct key and capitalised label', () => {
|
||||
const relations: AuthtypesGettableResourcesDTO['relations'] = {
|
||||
const relations: AuthzResources['relations'] = {
|
||||
create: ['metaresource'],
|
||||
read: ['metaresource'],
|
||||
delete: ['metaresource'],
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import React from 'react';
|
||||
import { Badge } from '@signozhq/ui';
|
||||
import type {
|
||||
AuthtypesGettableObjectsDTO,
|
||||
AuthtypesGettableResourcesDTO,
|
||||
CoretypesResourceRefDTO,
|
||||
CoretypesObjectGroupDTO,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
import { DATE_TIME_FORMATS } from 'constants/dateTimeFormats';
|
||||
import { capitalize } from 'lodash-es';
|
||||
@@ -19,6 +19,11 @@ import {
|
||||
PERMISSION_ICON_MAP,
|
||||
} from './RoleDetails/constants';
|
||||
|
||||
export type AuthzResources = {
|
||||
resources: ReadonlyArray<CoretypesResourceRefDTO>;
|
||||
relations: Readonly<Record<string, ReadonlyArray<string>>>;
|
||||
};
|
||||
|
||||
export interface PermissionType {
|
||||
key: string;
|
||||
label: string;
|
||||
@@ -29,11 +34,11 @@ export interface PatchPayloadOptions {
|
||||
newConfig: PermissionConfig;
|
||||
initialConfig: PermissionConfig;
|
||||
resources: ResourceDefinition[];
|
||||
authzRes: AuthtypesGettableResourcesDTO;
|
||||
authzRes: AuthzResources;
|
||||
}
|
||||
|
||||
export function derivePermissionTypes(
|
||||
relations: AuthtypesGettableResourcesDTO['relations'] | null,
|
||||
relations: AuthzResources['relations'] | null,
|
||||
): PermissionType[] {
|
||||
const iconSize = { size: 14 };
|
||||
|
||||
@@ -55,7 +60,7 @@ export function derivePermissionTypes(
|
||||
}
|
||||
|
||||
export function deriveResourcesForRelation(
|
||||
authzResources: AuthtypesGettableResourcesDTO | null,
|
||||
authzResources: AuthzResources | null,
|
||||
relation: string,
|
||||
): ResourceDefinition[] {
|
||||
if (!authzResources?.relations) {
|
||||
@@ -65,19 +70,19 @@ export function deriveResourcesForRelation(
|
||||
return authzResources.resources
|
||||
.filter((r) => supportedTypes.includes(r.type))
|
||||
.map((r) => ({
|
||||
id: r.name,
|
||||
label: capitalize(r.name).replace(/_/g, ' '),
|
||||
id: r.kind,
|
||||
label: capitalize(r.kind).replaceAll('_', ' '),
|
||||
options: [],
|
||||
}));
|
||||
}
|
||||
|
||||
export function objectsToPermissionConfig(
|
||||
objects: AuthtypesGettableObjectsDTO[],
|
||||
objects: CoretypesObjectGroupDTO[],
|
||||
resources: ResourceDefinition[],
|
||||
): PermissionConfig {
|
||||
const config: PermissionConfig = {};
|
||||
for (const res of resources) {
|
||||
const obj = objects.find((o) => o.resource.name === res.id);
|
||||
const obj = objects.find((o) => o.resource.kind === res.id);
|
||||
if (!obj) {
|
||||
config[res.id] = {
|
||||
scope: PermissionScope.ONLY_SELECTED,
|
||||
@@ -101,19 +106,19 @@ export function buildPatchPayload({
|
||||
resources,
|
||||
authzRes,
|
||||
}: PatchPayloadOptions): {
|
||||
additions: AuthtypesGettableObjectsDTO[] | null;
|
||||
deletions: AuthtypesGettableObjectsDTO[] | null;
|
||||
additions: CoretypesObjectGroupDTO[] | null;
|
||||
deletions: CoretypesObjectGroupDTO[] | null;
|
||||
} {
|
||||
if (!authzRes) {
|
||||
return { additions: null, deletions: null };
|
||||
}
|
||||
const additions: AuthtypesGettableObjectsDTO[] = [];
|
||||
const deletions: AuthtypesGettableObjectsDTO[] = [];
|
||||
const additions: CoretypesObjectGroupDTO[] = [];
|
||||
const deletions: CoretypesObjectGroupDTO[] = [];
|
||||
|
||||
for (const res of resources) {
|
||||
const initial = initialConfig[res.id];
|
||||
const current = newConfig[res.id];
|
||||
const resourceDef = authzRes.resources.find((r) => r.name === res.id);
|
||||
const resourceDef = authzRes.resources.find((r) => r.kind === res.id);
|
||||
if (!resourceDef) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1,32 +1,29 @@
|
||||
// AUTO GENERATED FILE - DO NOT EDIT - GENERATED BY scripts/generate-permissions-type
|
||||
// AUTO GENERATED FILE - DO NOT EDIT - GENERATED BY cmd/enterprise/*.go generate authz
|
||||
export default {
|
||||
status: 'success',
|
||||
data: {
|
||||
resources: [
|
||||
{
|
||||
name: 'dashboard',
|
||||
type: 'metaresource',
|
||||
},
|
||||
{
|
||||
name: 'dashboards',
|
||||
kind: 'role',
|
||||
type: 'metaresources',
|
||||
},
|
||||
{
|
||||
name: 'role',
|
||||
kind: 'role',
|
||||
type: 'role',
|
||||
},
|
||||
{
|
||||
name: 'roles',
|
||||
type: 'metaresources',
|
||||
kind: 'serviceaccount',
|
||||
type: 'serviceaccount',
|
||||
},
|
||||
],
|
||||
relations: {
|
||||
assignee: ['role'],
|
||||
attach: ['role', 'serviceaccount'],
|
||||
create: ['metaresources'],
|
||||
delete: ['user', 'serviceaccount', 'role', 'organization', 'metaresource'],
|
||||
delete: ['role', 'serviceaccount'],
|
||||
list: ['metaresources'],
|
||||
read: ['user', 'serviceaccount', 'role', 'organization', 'metaresource'],
|
||||
update: ['user', 'serviceaccount', 'role', 'organization', 'metaresource'],
|
||||
read: ['role', 'serviceaccount'],
|
||||
update: ['role', 'serviceaccount'],
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
import permissionsType from './permissions.type';
|
||||
import { ObjectSeparator } from './utils';
|
||||
|
||||
const ObjectSeparator = ':';
|
||||
|
||||
type PermissionsData = typeof permissionsType.data;
|
||||
export type Resource = PermissionsData['resources'][number];
|
||||
export type ResourceName = Resource['name'];
|
||||
export type ResourceName = Resource['kind'];
|
||||
export type ResourceType = Resource['type'];
|
||||
|
||||
type RelationsByType = PermissionsData['relations'];
|
||||
|
||||
type ResourceTypeMap = {
|
||||
[K in ResourceName]: Extract<Resource, { name: K }>['type'];
|
||||
[K in ResourceName]: Extract<Resource, { kind: K }>['type'];
|
||||
};
|
||||
|
||||
type RelationName = keyof RelationsByType;
|
||||
@@ -17,7 +18,7 @@ type RelationName = keyof RelationsByType;
|
||||
export type ResourcesForRelation<R extends RelationName> = Extract<
|
||||
Resource,
|
||||
{ type: RelationsByType[R][number] }
|
||||
>['name'];
|
||||
>['kind'];
|
||||
|
||||
type IsPluralResource<R extends ResourceName> =
|
||||
ResourceTypeMap[R] extends 'metaresources' ? true : false;
|
||||
|
||||
@@ -36,8 +36,8 @@ const wrapper = ({ children }: { children: ReactElement }): ReactElement => (
|
||||
|
||||
describe('useAuthZ', () => {
|
||||
it('should fetch and return permissions successfully', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
|
||||
const expectedResponse = {
|
||||
[permission1]: {
|
||||
@@ -74,7 +74,7 @@ describe('useAuthZ', () => {
|
||||
});
|
||||
|
||||
it('should handle API errors', async () => {
|
||||
const permission = buildPermission('read', 'dashboard:*');
|
||||
const permission = buildPermission('read', 'role:*');
|
||||
|
||||
server.use(
|
||||
rest.post(AUTHZ_CHECK_URL, (_req, res, ctx) => {
|
||||
@@ -95,9 +95,9 @@ describe('useAuthZ', () => {
|
||||
});
|
||||
|
||||
it('should refetch when permissions array changes', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission3 = buildPermission('delete', 'dashboard:456');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
const permission3 = buildPermission('delete', 'role:456');
|
||||
|
||||
let requestCount = 0;
|
||||
|
||||
@@ -161,8 +161,8 @@ describe('useAuthZ', () => {
|
||||
});
|
||||
|
||||
it('should not refetch when permissions array order changes but content is the same', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
|
||||
let requestCount = 0;
|
||||
|
||||
@@ -217,8 +217,8 @@ describe('useAuthZ', () => {
|
||||
});
|
||||
|
||||
it('should send correct payload format to API', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
|
||||
let receivedPayload: any = null;
|
||||
|
||||
@@ -244,23 +244,23 @@ describe('useAuthZ', () => {
|
||||
expect(receivedPayload[0]).toMatchObject({
|
||||
relation: 'read',
|
||||
object: {
|
||||
resource: { name: 'dashboard', type: 'metaresource' },
|
||||
resource: { kind: 'role', type: 'role' },
|
||||
selector: '*',
|
||||
},
|
||||
});
|
||||
expect(receivedPayload[1]).toMatchObject({
|
||||
relation: 'update',
|
||||
object: {
|
||||
resource: { name: 'dashboard', type: 'metaresource' },
|
||||
resource: { kind: 'role', type: 'role' },
|
||||
selector: '123',
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('should batch multiple hooks into single flight request', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission3 = buildPermission('delete', 'dashboard:456');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
const permission3 = buildPermission('delete', 'role:456');
|
||||
|
||||
let requestCount = 0;
|
||||
const receivedPayloads: any[] = [];
|
||||
@@ -304,17 +304,17 @@ describe('useAuthZ', () => {
|
||||
expect(receivedPayloads[0][0]).toMatchObject({
|
||||
relation: 'read',
|
||||
object: {
|
||||
resource: { name: 'dashboard', type: 'metaresource' },
|
||||
resource: { kind: 'role', type: 'role' },
|
||||
selector: '*',
|
||||
},
|
||||
});
|
||||
expect(receivedPayloads[0][1]).toMatchObject({
|
||||
relation: 'update',
|
||||
object: { resource: { name: 'dashboard' }, selector: '123' },
|
||||
object: { resource: { kind: 'role' }, selector: '123' },
|
||||
});
|
||||
expect(receivedPayloads[0][2]).toMatchObject({
|
||||
relation: 'delete',
|
||||
object: { resource: { name: 'dashboard' }, selector: '456' },
|
||||
object: { resource: { kind: 'role' }, selector: '456' },
|
||||
});
|
||||
|
||||
expect(result1.current.permissions).toStrictEqual({
|
||||
@@ -329,9 +329,9 @@ describe('useAuthZ', () => {
|
||||
});
|
||||
|
||||
it('should create separate batches for calls after single flight window', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission3 = buildPermission('delete', 'dashboard:456');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
const permission3 = buildPermission('delete', 'role:456');
|
||||
|
||||
let requestCount = 0;
|
||||
const receivedPayloads: any[] = [];
|
||||
@@ -386,18 +386,18 @@ describe('useAuthZ', () => {
|
||||
expect(receivedPayloads[1]).toHaveLength(2);
|
||||
expect(receivedPayloads[1][0]).toMatchObject({
|
||||
relation: 'update',
|
||||
object: { resource: { name: 'dashboard' }, selector: '123' },
|
||||
object: { resource: { kind: 'role' }, selector: '123' },
|
||||
});
|
||||
expect(receivedPayloads[1][1]).toMatchObject({
|
||||
relation: 'delete',
|
||||
object: { resource: { name: 'dashboard' }, selector: '456' },
|
||||
object: { resource: { kind: 'role' }, selector: '456' },
|
||||
});
|
||||
});
|
||||
|
||||
it('should map permissions correctly when API returns response out of order', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission3 = buildPermission('delete', 'dashboard:456');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
const permission3 = buildPermission('delete', 'role:456');
|
||||
|
||||
server.use(
|
||||
rest.post(AUTHZ_CHECK_URL, async (req, res, ctx) => {
|
||||
@@ -435,8 +435,8 @@ describe('useAuthZ', () => {
|
||||
});
|
||||
|
||||
it('should not leak state between separate batches', async () => {
|
||||
const permission1 = buildPermission('read', 'dashboard:*');
|
||||
const permission2 = buildPermission('update', 'dashboard:123');
|
||||
const permission1 = buildPermission('read', 'role:*');
|
||||
const permission2 = buildPermission('update', 'role:123');
|
||||
|
||||
let requestCount = 0;
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import { useCallback, useMemo } from 'react';
|
||||
import { useQueries } from 'react-query';
|
||||
import { authzCheck } from 'api/generated/services/authz';
|
||||
import type {
|
||||
AuthtypesObjectDTO,
|
||||
CoretypesObjectDTO,
|
||||
AuthtypesTransactionDTO,
|
||||
} from 'api/generated/services/sigNoz.schemas';
|
||||
|
||||
@@ -34,7 +34,7 @@ function dispatchPermission(
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
const copiedPermissions = pendingPermissions.slice();
|
||||
const copiedPermissions = [...pendingPermissions];
|
||||
pendingPermissions = [];
|
||||
ctx = null;
|
||||
|
||||
@@ -50,9 +50,9 @@ async function fetchManyPermissions(
|
||||
): Promise<AuthZCheckResponse> {
|
||||
const payload: AuthtypesTransactionDTO[] = permissions.map((permission) => {
|
||||
const dto = permissionToTransactionDto(permission);
|
||||
const object: AuthtypesObjectDTO = {
|
||||
const object: CoretypesObjectDTO = {
|
||||
resource: {
|
||||
name: dto.object.resource.name,
|
||||
kind: dto.object.resource.kind,
|
||||
type: dto.object.resource.type,
|
||||
},
|
||||
selector: dto.object.selector,
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { AuthtypesTransactionDTO } from '../../api/generated/services/sigNoz.schemas';
|
||||
import {
|
||||
AuthtypesTransactionDTO,
|
||||
CoretypesTypeDTO,
|
||||
AuthtypesRelationDTO,
|
||||
} from '../../api/generated/services/sigNoz.schemas';
|
||||
import permissionsType from './permissions.type';
|
||||
import {
|
||||
AuthZObject,
|
||||
@@ -33,36 +37,72 @@ export function parsePermission(permission: BrandedPermission): {
|
||||
return { relation: relation as AuthZRelation, object };
|
||||
}
|
||||
|
||||
const resourceNameToType = permissionsType.data.resources.reduce(
|
||||
const kindsByType = permissionsType.data.resources.reduce(
|
||||
(acc, r) => {
|
||||
acc[r.name] = r.type;
|
||||
if (!acc[r.type]) {
|
||||
acc[r.type] = new Set();
|
||||
}
|
||||
acc[r.type].add(r.kind);
|
||||
return acc;
|
||||
},
|
||||
{} as Record<ResourceName, ResourceType>,
|
||||
{} as Record<string, Set<string>>,
|
||||
);
|
||||
|
||||
function resolveType(
|
||||
relation: AuthZRelation,
|
||||
kind: string,
|
||||
): ResourceType | undefined {
|
||||
const candidates: readonly string[] =
|
||||
permissionsType.data.relations[relation] ?? [];
|
||||
for (const t of candidates) {
|
||||
if (kindsByType[t]?.has(kind)) {
|
||||
return t as ResourceType;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function splitObjectString(objectStr: string): {
|
||||
resourceName: string;
|
||||
selector: string;
|
||||
} {
|
||||
const idx = objectStr.indexOf(ObjectSeparator);
|
||||
if (idx === -1) {
|
||||
return { resourceName: objectStr, selector: '' };
|
||||
}
|
||||
return {
|
||||
resourceName: objectStr.slice(0, idx),
|
||||
selector: objectStr.slice(idx + 1),
|
||||
};
|
||||
}
|
||||
|
||||
export function permissionToTransactionDto(
|
||||
permission: BrandedPermission,
|
||||
): AuthtypesTransactionDTO {
|
||||
const { relation, object: objectStr } = parsePermission(permission);
|
||||
const directType = resourceNameToType[objectStr as ResourceName];
|
||||
const directType = resolveType(relation, objectStr);
|
||||
if (directType === 'metaresources') {
|
||||
return {
|
||||
relation,
|
||||
relation: relation as AuthtypesRelationDTO,
|
||||
object: {
|
||||
resource: { name: objectStr, type: directType },
|
||||
resource: {
|
||||
kind: objectStr as ResourceName,
|
||||
type: directType as CoretypesTypeDTO,
|
||||
},
|
||||
selector: '*',
|
||||
},
|
||||
};
|
||||
}
|
||||
const [resourceName, selector] = objectStr.split(ObjectSeparator);
|
||||
const type =
|
||||
resourceNameToType[resourceName as ResourceName] ?? 'metaresource';
|
||||
const { resourceName, selector } = splitObjectString(objectStr);
|
||||
const type = resolveType(relation, resourceName) ?? 'metaresource';
|
||||
|
||||
return {
|
||||
relation,
|
||||
relation: relation as AuthtypesRelationDTO,
|
||||
object: {
|
||||
resource: { name: resourceName, type },
|
||||
resource: {
|
||||
kind: resourceName as ResourceName,
|
||||
type: type as CoretypesTypeDTO,
|
||||
},
|
||||
selector: selector || '*',
|
||||
},
|
||||
};
|
||||
@@ -75,7 +115,7 @@ export function gettableTransactionToPermission(
|
||||
relation,
|
||||
object: { resource, selector },
|
||||
} = item;
|
||||
const resourceName = String(resource.name);
|
||||
const resourceName = String(resource.kind);
|
||||
const selectorStr = typeof selector === 'string' ? selector : '*';
|
||||
const objectStr =
|
||||
resource.type === 'metaresources'
|
||||
|
||||
@@ -24,10 +24,15 @@ export default function Tooltip({
|
||||
);
|
||||
|
||||
const showHeader = showTooltipHeader || activeItem != null;
|
||||
// With a single series the active item is fully represented in the header —
|
||||
// hide the divider and list to avoid showing a duplicate row.
|
||||
const showList = tooltipContent.length > 1;
|
||||
const showDivider = showList && showHeader;
|
||||
// A single row collapses into the header when it's the active item, but
|
||||
// must stay in the list when there's no active item (e.g. sync-driven
|
||||
// tooltips with no focused series) — otherwise the row would vanish.
|
||||
const showList =
|
||||
tooltipContent.length > 1 ||
|
||||
(tooltipContent.length === 1 && activeItem == null);
|
||||
// The divider separates the active row in the header from the list; with
|
||||
// no active item it has nothing to separate.
|
||||
const showDivider = showList && showHeader && activeItem != null;
|
||||
|
||||
return (
|
||||
<div
|
||||
|
||||
@@ -137,7 +137,7 @@ function applyReceiverSync({
|
||||
|
||||
if (commonKeys.length === 0) {
|
||||
uPlotInstance.setSeries(null, { focus: false });
|
||||
return [];
|
||||
return noMatchResult;
|
||||
}
|
||||
|
||||
if ((uPlotInstance.cursor.left ?? -1) < 0) {
|
||||
|
||||
@@ -26,22 +26,5 @@ func (provider *provider) addAuthzRoutes(router *mux.Router) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/authz/resources", handler.New(provider.authZ.OpenAccess(provider.authzHandler.GetResources), handler.OpenAPIDef{
|
||||
ID: "AuthzResources",
|
||||
Tags: []string{"authz"},
|
||||
Summary: "Get resources",
|
||||
Description: "Gets all the available resources",
|
||||
Request: nil,
|
||||
RequestContentType: "",
|
||||
Response: new(authtypes.GettableResources),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{},
|
||||
Deprecated: false,
|
||||
SecuritySchemes: nil,
|
||||
})).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/http/handler"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -83,9 +84,9 @@ func (provider *provider) addDashboardRoutes(router *mux.Router) error {
|
||||
|
||||
if err := router.Handle("/api/v1/public/dashboards/{id}", handler.New(provider.authZ.CheckWithoutClaims(
|
||||
provider.dashboardHandler.GetPublicData,
|
||||
authtypes.RelationRead,
|
||||
dashboardtypes.TypeableMetaResourcePublicDashboard,
|
||||
func(req *http.Request, orgs []*types.Organization) ([]authtypes.Selector, valuer.UUID, error) {
|
||||
authtypes.Relation{Verb: coretypes.VerbRead},
|
||||
coretypes.ResourceMetaResourcePublicDashboard,
|
||||
func(req *http.Request, orgs []*types.Organization) ([]coretypes.Selector, valuer.UUID, error) {
|
||||
id, err := valuer.NewUUID(mux.Vars(req)["id"])
|
||||
if err != nil {
|
||||
return nil, valuer.UUID{}, err
|
||||
@@ -104,16 +105,16 @@ func (provider *provider) addDashboardRoutes(router *mux.Router) error {
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{},
|
||||
Deprecated: false,
|
||||
SecuritySchemes: newAnonymousSecuritySchemes([]string{dashboardtypes.TypeableMetaResourcePublicDashboard.Scope(authtypes.RelationRead)}),
|
||||
SecuritySchemes: newAnonymousSecuritySchemes([]string{coretypes.ResourceMetaResourcePublicDashboard.Scope(coretypes.VerbRead)}),
|
||||
})).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := router.Handle("/api/v1/public/dashboards/{id}/widgets/{idx}/query_range", handler.New(provider.authZ.CheckWithoutClaims(
|
||||
provider.dashboardHandler.GetPublicWidgetQueryRange,
|
||||
authtypes.RelationRead,
|
||||
dashboardtypes.TypeableMetaResourcePublicDashboard,
|
||||
func(req *http.Request, orgs []*types.Organization) ([]authtypes.Selector, valuer.UUID, error) {
|
||||
authtypes.Relation{Verb: coretypes.VerbRead},
|
||||
coretypes.ResourceMetaResourcePublicDashboard,
|
||||
func(req *http.Request, orgs []*types.Organization) ([]coretypes.Selector, valuer.UUID, error) {
|
||||
id, err := valuer.NewUUID(mux.Vars(req)["id"])
|
||||
if err != nil {
|
||||
return nil, valuer.UUID{}, err
|
||||
@@ -132,7 +133,7 @@ func (provider *provider) addDashboardRoutes(router *mux.Router) error {
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{},
|
||||
Deprecated: false,
|
||||
SecuritySchemes: newAnonymousSecuritySchemes([]string{dashboardtypes.TypeableMetaResourcePublicDashboard.Scope(authtypes.RelationRead)}),
|
||||
SecuritySchemes: newAnonymousSecuritySchemes([]string{coretypes.ResourceMetaResourcePublicDashboard.Scope(coretypes.VerbRead)}),
|
||||
})).Methods(http.MethodGet).GetError(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/http/handler"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
@@ -68,7 +69,7 @@ func (provider *provider) addRoleRoutes(router *mux.Router) error {
|
||||
Description: "Gets all objects connected to the specified role via a given relation type",
|
||||
Request: nil,
|
||||
RequestContentType: "",
|
||||
Response: make([]*authtypes.GettableObjects, 0),
|
||||
Response: make([]*coretypes.ObjectGroup, 0),
|
||||
ResponseContentType: "application/json",
|
||||
SuccessStatusCode: http.StatusOK,
|
||||
ErrorStatusCodes: []int{http.StatusNotFound, http.StatusNotImplemented, http.StatusUnavailableForLegalReasons},
|
||||
@@ -100,7 +101,7 @@ func (provider *provider) addRoleRoutes(router *mux.Router) error {
|
||||
Tags: []string{"role"},
|
||||
Summary: "Patch objects for a role by relation",
|
||||
Description: "Patches the objects connected to the specified role via a given relation type",
|
||||
Request: new(authtypes.PatchableObjects),
|
||||
Request: new(coretypes.PatchableObjects),
|
||||
RequestContentType: "",
|
||||
Response: nil,
|
||||
ResponseContentType: "application/json",
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
|
||||
"github.com/SigNoz/signoz/pkg/types/audittypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -19,16 +20,16 @@ func newTestSettings() factory.ScopedProviderSettings {
|
||||
return factory.NewScopedProviderSettings(instrumentationtest.New().ToProviderSettings(), "auditorserver_test")
|
||||
}
|
||||
|
||||
func newTestEvent(resource string, action audittypes.Action) audittypes.AuditEvent {
|
||||
func newTestEvent(resource string, action coretypes.Verb) audittypes.AuditEvent {
|
||||
return audittypes.AuditEvent{
|
||||
Timestamp: time.Now(),
|
||||
EventName: audittypes.NewEventName(resource, action),
|
||||
EventName: audittypes.NewEventName(coretypes.MustNewKind(resource), action),
|
||||
AuditAttributes: audittypes.AuditAttributes{
|
||||
Action: action,
|
||||
Outcome: audittypes.OutcomeSuccess,
|
||||
},
|
||||
ResourceAttributes: audittypes.ResourceAttributes{
|
||||
ResourceKind: resource,
|
||||
ResourceKind: coretypes.MustNewKind(resource),
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -83,7 +84,7 @@ func TestAdd_FlushesOnBatchSize(t *testing.T) {
|
||||
go func() { _ = server.Start(ctx) }()
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
server.Add(ctx, newTestEvent("dashboard", audittypes.ActionCreate))
|
||||
server.Add(ctx, newTestEvent("dashboard", coretypes.VerbCreate))
|
||||
}
|
||||
|
||||
assert.Eventually(t, func() bool {
|
||||
@@ -112,7 +113,7 @@ func TestAdd_FlushesOnInterval(t *testing.T) {
|
||||
|
||||
go func() { _ = server.Start(ctx) }()
|
||||
|
||||
server.Add(ctx, newTestEvent("user", audittypes.ActionUpdate))
|
||||
server.Add(ctx, newTestEvent("user", coretypes.VerbUpdate))
|
||||
|
||||
assert.Eventually(t, func() bool {
|
||||
return exported.Load() == 1
|
||||
@@ -130,9 +131,9 @@ func TestAdd_DropsWhenBufferFull(t *testing.T) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
server.Add(ctx, newTestEvent("dashboard", audittypes.ActionCreate))
|
||||
server.Add(ctx, newTestEvent("dashboard", audittypes.ActionUpdate))
|
||||
server.Add(ctx, newTestEvent("dashboard", audittypes.ActionDelete))
|
||||
server.Add(ctx, newTestEvent("dashboard", coretypes.VerbCreate))
|
||||
server.Add(ctx, newTestEvent("dashboard", coretypes.VerbUpdate))
|
||||
server.Add(ctx, newTestEvent("dashboard", coretypes.VerbDelete))
|
||||
|
||||
assert.Equal(t, 2, server.queueLen())
|
||||
}
|
||||
@@ -155,7 +156,7 @@ func TestStop_DrainsRemainingEvents(t *testing.T) {
|
||||
go func() { _ = server.Start(ctx) }()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
server.Add(ctx, newTestEvent("alert-rule", audittypes.ActionCreate))
|
||||
server.Add(ctx, newTestEvent("alert-rule", coretypes.VerbCreate))
|
||||
}
|
||||
|
||||
require.NoError(t, server.Stop(ctx))
|
||||
@@ -180,8 +181,8 @@ func TestAdd_ContinuesAfterExportFailure(t *testing.T) {
|
||||
|
||||
go func() { _ = server.Start(ctx) }()
|
||||
|
||||
server.Add(ctx, newTestEvent("user", audittypes.ActionDelete))
|
||||
server.Add(ctx, newTestEvent("user", audittypes.ActionDelete))
|
||||
server.Add(ctx, newTestEvent("user", coretypes.VerbDelete))
|
||||
server.Add(ctx, newTestEvent("user", coretypes.VerbDelete))
|
||||
|
||||
assert.Eventually(t, func() bool {
|
||||
return calls.Load() >= 1
|
||||
@@ -212,7 +213,7 @@ func TestAdd_ConcurrentSafety(t *testing.T) {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
server.Add(ctx, newTestEvent("dashboard", audittypes.ActionCreate))
|
||||
server.Add(ctx, newTestEvent("dashboard", coretypes.VerbCreate))
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
openfgav1 "github.com/openfga/api/proto/openfga/v1"
|
||||
)
|
||||
@@ -14,10 +15,10 @@ type AuthZ interface {
|
||||
factory.ServiceWithHealthy
|
||||
|
||||
// CheckWithTupleCreation takes upon the responsibility for generating the tuples alongside everything Check does.
|
||||
CheckWithTupleCreation(context.Context, authtypes.Claims, valuer.UUID, authtypes.Relation, authtypes.Typeable, []authtypes.Selector, []authtypes.Selector) error
|
||||
CheckWithTupleCreation(context.Context, authtypes.Claims, valuer.UUID, authtypes.Relation, coretypes.Resource, []coretypes.Selector, []coretypes.Selector) error
|
||||
|
||||
// CheckWithTupleCreationWithoutClaims checks permissions for anonymous users.
|
||||
CheckWithTupleCreationWithoutClaims(context.Context, valuer.UUID, authtypes.Relation, authtypes.Typeable, []authtypes.Selector, []authtypes.Selector) error
|
||||
CheckWithTupleCreationWithoutClaims(context.Context, valuer.UUID, authtypes.Relation, coretypes.Resource, []coretypes.Selector, []coretypes.Selector) error
|
||||
|
||||
// BatchCheck accepts a map of ID → tuple and returns a map of ID → authorization result.
|
||||
BatchCheck(context.Context, map[string]*openfgav1.TupleKey) (map[string]*authtypes.TupleKeyAuthorization, error)
|
||||
@@ -30,7 +31,7 @@ type AuthZ interface {
|
||||
Write(context.Context, []*openfgav1.TupleKey, []*openfgav1.TupleKey) error
|
||||
|
||||
// Lists the selectors for objects assigned to subject (s) with relation (r) on resource (s)
|
||||
ListObjects(context.Context, string, authtypes.Relation, authtypes.Type) ([]*authtypes.Object, error)
|
||||
ListObjects(context.Context, string, authtypes.Relation, coretypes.Type) ([]*coretypes.Object, error)
|
||||
|
||||
// Creates the role.
|
||||
Create(context.Context, valuer.UUID, *authtypes.Role) error
|
||||
@@ -39,16 +40,13 @@ type AuthZ interface {
|
||||
GetOrCreate(context.Context, valuer.UUID, *authtypes.Role) (*authtypes.Role, error)
|
||||
|
||||
// Gets the objects associated with the given role and relation.
|
||||
GetObjects(context.Context, valuer.UUID, valuer.UUID, authtypes.Relation) ([]*authtypes.Object, error)
|
||||
|
||||
// Gets all the typeable resources registered from role registry.
|
||||
GetResources(context.Context) []*authtypes.Resource
|
||||
GetObjects(context.Context, valuer.UUID, valuer.UUID, authtypes.Relation) ([]*coretypes.Object, error)
|
||||
|
||||
// Patches the role.
|
||||
Patch(context.Context, valuer.UUID, *authtypes.Role) error
|
||||
|
||||
// Patches the objects in authorization server associated with the given role and relation
|
||||
PatchObjects(context.Context, valuer.UUID, string, authtypes.Relation, []*authtypes.Object, []*authtypes.Object) error
|
||||
PatchObjects(context.Context, valuer.UUID, string, authtypes.Relation, []*coretypes.Object, []*coretypes.Object) error
|
||||
|
||||
// Deletes the role and tuples in authorization server.
|
||||
Delete(context.Context, valuer.UUID, valuer.UUID) error
|
||||
@@ -90,12 +88,6 @@ type AuthZ interface {
|
||||
// OnBeforeRoleDelete is a callback invoked before a role is deleted.
|
||||
type OnBeforeRoleDelete func(context.Context, valuer.UUID, valuer.UUID) error
|
||||
|
||||
type RegisterTypeable interface {
|
||||
MustGetTypeables() []authtypes.Typeable
|
||||
|
||||
MustGetManagedRoleTransactions() map[string][]*authtypes.Transaction
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
Create(http.ResponseWriter, *http.Request)
|
||||
|
||||
@@ -103,8 +95,6 @@ type Handler interface {
|
||||
|
||||
GetObjects(http.ResponseWriter, *http.Request)
|
||||
|
||||
GetResources(http.ResponseWriter, *http.Request)
|
||||
|
||||
List(http.ResponseWriter, *http.Request)
|
||||
|
||||
Patch(http.ResponseWriter, *http.Request)
|
||||
|
||||
@@ -25,7 +25,7 @@ func newConfig() factory.Config {
|
||||
return &Config{
|
||||
Provider: "openfga",
|
||||
OpenFGA: OpenFGAConfig{
|
||||
MaxTuplesPerWrite: 100,
|
||||
MaxTuplesPerWrite: 300,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/authz/openfgaserver"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
@@ -18,31 +19,27 @@ import (
|
||||
)
|
||||
|
||||
type provider struct {
|
||||
server *openfgaserver.Server
|
||||
store authtypes.RoleStore
|
||||
registry []authz.RegisterTypeable
|
||||
managedRolesByTransaction map[string][]string
|
||||
server *openfgaserver.Server
|
||||
store authtypes.RoleStore
|
||||
registry *authtypes.Registry
|
||||
}
|
||||
|
||||
func NewProviderFactory(sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, registry ...authz.RegisterTypeable) factory.ProviderFactory[authz.AuthZ, authz.Config] {
|
||||
func NewProviderFactory(sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, registry *authtypes.Registry) factory.ProviderFactory[authz.AuthZ, authz.Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("openfga"), func(ctx context.Context, ps factory.ProviderSettings, config authz.Config) (authz.AuthZ, error) {
|
||||
return newOpenfgaProvider(ctx, ps, config, sqlstore, openfgaSchema, openfgaDataStore, registry)
|
||||
})
|
||||
}
|
||||
|
||||
func newOpenfgaProvider(ctx context.Context, settings factory.ProviderSettings, config authz.Config, sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, registry []authz.RegisterTypeable) (authz.AuthZ, error) {
|
||||
func newOpenfgaProvider(ctx context.Context, settings factory.ProviderSettings, config authz.Config, sqlstore sqlstore.SQLStore, openfgaSchema []openfgapkgtransformer.ModuleFile, openfgaDataStore storage.OpenFGADatastore, registry *authtypes.Registry) (authz.AuthZ, error) {
|
||||
server, err := openfgaserver.NewOpenfgaServer(ctx, settings, config, sqlstore, openfgaSchema, openfgaDataStore)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
managedRolesByTransaction := buildManagedRolesByTransaction(registry)
|
||||
|
||||
return &provider{
|
||||
server: server,
|
||||
store: sqlauthzstore.NewSqlAuthzStore(sqlstore),
|
||||
registry: registry,
|
||||
managedRolesByTransaction: managedRolesByTransaction,
|
||||
server: server,
|
||||
store: sqlauthzstore.NewSqlAuthzStore(sqlstore),
|
||||
registry: registry,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -62,11 +59,11 @@ func (provider *provider) BatchCheck(ctx context.Context, tupleReq map[string]*o
|
||||
return provider.server.BatchCheck(ctx, tupleReq)
|
||||
}
|
||||
|
||||
func (provider *provider) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, relation authtypes.Relation, typeable authtypes.Typeable, selectors []authtypes.Selector, roleSelectors []authtypes.Selector) error {
|
||||
func (provider *provider) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, relation authtypes.Relation, typeable coretypes.Resource, selectors []coretypes.Selector, roleSelectors []coretypes.Selector) error {
|
||||
return provider.server.CheckWithTupleCreation(ctx, claims, orgID, relation, typeable, selectors, roleSelectors)
|
||||
}
|
||||
|
||||
func (provider *provider) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, relation authtypes.Relation, typeable authtypes.Typeable, selectors []authtypes.Selector, roleSelectors []authtypes.Selector) error {
|
||||
func (provider *provider) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, relation authtypes.Relation, typeable coretypes.Resource, selectors []coretypes.Selector, roleSelectors []coretypes.Selector) error {
|
||||
return provider.server.CheckWithTupleCreationWithoutClaims(ctx, orgID, relation, typeable, selectors, roleSelectors)
|
||||
}
|
||||
|
||||
@@ -78,7 +75,7 @@ func (provider *provider) ReadTuples(ctx context.Context, tupleKey *openfgav1.Re
|
||||
return provider.server.ReadTuples(ctx, tupleKey)
|
||||
}
|
||||
|
||||
func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
|
||||
func (provider *provider) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType coretypes.Type) ([]*coretypes.Object, error) {
|
||||
return provider.server.ListObjects(ctx, subject, relation, objectType)
|
||||
}
|
||||
|
||||
@@ -103,22 +100,14 @@ func (provider *provider) ListByOrgIDAndIDs(ctx context.Context, orgID valuer.UU
|
||||
}
|
||||
|
||||
func (provider *provider) Grant(ctx context.Context, orgID valuer.UUID, names []string, subject string) error {
|
||||
selectors := make([]authtypes.Selector, len(names))
|
||||
selectors := make([]coretypes.Selector, len(names))
|
||||
for idx, name := range names {
|
||||
selectors[idx] = authtypes.MustNewSelector(authtypes.TypeRole, name)
|
||||
selectors[idx] = coretypes.TypeRole.MustSelector(name)
|
||||
}
|
||||
|
||||
tuples, err := authtypes.TypeableRole.Tuples(
|
||||
subject,
|
||||
authtypes.RelationAssignee,
|
||||
selectors,
|
||||
orgID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tuples := authtypes.NewTuples(coretypes.NewResourceRole(), subject, authtypes.Relation{Verb: coretypes.VerbAssignee}, selectors, orgID)
|
||||
|
||||
err = provider.Write(ctx, tuples, nil)
|
||||
err := provider.Write(ctx, tuples, nil)
|
||||
if err != nil {
|
||||
return errors.WithAdditionalf(err, "failed to grant roles: %v to subject: %s", names, subject)
|
||||
}
|
||||
@@ -141,22 +130,14 @@ func (provider *provider) ModifyGrant(ctx context.Context, orgID valuer.UUID, ex
|
||||
}
|
||||
|
||||
func (provider *provider) Revoke(ctx context.Context, orgID valuer.UUID, names []string, subject string) error {
|
||||
selectors := make([]authtypes.Selector, len(names))
|
||||
selectors := make([]coretypes.Selector, len(names))
|
||||
for idx, name := range names {
|
||||
selectors[idx] = authtypes.MustNewSelector(authtypes.TypeRole, name)
|
||||
selectors[idx] = coretypes.TypeRole.MustSelector(name)
|
||||
}
|
||||
|
||||
tuples, err := authtypes.TypeableRole.Tuples(
|
||||
subject,
|
||||
authtypes.RelationAssignee,
|
||||
selectors,
|
||||
orgID,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tuples := authtypes.NewTuples(coretypes.NewResourceRole(), subject, authtypes.Relation{Verb: coretypes.VerbAssignee}, selectors, orgID)
|
||||
|
||||
err = provider.Write(ctx, nil, tuples)
|
||||
err := provider.Write(ctx, nil, tuples)
|
||||
if err != nil {
|
||||
return errors.WithAdditionalf(err, "failed to revoke roles: %v to subject: %s", names, subject)
|
||||
}
|
||||
@@ -184,7 +165,7 @@ func (provider *provider) CreateManagedRoles(ctx context.Context, _ valuer.UUID,
|
||||
}
|
||||
|
||||
func (provider *provider) CreateManagedUserRoleTransactions(ctx context.Context, orgID valuer.UUID, userID valuer.UUID) error {
|
||||
return provider.Grant(ctx, orgID, []string{authtypes.SigNozAdminRoleName}, authtypes.MustNewSubject(authtypes.TypeableUser, userID.String(), orgID, nil))
|
||||
return provider.Grant(ctx, orgID, []string{authtypes.SigNozAdminRoleName}, authtypes.MustNewSubject(coretypes.NewResourceUser(), userID.String(), orgID, nil))
|
||||
}
|
||||
|
||||
func (setter *provider) Create(_ context.Context, _ valuer.UUID, _ *authtypes.Role) error {
|
||||
@@ -195,11 +176,7 @@ func (provider *provider) GetOrCreate(_ context.Context, _ valuer.UUID, _ *autht
|
||||
return nil, errors.Newf(errors.TypeUnsupported, authtypes.ErrCodeRoleUnsupported, "not implemented")
|
||||
}
|
||||
|
||||
func (provider *provider) GetResources(_ context.Context) []*authtypes.Resource {
|
||||
return []*authtypes.Resource{}
|
||||
}
|
||||
|
||||
func (provider *provider) GetObjects(ctx context.Context, orgID valuer.UUID, id valuer.UUID, relation authtypes.Relation) ([]*authtypes.Object, error) {
|
||||
func (provider *provider) GetObjects(ctx context.Context, orgID valuer.UUID, id valuer.UUID, relation authtypes.Relation) ([]*coretypes.Object, error) {
|
||||
return nil, errors.Newf(errors.TypeUnsupported, authtypes.ErrCodeRoleUnsupported, "not implemented")
|
||||
}
|
||||
|
||||
@@ -207,7 +184,7 @@ func (provider *provider) Patch(_ context.Context, _ valuer.UUID, _ *authtypes.R
|
||||
return errors.Newf(errors.TypeUnsupported, authtypes.ErrCodeRoleUnsupported, "not implemented")
|
||||
}
|
||||
|
||||
func (provider *provider) PatchObjects(_ context.Context, _ valuer.UUID, _ string, _ authtypes.Relation, _, _ []*authtypes.Object) error {
|
||||
func (provider *provider) PatchObjects(_ context.Context, _ valuer.UUID, _ string, _ authtypes.Relation, _, _ []*coretypes.Object) error {
|
||||
return errors.Newf(errors.TypeUnsupported, authtypes.ErrCodeRoleUnsupported, "not implemented")
|
||||
}
|
||||
|
||||
@@ -220,7 +197,7 @@ func (provider *provider) CheckTransactions(ctx context.Context, subject string,
|
||||
return make([]*authtypes.TransactionWithAuthorization, 0), nil
|
||||
}
|
||||
|
||||
tuples, preResolved, roleCorrelations, err := authtypes.NewTuplesFromTransactionsWithManagedRoles(transactions, subject, orgID, provider.managedRolesByTransaction)
|
||||
tuples, preResolved, roleCorrelations, err := authtypes.NewTuplesFromTransactionsWithManagedRoles(transactions, subject, orgID, provider.registry.ManagedRolesByTransaction())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -236,21 +213,3 @@ func (provider *provider) CheckTransactions(ctx context.Context, subject string,
|
||||
|
||||
return authtypes.NewTransactionWithAuthorizationFromBatchResults(transactions, batchResults, preResolved, roleCorrelations), nil
|
||||
}
|
||||
|
||||
func buildManagedRolesByTransaction(registry []authz.RegisterTypeable) map[string][]string {
|
||||
managedRolesByTransaction := make(map[string][]string)
|
||||
for _, register := range registry {
|
||||
for roleName, transactions := range register.MustGetManagedRoleTransactions() {
|
||||
for _, txn := range transactions {
|
||||
key := txn.TransactionKey()
|
||||
managedRolesByTransaction[key] = append(managedRolesByTransaction[key], roleName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return managedRolesByTransaction
|
||||
}
|
||||
|
||||
func (provider *provider) MustGetTypeables() []authtypes.Typeable {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,15 +2,15 @@ module base
|
||||
|
||||
type user
|
||||
|
||||
type serviceaccount
|
||||
type serviceaccount
|
||||
|
||||
type role
|
||||
type role
|
||||
relations
|
||||
define assignee: [user, serviceaccount]
|
||||
|
||||
type organisation
|
||||
type organization
|
||||
relations
|
||||
define create: [role#assignee]
|
||||
define read: [role#assignee]
|
||||
define update: [role#assignee]
|
||||
define delete: [role#assignee]
|
||||
define delete: [role#assignee]
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
authz "github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
@@ -141,18 +142,18 @@ func (server *Server) BatchCheck(ctx context.Context, tupleReq map[string]*openf
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, _ authtypes.Relation, _ authtypes.Typeable, _ []authtypes.Selector, roleSelectors []authtypes.Selector) error {
|
||||
func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtypes.Claims, orgID valuer.UUID, _ authtypes.Relation, _ coretypes.Resource, _ []coretypes.Selector, roleSelectors []coretypes.Selector) error {
|
||||
subject := ""
|
||||
switch claims.Principal {
|
||||
case authtypes.PrincipalUser:
|
||||
user, err := authtypes.NewSubject(authtypes.TypeableUser, claims.UserID, orgID, nil)
|
||||
user, err := authtypes.NewSubject(coretypes.NewResourceUser(), claims.UserID, orgID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
subject = user
|
||||
case authtypes.PrincipalServiceAccount:
|
||||
serviceAccount, err := authtypes.NewSubject(authtypes.TypeableServiceAccount, claims.ServiceAccountID, orgID, nil)
|
||||
serviceAccount, err := authtypes.NewSubject(coretypes.NewResourceServiceAccount(), claims.ServiceAccountID, orgID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -160,10 +161,7 @@ func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtyp
|
||||
subject = serviceAccount
|
||||
}
|
||||
|
||||
tupleSlice, err := authtypes.TypeableRole.Tuples(subject, authtypes.RelationAssignee, roleSelectors, orgID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tupleSlice := authtypes.NewTuples(coretypes.NewResourceRole(), subject, authtypes.Relation{Verb: coretypes.VerbAssignee}, roleSelectors, orgID)
|
||||
|
||||
// Convert slice to map with generated IDs for internal use
|
||||
tuples := make(map[string]*openfgav1.TupleKey, len(tupleSlice))
|
||||
@@ -185,16 +183,13 @@ func (server *Server) CheckWithTupleCreation(ctx context.Context, claims authtyp
|
||||
return errors.Newf(errors.TypeForbidden, authtypes.ErrCodeAuthZForbidden, "subjects are not authorized for requested access")
|
||||
}
|
||||
|
||||
func (server *Server) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, _ authtypes.Relation, _ authtypes.Typeable, _ []authtypes.Selector, roleSelectors []authtypes.Selector) error {
|
||||
subject, err := authtypes.NewSubject(authtypes.TypeableAnonymous, authtypes.AnonymousUser.String(), orgID, nil)
|
||||
func (server *Server) CheckWithTupleCreationWithoutClaims(ctx context.Context, orgID valuer.UUID, _ authtypes.Relation, _ coretypes.Resource, _ []coretypes.Selector, roleSelectors []coretypes.Selector) error {
|
||||
subject, err := authtypes.NewSubject(coretypes.NewResourceAnonymous(), coretypes.AnonymousUser.String(), orgID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tupleSlice, err := authtypes.TypeableRole.Tuples(subject, authtypes.RelationAssignee, roleSelectors, orgID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tupleSlice := authtypes.NewTuples(coretypes.NewResourceRole(), subject, authtypes.Relation{Verb: coretypes.VerbAssignee}, roleSelectors, orgID)
|
||||
|
||||
// Convert slice to map with generated IDs for internal use
|
||||
tuples := make(map[string]*openfgav1.TupleKey, len(tupleSlice))
|
||||
@@ -293,7 +288,7 @@ func (server *Server) ReadTuples(ctx context.Context, tupleKey *openfgav1.ReadRe
|
||||
return tuples, nil
|
||||
}
|
||||
|
||||
func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType authtypes.Type) ([]*authtypes.Object, error) {
|
||||
func (server *Server) ListObjects(ctx context.Context, subject string, relation authtypes.Relation, objectType coretypes.Type) ([]*coretypes.Object, error) {
|
||||
storeID, modelID := server.getStoreIDandModelID()
|
||||
response, err := server.openfgaServer.ListObjects(ctx, &openfgav1.ListObjectsRequest{
|
||||
StoreId: storeID,
|
||||
@@ -306,7 +301,7 @@ func (server *Server) ListObjects(ctx context.Context, subject string, relation
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, authtypes.ErrCodeAuthZUnavailable, "cannot list objects for subject %s with relation %s for type %s", subject, relation.StringValue(), objectType.StringValue())
|
||||
}
|
||||
|
||||
return authtypes.MustNewObjectsFromStringSlice(response.Objects), nil
|
||||
return coretypes.MustNewObjectsFromStringSlice(response.Objects), nil
|
||||
}
|
||||
|
||||
func (server *Server) getOrCreateStore(ctx context.Context, name string) (string, error) {
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestProviderStartStop(t *testing.T) {
|
||||
providerSettings := instrumentationtest.New().ToProviderSettings()
|
||||
sqlstore := sqlstoretest.New(sqlstore.Config{Provider: "sqlite"}, sqlmock.QueryMatcherRegexp)
|
||||
|
||||
openfgaDataStore, err := NewSQLStore(sqlstore)
|
||||
openfgaDataStore, err := NewSQLStore(sqlstore, authz.Config{OpenFGA: authz.OpenFGAConfig{MaxTuplesPerWrite: 100}})
|
||||
require.NoError(t, err)
|
||||
|
||||
expectedModel := `module base
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package openfgaserver
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/openfga/openfga/pkg/storage"
|
||||
@@ -8,11 +9,11 @@ import (
|
||||
"github.com/openfga/openfga/pkg/storage/sqlite"
|
||||
)
|
||||
|
||||
func NewSQLStore(store sqlstore.SQLStore) (storage.OpenFGADatastore, error) {
|
||||
func NewSQLStore(store sqlstore.SQLStore, config authz.Config) (storage.OpenFGADatastore, error) {
|
||||
switch store.BunDB().Dialect().Name().String() {
|
||||
case "sqlite":
|
||||
return sqlite.NewWithDB(store.SQLDB(), &sqlcommon.Config{
|
||||
MaxTuplesPerWriteField: 100,
|
||||
MaxTuplesPerWriteField: config.OpenFGA.MaxTuplesPerWrite,
|
||||
MaxTypesPerModelField: 100,
|
||||
})
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/http/render"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
@@ -97,25 +98,20 @@ func (handler *handler) GetObjects(rw http.ResponseWriter, r *http.Request) {
|
||||
render.Error(rw, errors.New(errors.TypeInvalidInput, authtypes.ErrCodeRoleInvalidInput, "relation is missing from the request"))
|
||||
return
|
||||
}
|
||||
relation, err := authtypes.NewRelation(relationStr)
|
||||
|
||||
relation, err := coretypes.NewVerb(relationStr)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
objects, err := handler.authz.GetObjects(ctx, valuer.MustNewUUID(claims.OrgID), roleID, relation)
|
||||
objects, err := handler.authz.GetObjects(ctx, valuer.MustNewUUID(claims.OrgID), roleID, authtypes.Relation{Verb: relation})
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
render.Success(rw, http.StatusOK, authtypes.NewGettableObjects(objects))
|
||||
}
|
||||
|
||||
func (handler *handler) GetResources(rw http.ResponseWriter, r *http.Request) {
|
||||
resources := handler.authz.GetResources(r.Context())
|
||||
|
||||
render.Success(rw, http.StatusOK, authtypes.NewGettableResources(resources))
|
||||
render.Success(rw, http.StatusOK, coretypes.NewObjectGroupsFromObjects(objects))
|
||||
}
|
||||
|
||||
func (handler *handler) List(rw http.ResponseWriter, r *http.Request) {
|
||||
@@ -190,7 +186,7 @@ func (handler *handler) PatchObjects(rw http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
relation, err := authtypes.NewRelation(mux.Vars(r)["relation"])
|
||||
relation, err := coretypes.NewVerb(mux.Vars(r)["relation"])
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
@@ -207,19 +203,19 @@ func (handler *handler) PatchObjects(rw http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
req := new(authtypes.PatchableObjects)
|
||||
req := new(coretypes.PatchableObjects)
|
||||
if err := binding.JSON.BindBody(r.Body, req); err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
additions, deletions, err := authtypes.NewPatchableObjects(req.Additions, req.Deletions, relation)
|
||||
additions, deletions, err := coretypes.NewPatchableObjects(req.Additions, req.Deletions, relation)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
}
|
||||
|
||||
err = handler.authz.PatchObjects(ctx, valuer.MustNewUUID(claims.OrgID), role.Name, relation, additions, deletions)
|
||||
err = handler.authz.PatchObjects(ctx, valuer.MustNewUUID(claims.OrgID), role.Name, authtypes.Relation{Verb: relation}, additions, deletions)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
@@ -266,7 +262,7 @@ func (handler *handler) Check(rw http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
orgID := valuer.MustNewUUID(claims.OrgID)
|
||||
subject, err := authtypes.NewSubject(authtypes.TypeableUser, claims.UserID, orgID, nil)
|
||||
subject, err := authtypes.NewSubject(coretypes.NewResourceUser(), claims.UserID, orgID, nil)
|
||||
if err != nil {
|
||||
render.Error(rw, err)
|
||||
return
|
||||
|
||||
@@ -171,7 +171,7 @@ func TestAwaitHealthy(t *testing.T) {
|
||||
func TestAwaitHealthyWithFailure(t *testing.T) {
|
||||
s1 := &failingHealthyService{
|
||||
healthyC: make(chan struct{}),
|
||||
err: errors.Newf(errors.TypeInternal, errors.CodeInternal,"startup failed"),
|
||||
err: errors.Newf(errors.TypeInternal, errors.CodeInternal, "startup failed"),
|
||||
}
|
||||
|
||||
registry, err := NewRegistry(context.Background(), slog.New(slog.DiscardHandler), NewNamedService(MustNewName("s1"), s1))
|
||||
@@ -245,7 +245,7 @@ func TestDependsOnStartsAfterDependency(t *testing.T) {
|
||||
func TestDependsOnFailsWhenDependencyFails(t *testing.T) {
|
||||
s1 := &failingHealthyService{
|
||||
healthyC: make(chan struct{}),
|
||||
err: errors.Newf(errors.TypeInternal, errors.CodeInternal,"s1 crashed"),
|
||||
err: errors.Newf(errors.TypeInternal, errors.CodeInternal, "s1 crashed"),
|
||||
}
|
||||
s2 := newTestService(t)
|
||||
|
||||
@@ -291,7 +291,7 @@ func TestDependsOnUnknownServiceIsIgnored(t *testing.T) {
|
||||
func TestServiceStateFailed(t *testing.T) {
|
||||
s1 := &failingHealthyService{
|
||||
healthyC: make(chan struct{}),
|
||||
err: errors.Newf(errors.TypeInternal, errors.CodeInternal,"fatal error"),
|
||||
err: errors.Newf(errors.TypeInternal, errors.CodeInternal, "fatal error"),
|
||||
}
|
||||
|
||||
registry, err := NewRegistry(context.Background(), slog.New(slog.DiscardHandler), NewNamedService(MustNewName("s1"), s1))
|
||||
|
||||
@@ -8,6 +8,7 @@ var (
|
||||
FeatureHideRootUser = featuretypes.MustNewName("hide_root_user")
|
||||
FeatureGetMetersFromZeus = featuretypes.MustNewName("get_meters_from_zeus")
|
||||
FeaturePutMetersInZeus = featuretypes.MustNewName("put_meters_in_zeus")
|
||||
FeatureUseMeterReporter = featuretypes.MustNewName("use_meter_reporter")
|
||||
FeatureUseJSONBody = featuretypes.MustNewName("use_json_body")
|
||||
)
|
||||
|
||||
@@ -53,6 +54,14 @@ func MustNewRegistry() featuretypes.Registry {
|
||||
DefaultVariant: featuretypes.MustNewName("disabled"),
|
||||
Variants: featuretypes.NewBooleanVariants(),
|
||||
},
|
||||
&featuretypes.Feature{
|
||||
Name: FeatureUseMeterReporter,
|
||||
Kind: featuretypes.KindBoolean,
|
||||
Stage: featuretypes.StageExperimental,
|
||||
Description: "Controls whether the enterprise meter reporter runs instead of the noop reporter",
|
||||
DefaultVariant: featuretypes.MustNewName("disabled"),
|
||||
Variants: featuretypes.NewBooleanVariants(),
|
||||
},
|
||||
&featuretypes.Feature{
|
||||
Name: FeatureUseJSONBody,
|
||||
Kind: featuretypes.KindBoolean,
|
||||
|
||||
@@ -15,12 +15,24 @@ func TestQueryBinding_BindQuery(t *testing.T) {
|
||||
obj any
|
||||
expected any
|
||||
}{
|
||||
{name: "SingleIntField_NonEmptyValue", query: map[string][]string{"a": {"1"}}, obj: &struct{A int `query:"a"`}{}, expected: &struct{ A int }{A: 1}},
|
||||
{name: "SingleIntField_EmptyValue", query: map[string][]string{"a": {""}}, obj: &struct{A int `query:"a"`}{}, expected: &struct{ A int }{A: 0}},
|
||||
{name: "SingleIntField_MissingField", query: map[string][]string{}, obj: &struct{A int `query:"a"`}{}, expected: &struct{ A int }{A: 0}},
|
||||
{name: "SinglePointerIntField_NonEmptyValue", query: map[string][]string{"a": {"1"}}, obj: &struct{A *int `query:"a"`}{}, expected: &struct{ A *int }{A: &one}},
|
||||
{name: "SinglePointerIntField_EmptyValue", query: map[string][]string{"a": {""}}, obj: &struct{A *int `query:"a"`}{}, expected: &struct{ A *int }{A: &zero}},
|
||||
{name: "SinglePointerIntField_MissingField", query: map[string][]string{}, obj: &struct{A *int `query:"a"`}{}, expected: &struct{ A *int }{A: nil}},
|
||||
{name: "SingleIntField_NonEmptyValue", query: map[string][]string{"a": {"1"}}, obj: &struct {
|
||||
A int `query:"a"`
|
||||
}{}, expected: &struct{ A int }{A: 1}},
|
||||
{name: "SingleIntField_EmptyValue", query: map[string][]string{"a": {""}}, obj: &struct {
|
||||
A int `query:"a"`
|
||||
}{}, expected: &struct{ A int }{A: 0}},
|
||||
{name: "SingleIntField_MissingField", query: map[string][]string{}, obj: &struct {
|
||||
A int `query:"a"`
|
||||
}{}, expected: &struct{ A int }{A: 0}},
|
||||
{name: "SinglePointerIntField_NonEmptyValue", query: map[string][]string{"a": {"1"}}, obj: &struct {
|
||||
A *int `query:"a"`
|
||||
}{}, expected: &struct{ A *int }{A: &one}},
|
||||
{name: "SinglePointerIntField_EmptyValue", query: map[string][]string{"a": {""}}, obj: &struct {
|
||||
A *int `query:"a"`
|
||||
}{}, expected: &struct{ A *int }{A: &zero}},
|
||||
{name: "SinglePointerIntField_MissingField", query: map[string][]string{}, obj: &struct {
|
||||
A *int `query:"a"`
|
||||
}{}, expected: &struct{ A *int }{A: nil}},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
|
||||
@@ -2,14 +2,15 @@ package handler
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/types/audittypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
)
|
||||
|
||||
// Option configures optional behaviour on a handler created by New.
|
||||
type Option func(*handler)
|
||||
|
||||
type AuditDef struct {
|
||||
ResourceKind string // AuthZ Typeable.Kind() value, e.g. "dashboard", "user".
|
||||
Action audittypes.Action // create, update, delete, login, etc.
|
||||
ResourceKind coretypes.Kind // Typeable.Kind() value, e.g. "dashboard", "user".
|
||||
Action coretypes.Verb // create, update, delete, etc.
|
||||
Category audittypes.ActionCategory // access_control, configuration_change, etc.
|
||||
ResourceIDParam string // Gorilla mux path param name for the resource ID.
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/http/render"
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
@@ -40,18 +41,18 @@ func (middleware *AuthZ) ViewAccess(next http.HandlerFunc) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
selectors := []authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAdminRoleName),
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozEditorRoleName),
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozViewerRoleName),
|
||||
selectors := []coretypes.Selector{
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozAdminRoleName),
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozEditorRoleName),
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozViewerRoleName),
|
||||
}
|
||||
|
||||
err = middleware.authzService.CheckWithTupleCreation(
|
||||
ctx,
|
||||
claims,
|
||||
valuer.MustNewUUID(claims.OrgID),
|
||||
authtypes.RelationAssignee,
|
||||
authtypes.TypeableRole,
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
coretypes.NewResourceRole(),
|
||||
selectors,
|
||||
selectors,
|
||||
)
|
||||
@@ -79,17 +80,17 @@ func (middleware *AuthZ) EditAccess(next http.HandlerFunc) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
selectors := []authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAdminRoleName),
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozEditorRoleName),
|
||||
selectors := []coretypes.Selector{
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozAdminRoleName),
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozEditorRoleName),
|
||||
}
|
||||
|
||||
err = middleware.authzService.CheckWithTupleCreation(
|
||||
ctx,
|
||||
claims,
|
||||
valuer.MustNewUUID(claims.OrgID),
|
||||
authtypes.RelationAssignee,
|
||||
authtypes.TypeableRole,
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
coretypes.NewResourceRole(),
|
||||
selectors,
|
||||
selectors,
|
||||
)
|
||||
@@ -117,16 +118,16 @@ func (middleware *AuthZ) AdminAccess(next http.HandlerFunc) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
selectors := []authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAdminRoleName),
|
||||
selectors := []coretypes.Selector{
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozAdminRoleName),
|
||||
}
|
||||
|
||||
err = middleware.authzService.CheckWithTupleCreation(
|
||||
ctx,
|
||||
claims,
|
||||
valuer.MustNewUUID(claims.OrgID),
|
||||
authtypes.RelationAssignee,
|
||||
authtypes.TypeableRole,
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
coretypes.NewResourceRole(),
|
||||
selectors,
|
||||
selectors,
|
||||
)
|
||||
@@ -153,16 +154,16 @@ func (middleware *AuthZ) SelfAccess(next http.HandlerFunc) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
selectors := []authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAdminRoleName),
|
||||
selectors := []coretypes.Selector{
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozAdminRoleName),
|
||||
}
|
||||
|
||||
err = middleware.authzService.CheckWithTupleCreation(
|
||||
req.Context(),
|
||||
claims,
|
||||
valuer.MustNewUUID(claims.OrgID),
|
||||
authtypes.RelationAssignee,
|
||||
authtypes.TypeableRole,
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
coretypes.NewResourceRole(),
|
||||
selectors,
|
||||
selectors,
|
||||
)
|
||||
@@ -185,7 +186,7 @@ func (middleware *AuthZ) OpenAccess(next http.HandlerFunc) http.HandlerFunc {
|
||||
})
|
||||
}
|
||||
|
||||
func (middleware *AuthZ) Check(next http.HandlerFunc, relation authtypes.Relation, typeable authtypes.Typeable, cb authtypes.SelectorCallbackWithClaimsFn, roles []string) http.HandlerFunc {
|
||||
func (middleware *AuthZ) Check(next http.HandlerFunc, relation authtypes.Relation, typeable coretypes.Resource, cb authtypes.SelectorCallbackWithClaimsFn, roles []string) http.HandlerFunc {
|
||||
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
claims, err := authtypes.ClaimsFromContext(ctx)
|
||||
@@ -200,9 +201,9 @@ func (middleware *AuthZ) Check(next http.HandlerFunc, relation authtypes.Relatio
|
||||
return
|
||||
}
|
||||
|
||||
roleSelectors := []authtypes.Selector{}
|
||||
roleSelectors := []coretypes.Selector{}
|
||||
for _, role := range roles {
|
||||
roleSelectors = append(roleSelectors, authtypes.MustNewSelector(authtypes.TypeRole, role))
|
||||
roleSelectors = append(roleSelectors, coretypes.TypeRole.MustSelector(role))
|
||||
}
|
||||
|
||||
err = middleware.authzService.CheckWithTupleCreation(ctx, claims, valuer.MustNewUUID(claims.OrgID), relation, typeable, selectors, roleSelectors)
|
||||
@@ -215,7 +216,7 @@ func (middleware *AuthZ) Check(next http.HandlerFunc, relation authtypes.Relatio
|
||||
})
|
||||
}
|
||||
|
||||
func (middleware *AuthZ) CheckWithoutClaims(next http.HandlerFunc, relation authtypes.Relation, typeable authtypes.Typeable, cb authtypes.SelectorCallbackWithoutClaimsFn, roles []string) http.HandlerFunc {
|
||||
func (middleware *AuthZ) CheckWithoutClaims(next http.HandlerFunc, relation authtypes.Relation, typeable coretypes.Resource, cb authtypes.SelectorCallbackWithoutClaimsFn, roles []string) http.HandlerFunc {
|
||||
return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
|
||||
ctx := req.Context()
|
||||
orgs, err := middleware.orgGetter.ListByOwnedKeyRange(ctx)
|
||||
@@ -230,9 +231,9 @@ func (middleware *AuthZ) CheckWithoutClaims(next http.HandlerFunc, relation auth
|
||||
return
|
||||
}
|
||||
|
||||
roleSelectors := []authtypes.Selector{}
|
||||
roleSelectors := []coretypes.Selector{}
|
||||
for _, role := range roles {
|
||||
roleSelectors = append(roleSelectors, authtypes.MustNewSelector(authtypes.TypeRole, role))
|
||||
roleSelectors = append(roleSelectors, coretypes.TypeRole.MustSelector(role))
|
||||
}
|
||||
|
||||
err = middleware.authzService.CheckWithTupleCreationWithoutClaims(ctx, orgId, relation, typeable, selectors, roleSelectors)
|
||||
|
||||
18
pkg/metercollector/metercollector.go
Normal file
18
pkg/metercollector/metercollector.go
Normal file
@@ -0,0 +1,18 @@
|
||||
// Package metercollector defines the contract for billing meter collectors.
|
||||
package metercollector
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// MeterCollector owns one billing meter's metadata and collection query.
|
||||
// Collect stamps zeustypes.MeterDimensionOrganizationID and returns errors instead of panics.
|
||||
type MeterCollector interface {
|
||||
Name() zeustypes.MeterName
|
||||
Unit() zeustypes.MeterUnit
|
||||
Aggregation() zeustypes.MeterAggregation
|
||||
Collect(ctx context.Context, orgID valuer.UUID, window zeustypes.MeterWindow) ([]zeustypes.Meter, error)
|
||||
}
|
||||
53
pkg/meterreporter/config.go
Normal file
53
pkg/meterreporter/config.go
Normal file
@@ -0,0 +1,53 @@
|
||||
package meterreporter
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
)
|
||||
|
||||
var _ factory.Config = (*Config)(nil)
|
||||
|
||||
type Config struct {
|
||||
// Interval is how often the reporter collects and ships meters.
|
||||
Interval time.Duration `mapstructure:"interval"`
|
||||
|
||||
// Timeout bounds one collect-and-ship tick.
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
|
||||
// CatchupMaxDaysPerTick caps sealed-day catchup work per tick.
|
||||
CatchupMaxDaysPerTick int `mapstructure:"catchup_max_days_per_tick"`
|
||||
}
|
||||
|
||||
func newConfig() factory.Config {
|
||||
return Config{
|
||||
Interval: 6 * time.Hour,
|
||||
Timeout: 5 * time.Minute,
|
||||
CatchupMaxDaysPerTick: 180,
|
||||
}
|
||||
}
|
||||
|
||||
func NewConfigFactory() factory.ConfigFactory {
|
||||
return factory.NewConfigFactory(factory.MustNewName("meterreporter"), newConfig)
|
||||
}
|
||||
|
||||
func (c Config) Validate() error {
|
||||
if c.Interval < 5*time.Minute {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "meterreporter::interval must be at least 5m")
|
||||
}
|
||||
|
||||
if c.Timeout < 3*time.Minute {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "meterreporter::timeout must be at least 3m")
|
||||
}
|
||||
|
||||
if c.Timeout >= c.Interval {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "meterreporter::timeout must be less than meterreporter::interval")
|
||||
}
|
||||
|
||||
if c.CatchupMaxDaysPerTick < 1 || c.CatchupMaxDaysPerTick > 180 {
|
||||
return errors.New(errors.TypeInvalidInput, ErrCodeInvalidInput, "meterreporter::catchup_max_days_per_tick must be between 1 and 180")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
14
pkg/meterreporter/meterreporter.go
Normal file
14
pkg/meterreporter/meterreporter.go
Normal file
@@ -0,0 +1,14 @@
|
||||
package meterreporter
|
||||
|
||||
import (
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrCodeInvalidInput = errors.MustNewCode("meterreporter_invalid_input")
|
||||
)
|
||||
|
||||
type Reporter interface {
|
||||
factory.ServiceWithHealthy
|
||||
}
|
||||
39
pkg/meterreporter/noopmeterreporter/provider.go
Normal file
39
pkg/meterreporter/noopmeterreporter/provider.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package noopmeterreporter
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/factory"
|
||||
"github.com/SigNoz/signoz/pkg/meterreporter"
|
||||
)
|
||||
|
||||
type provider struct {
|
||||
healthyC chan struct{}
|
||||
stopC chan struct{}
|
||||
}
|
||||
|
||||
func NewFactory() factory.ProviderFactory[meterreporter.Reporter, meterreporter.Config] {
|
||||
return factory.NewProviderFactory(factory.MustNewName("noop"), New)
|
||||
}
|
||||
|
||||
func New(_ context.Context, _ factory.ProviderSettings, _ meterreporter.Config) (meterreporter.Reporter, error) {
|
||||
return &provider{
|
||||
healthyC: make(chan struct{}),
|
||||
stopC: make(chan struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *provider) Start(_ context.Context) error {
|
||||
close(p.healthyC)
|
||||
<-p.stopC
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *provider) Stop(_ context.Context) error {
|
||||
close(p.stopC)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *provider) Healthy() <-chan struct{} {
|
||||
return p.healthyC
|
||||
}
|
||||
@@ -4,10 +4,9 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/authz"
|
||||
"github.com/SigNoz/signoz/pkg/statsreporter"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -27,7 +26,7 @@ type Module interface {
|
||||
GetPublicWidgetQueryRange(context.Context, valuer.UUID, uint64, uint64, uint64) (*querybuildertypesv5.QueryRangeResponse, error)
|
||||
|
||||
// gets the selectors and org for the given public dashboard
|
||||
GetPublicDashboardSelectorsAndOrg(context.Context, valuer.UUID, []*types.Organization) ([]authtypes.Selector, valuer.UUID, error)
|
||||
GetPublicDashboardSelectorsAndOrg(context.Context, valuer.UUID, []*types.Organization) ([]coretypes.Selector, valuer.UUID, error)
|
||||
|
||||
// updates the public sharing config for a dashboard
|
||||
UpdatePublic(context.Context, valuer.UUID, *dashboardtypes.PublicDashboard) error
|
||||
@@ -50,8 +49,6 @@ type Module interface {
|
||||
GetByMetricNames(ctx context.Context, orgID valuer.UUID, metricNames []string) (map[string][]map[string]string, error)
|
||||
|
||||
statsreporter.StatsCollector
|
||||
|
||||
authz.RegisterTypeable
|
||||
}
|
||||
|
||||
type Handler interface {
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/transition"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/gorilla/mux"
|
||||
@@ -158,15 +159,15 @@ func (handler *handler) LockUnlock(rw http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
isAdmin := false
|
||||
selectors := []authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAdminRoleName),
|
||||
selectors := []coretypes.Selector{
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozAdminRoleName),
|
||||
}
|
||||
err = handler.authz.CheckWithTupleCreation(
|
||||
ctx,
|
||||
claims,
|
||||
valuer.MustNewUUID(claims.OrgID),
|
||||
authtypes.RelationAssignee,
|
||||
authtypes.TypeableRole,
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
coretypes.NewResourceRole(),
|
||||
selectors,
|
||||
selectors,
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/organization"
|
||||
"github.com/SigNoz/signoz/pkg/queryparser"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/dashboardtypes"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -202,14 +202,6 @@ func (module *module) Collect(ctx context.Context, orgID valuer.UUID) (map[strin
|
||||
return dashboardtypes.NewStatsFromStorableDashboards(dashboards), nil
|
||||
}
|
||||
|
||||
func (module *module) MustGetTypeables() []authtypes.Typeable {
|
||||
return []authtypes.Typeable{dashboardtypes.TypeableMetaResourceDashboard, dashboardtypes.TypeableMetaResourcesDashboards}
|
||||
}
|
||||
|
||||
func (module *module) MustGetManagedRoleTransactions() map[string][]*authtypes.Transaction {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreatePublic is not supported.
|
||||
func (module *module) CreatePublic(ctx context.Context, orgID valuer.UUID, publicDashboard *dashboardtypes.PublicDashboard) error {
|
||||
return errors.Newf(errors.TypeUnsupported, dashboardtypes.ErrCodePublicDashboardUnsupported, "not implemented")
|
||||
@@ -227,7 +219,7 @@ func (module *module) GetPublicWidgetQueryRange(context.Context, valuer.UUID, ui
|
||||
return nil, errors.Newf(errors.TypeUnsupported, dashboardtypes.ErrCodePublicDashboardUnsupported, "not implemented")
|
||||
}
|
||||
|
||||
func (module *module) GetPublicDashboardSelectorsAndOrg(_ context.Context, _ valuer.UUID, _ []*types.Organization) ([]authtypes.Selector, valuer.UUID, error) {
|
||||
func (module *module) GetPublicDashboardSelectorsAndOrg(_ context.Context, _ valuer.UUID, _ []*types.Organization) ([]coretypes.Selector, valuer.UUID, error) {
|
||||
return nil, valuer.UUID{}, errors.Newf(errors.TypeUnsupported, dashboardtypes.ErrCodePublicDashboardUnsupported, "not implemented")
|
||||
}
|
||||
|
||||
|
||||
248
pkg/modules/retention/implretention/getter.go
Normal file
248
pkg/modules/retention/implretention/getter.go
Normal file
@@ -0,0 +1,248 @@
|
||||
package implretention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/modules/retention"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
const secondsPerDay = 24 * 60 * 60
|
||||
|
||||
var (
|
||||
labelKeyPattern = regexp.MustCompile(`^[A-Za-z0-9_.\-]+$`)
|
||||
labelValuePattern = regexp.MustCompile(`^[A-Za-z0-9_.\-:]+$`)
|
||||
)
|
||||
|
||||
type getter struct {
|
||||
store retentiontypes.Store
|
||||
}
|
||||
|
||||
// NewGetter creates a retention getter backed by the retention store.
|
||||
func NewGetter(store retentiontypes.Store) retention.Getter {
|
||||
return &getter{
|
||||
store: store,
|
||||
}
|
||||
}
|
||||
|
||||
// ActiveSlices loads successful TTL changes and converts them into meter windows.
|
||||
func (getter *getter) ActiveSlices(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
dbName string,
|
||||
tableName string,
|
||||
fallbackDefaultDays int,
|
||||
startMs int64,
|
||||
endMs int64,
|
||||
) ([]retentiontypes.Slice, error) {
|
||||
if startMs >= endMs {
|
||||
return nil, nil
|
||||
}
|
||||
if dbName == "" {
|
||||
return nil, errors.New(errors.TypeInvalidInput, zeustypes.ErrCodeMeterCollectFailed, "dbName is empty")
|
||||
}
|
||||
if tableName == "" {
|
||||
return nil, errors.New(errors.TypeInvalidInput, zeustypes.ErrCodeMeterCollectFailed, "tableName is empty")
|
||||
}
|
||||
if fallbackDefaultDays <= 0 {
|
||||
return nil, errors.Newf(errors.TypeInvalidInput, zeustypes.ErrCodeMeterCollectFailed, "non-positive fallbackDefaultDays %d", fallbackDefaultDays)
|
||||
}
|
||||
|
||||
rows, err := getter.store.ListTTLSettings(ctx, orgID, dbName+"."+tableName, endMs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buildSlicesFromRows(rows, fallbackDefaultDays, startMs, endMs)
|
||||
}
|
||||
|
||||
func buildSlicesFromRows(rows []*retentiontypes.TTLSetting, fallbackDefaultDays int, startMs, endMs int64) ([]retentiontypes.Slice, error) {
|
||||
if startMs >= endMs {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var activeAtStart *retentiontypes.TTLSetting
|
||||
inWindow := make([]*retentiontypes.TTLSetting, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
rowMs := row.CreatedAt.UnixMilli()
|
||||
if rowMs <= startMs {
|
||||
activeAtStart = row
|
||||
continue
|
||||
}
|
||||
if rowMs >= endMs {
|
||||
continue
|
||||
}
|
||||
inWindow = append(inWindow, row)
|
||||
}
|
||||
|
||||
activeRules, activeDefault, err := parseTTLSetting(activeAtStart, fallbackDefaultDays)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
slices := make([]retentiontypes.Slice, 0, len(inWindow)+1)
|
||||
cursor := startMs
|
||||
for _, row := range inWindow {
|
||||
rowMs := row.CreatedAt.UnixMilli()
|
||||
if rowMs <= cursor {
|
||||
activeRules, activeDefault, err = parseTTLSetting(row, fallbackDefaultDays)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
slices = append(slices, retentiontypes.Slice{
|
||||
StartMs: cursor,
|
||||
EndMs: rowMs,
|
||||
Rules: activeRules,
|
||||
DefaultDays: activeDefault,
|
||||
})
|
||||
cursor = rowMs
|
||||
activeRules, activeDefault, err = parseTTLSetting(row, fallbackDefaultDays)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if cursor < endMs {
|
||||
slices = append(slices, retentiontypes.Slice{
|
||||
StartMs: cursor,
|
||||
EndMs: endMs,
|
||||
Rules: activeRules,
|
||||
DefaultDays: activeDefault,
|
||||
})
|
||||
}
|
||||
|
||||
return slices, nil
|
||||
}
|
||||
|
||||
func parseTTLSetting(row *retentiontypes.TTLSetting, fallbackDefaultDays int) ([]retentiontypes.CustomRetentionRule, int, error) {
|
||||
if row == nil {
|
||||
return nil, fallbackDefaultDays, nil
|
||||
}
|
||||
|
||||
defaultDays := row.TTL
|
||||
if row.Condition == "" {
|
||||
defaultDays = (row.TTL + secondsPerDay - 1) / secondsPerDay
|
||||
}
|
||||
if defaultDays <= 0 {
|
||||
defaultDays = fallbackDefaultDays
|
||||
}
|
||||
|
||||
if row.Condition == "" {
|
||||
return nil, defaultDays, nil
|
||||
}
|
||||
|
||||
var rules []retentiontypes.CustomRetentionRule
|
||||
if err := json.Unmarshal([]byte(row.Condition), &rules); err != nil {
|
||||
return nil, 0, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "parse ttl_setting condition for row %q", row.ID.StringValue())
|
||||
}
|
||||
|
||||
return rules, defaultDays, nil
|
||||
}
|
||||
|
||||
// BuildMultiIfSQL builds the retention-days expression used in collector queries.
|
||||
func (getter *getter) BuildMultiIfSQL(rules []retentiontypes.CustomRetentionRule, defaultDays int) (string, error) {
|
||||
if defaultDays <= 0 {
|
||||
return "", errors.Newf(errors.TypeInvalidInput, zeustypes.ErrCodeMeterCollectFailed, "non-positive default retention %d", defaultDays)
|
||||
}
|
||||
|
||||
if len(rules) == 0 {
|
||||
return "toInt32(" + strconv.Itoa(defaultDays) + ")", nil
|
||||
}
|
||||
|
||||
arms := make([]string, 0, 2*len(rules)+1)
|
||||
for ruleIndex, rule := range rules {
|
||||
if rule.TTLDays <= 0 {
|
||||
return "", errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "rule %d has non-positive ttl_days %d", ruleIndex, rule.TTLDays)
|
||||
}
|
||||
conditionExpr, err := buildRuleConditionSQL(ruleIndex, rule)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
arms = append(arms, conditionExpr)
|
||||
arms = append(arms, strconv.Itoa(rule.TTLDays))
|
||||
}
|
||||
arms = append(arms, strconv.Itoa(defaultDays))
|
||||
|
||||
return "toInt32(multiIf(" + strings.Join(arms, ", ") + "))", nil
|
||||
}
|
||||
|
||||
// BuildRuleIndexSQL builds the matched-rule expression, using -1 for fallback.
|
||||
func (getter *getter) BuildRuleIndexSQL(rules []retentiontypes.CustomRetentionRule) (string, error) {
|
||||
if len(rules) == 0 {
|
||||
return "toInt32(-1)", nil
|
||||
}
|
||||
|
||||
arms := make([]string, 0, 2*len(rules)+1)
|
||||
for ruleIndex, rule := range rules {
|
||||
conditionExpr, err := buildRuleConditionSQL(ruleIndex, rule)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
arms = append(arms, conditionExpr)
|
||||
arms = append(arms, strconv.Itoa(ruleIndex))
|
||||
}
|
||||
arms = append(arms, "-1")
|
||||
|
||||
return "toInt32(multiIf(" + strings.Join(arms, ", ") + "))", nil
|
||||
}
|
||||
|
||||
func buildRuleConditionSQL(ruleIndex int, rule retentiontypes.CustomRetentionRule) (string, error) {
|
||||
if len(rule.Filters) == 0 {
|
||||
return "", errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "rule %d has no filters", ruleIndex)
|
||||
}
|
||||
|
||||
filterExprs := make([]string, 0, len(rule.Filters))
|
||||
for filterIndex, filter := range rule.Filters {
|
||||
if !labelKeyPattern.MatchString(filter.Key) {
|
||||
return "", errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "rule %d filter %d has invalid key %q", ruleIndex, filterIndex, filter.Key)
|
||||
}
|
||||
if len(filter.Values) == 0 {
|
||||
return "", errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "rule %d filter %d has no values", ruleIndex, filterIndex)
|
||||
}
|
||||
|
||||
quoted := make([]string, len(filter.Values))
|
||||
for valueIndex, value := range filter.Values {
|
||||
if !labelValuePattern.MatchString(value) {
|
||||
return "", errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "rule %d filter %d value %d is invalid %q", ruleIndex, filterIndex, valueIndex, value)
|
||||
}
|
||||
quoted[valueIndex] = "'" + value + "'"
|
||||
}
|
||||
|
||||
filterExprs = append(filterExprs, fmt.Sprintf("JSONExtractString(labels, '%s') IN (%s)", filter.Key, strings.Join(quoted, ", ")))
|
||||
}
|
||||
|
||||
return strings.Join(filterExprs, " AND "), nil
|
||||
}
|
||||
|
||||
// RuleDimensionKeys lists labels needed to report custom-retention dimensions.
|
||||
func (getter *getter) RuleDimensionKeys(rules []retentiontypes.CustomRetentionRule) ([]string, error) {
|
||||
keys := make([]string, 0)
|
||||
seen := make(map[string]struct{})
|
||||
|
||||
for ruleIndex, rule := range rules {
|
||||
for filterIndex, filter := range rule.Filters {
|
||||
if !labelKeyPattern.MatchString(filter.Key) {
|
||||
return nil, errors.Newf(errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "rule %d filter %d has invalid key %q", ruleIndex, filterIndex, filter.Key)
|
||||
}
|
||||
if _, ok := seen[filter.Key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[filter.Key] = struct{}{}
|
||||
keys = append(keys, filter.Key)
|
||||
}
|
||||
}
|
||||
|
||||
return keys, nil
|
||||
}
|
||||
167
pkg/modules/retention/implretention/getter_test.go
Normal file
167
pkg/modules/retention/implretention/getter_test.go
Normal file
@@ -0,0 +1,167 @@
|
||||
package implretention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildSlicesFromRows(t *testing.T) {
|
||||
start := time.Date(2026, 5, 4, 0, 0, 0, 0, time.UTC)
|
||||
end := start.AddDate(0, 0, 1)
|
||||
|
||||
ruleA := retentiontypes.CustomRetentionRule{
|
||||
Filters: []retentiontypes.FilterCondition{{Key: "service.name", Values: []string{"api"}}},
|
||||
TTLDays: 7,
|
||||
}
|
||||
ruleB := retentiontypes.CustomRetentionRule{
|
||||
Filters: []retentiontypes.FilterCondition{{Key: "env", Values: []string{"prod"}}},
|
||||
TTLDays: 15,
|
||||
}
|
||||
|
||||
t.Run("row before window is active at start", func(t *testing.T) {
|
||||
slices, err := buildSlicesFromRows(
|
||||
[]*retentiontypes.TTLSetting{
|
||||
ttlSetting(t, start.Add(-time.Hour), 45, []retentiontypes.CustomRetentionRule{ruleA}),
|
||||
},
|
||||
30,
|
||||
start.UnixMilli(),
|
||||
end.UnixMilli(),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []retentiontypes.Slice{{
|
||||
StartMs: start.UnixMilli(),
|
||||
EndMs: end.UnixMilli(),
|
||||
Rules: []retentiontypes.CustomRetentionRule{ruleA},
|
||||
DefaultDays: 45,
|
||||
}}, slices)
|
||||
})
|
||||
|
||||
t.Run("row inside window splits slices", func(t *testing.T) {
|
||||
firstChange := start.Add(6 * time.Hour)
|
||||
secondChange := start.Add(18 * time.Hour)
|
||||
|
||||
slices, err := buildSlicesFromRows(
|
||||
[]*retentiontypes.TTLSetting{
|
||||
ttlSetting(t, firstChange, 21, []retentiontypes.CustomRetentionRule{ruleA}),
|
||||
ttlSetting(t, secondChange, 14, []retentiontypes.CustomRetentionRule{ruleB}),
|
||||
},
|
||||
30,
|
||||
start.UnixMilli(),
|
||||
end.UnixMilli(),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []retentiontypes.Slice{
|
||||
{
|
||||
StartMs: start.UnixMilli(),
|
||||
EndMs: firstChange.UnixMilli(),
|
||||
DefaultDays: 30,
|
||||
},
|
||||
{
|
||||
StartMs: firstChange.UnixMilli(),
|
||||
EndMs: secondChange.UnixMilli(),
|
||||
Rules: []retentiontypes.CustomRetentionRule{ruleA},
|
||||
DefaultDays: 21,
|
||||
},
|
||||
{
|
||||
StartMs: secondChange.UnixMilli(),
|
||||
EndMs: end.UnixMilli(),
|
||||
Rules: []retentiontypes.CustomRetentionRule{ruleB},
|
||||
DefaultDays: 14,
|
||||
},
|
||||
}, slices)
|
||||
})
|
||||
|
||||
t.Run("no rows uses fallback", func(t *testing.T) {
|
||||
slices, err := buildSlicesFromRows(nil, 30, start.UnixMilli(), end.UnixMilli())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []retentiontypes.Slice{{
|
||||
StartMs: start.UnixMilli(),
|
||||
EndMs: end.UnixMilli(),
|
||||
DefaultDays: 30,
|
||||
}}, slices)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRetentionSQL(t *testing.T) {
|
||||
retentionGetter := NewGetter(noopStore{})
|
||||
rules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service.name",
|
||||
Values: []string{"api", "worker"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
|
||||
retentionSQL, err := retentionGetter.BuildMultiIfSQL(rules, 30)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "toInt32(multiIf(JSONExtractString(labels, 'service.name') IN ('api', 'worker'), 7, 30))", retentionSQL)
|
||||
|
||||
ruleIndexSQL, err := retentionGetter.BuildRuleIndexSQL(rules)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "toInt32(multiIf(JSONExtractString(labels, 'service.name') IN ('api', 'worker'), 0, -1))", ruleIndexSQL)
|
||||
|
||||
invalidRules := []retentiontypes.CustomRetentionRule{{
|
||||
Filters: []retentiontypes.FilterCondition{{
|
||||
Key: "service name",
|
||||
Values: []string{"api"},
|
||||
}},
|
||||
TTLDays: 7,
|
||||
}}
|
||||
|
||||
_, err = retentionGetter.BuildMultiIfSQL(invalidRules, 30)
|
||||
require.Error(t, err)
|
||||
|
||||
_, err = retentionGetter.BuildRuleIndexSQL(invalidRules)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestRuleDimensionKeysDedupes(t *testing.T) {
|
||||
retentionGetter := NewGetter(noopStore{})
|
||||
|
||||
keys, err := retentionGetter.RuleDimensionKeys([]retentiontypes.CustomRetentionRule{
|
||||
{
|
||||
Filters: []retentiontypes.FilterCondition{
|
||||
{Key: "service.name", Values: []string{"api"}},
|
||||
{Key: "env", Values: []string{"prod"}},
|
||||
},
|
||||
TTLDays: 7,
|
||||
},
|
||||
{
|
||||
Filters: []retentiontypes.FilterCondition{
|
||||
{Key: "service.name", Values: []string{"worker"}},
|
||||
{Key: "cluster", Values: []string{"primary"}},
|
||||
},
|
||||
TTLDays: 15,
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{"service.name", "env", "cluster"}, keys)
|
||||
}
|
||||
|
||||
func ttlSetting(t *testing.T, createdAt time.Time, ttlDays int, rules []retentiontypes.CustomRetentionRule) *retentiontypes.TTLSetting {
|
||||
t.Helper()
|
||||
|
||||
condition, err := json.Marshal(rules)
|
||||
require.NoError(t, err)
|
||||
|
||||
return &retentiontypes.TTLSetting{
|
||||
TimeAuditable: types.TimeAuditable{
|
||||
CreatedAt: createdAt,
|
||||
},
|
||||
TTL: ttlDays,
|
||||
Condition: string(condition),
|
||||
}
|
||||
}
|
||||
|
||||
type noopStore struct{}
|
||||
|
||||
func (noopStore) ListTTLSettings(context.Context, valuer.UUID, string, int64) ([]*retentiontypes.TTLSetting, error) {
|
||||
return nil, nil
|
||||
}
|
||||
42
pkg/modules/retention/implretention/store.go
Normal file
42
pkg/modules/retention/implretention/store.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package implretention
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/sqlstore"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/zeustypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
type store struct {
|
||||
sqlstore sqlstore.SQLStore
|
||||
}
|
||||
|
||||
// NewStore creates a SQL-backed retention store.
|
||||
func NewStore(sqlstore sqlstore.SQLStore) retentiontypes.Store {
|
||||
return &store{sqlstore: sqlstore}
|
||||
}
|
||||
|
||||
// ListTTLSettings returns successful TTL settings before the given timestamp.
|
||||
func (store *store) ListTTLSettings(ctx context.Context, orgID valuer.UUID, tableName string, beforeMs int64) ([]*retentiontypes.TTLSetting, error) {
|
||||
rows := []*retentiontypes.TTLSetting{}
|
||||
err := store.
|
||||
sqlstore.
|
||||
BunDB().
|
||||
NewSelect().
|
||||
Model(&rows).
|
||||
Where("table_name = ?", tableName).
|
||||
Where("org_id = ?", orgID.StringValue()).
|
||||
Where("status = ?", retentiontypes.TTLSettingStatusSuccess).
|
||||
Where("created_at < ?", time.UnixMilli(beforeMs).UTC()).
|
||||
OrderExpr("created_at ASC").
|
||||
Scan(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, errors.TypeInternal, zeustypes.ErrCodeMeterCollectFailed, "load ttl_setting rows for org %q table %q", orgID.StringValue(), tableName)
|
||||
}
|
||||
|
||||
return rows, nil
|
||||
}
|
||||
31
pkg/modules/retention/retention.go
Normal file
31
pkg/modules/retention/retention.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package retention
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
// Getter resolves retention data and expressions for read paths.
|
||||
type Getter interface {
|
||||
// ActiveSlices returns retention rules active over a half-open meter window.
|
||||
ActiveSlices(
|
||||
ctx context.Context,
|
||||
orgID valuer.UUID,
|
||||
dbName string,
|
||||
tableName string,
|
||||
fallbackDefaultDays int,
|
||||
startMs int64,
|
||||
endMs int64,
|
||||
) ([]retentiontypes.Slice, error)
|
||||
|
||||
// BuildMultiIfSQL builds a ClickHouse expression for effective retention days.
|
||||
BuildMultiIfSQL([]retentiontypes.CustomRetentionRule, int) (string, error)
|
||||
|
||||
// BuildRuleIndexSQL builds a ClickHouse expression for the matched rule index.
|
||||
BuildRuleIndexSQL([]retentiontypes.CustomRetentionRule) (string, error)
|
||||
|
||||
// RuleDimensionKeys returns unique label keys used by retention filters.
|
||||
RuleDimensionKeys([]retentiontypes.CustomRetentionRule) ([]string, error)
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/serviceaccount"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/cachetypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/serviceaccounttypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
@@ -144,7 +145,7 @@ func (module *module) DeleteRole(ctx context.Context, orgID valuer.UUID, id valu
|
||||
return err
|
||||
}
|
||||
|
||||
err = module.authz.Revoke(ctx, orgID, []string{role.Name}, authtypes.MustNewSubject(authtypes.TypeableServiceAccount, id.String(), orgID, nil))
|
||||
err = module.authz.Revoke(ctx, orgID, []string{role.Name}, authtypes.MustNewSubject(coretypes.NewResourceServiceAccount(), id.String(), orgID, nil))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -187,7 +188,7 @@ func (module *module) Delete(ctx context.Context, orgID valuer.UUID, id valuer.U
|
||||
return err
|
||||
}
|
||||
|
||||
err = module.authz.Revoke(ctx, orgID, serviceAccount.RoleNames(), authtypes.MustNewSubject(authtypes.TypeableServiceAccount, id.StringValue(), orgID, nil))
|
||||
err = module.authz.Revoke(ctx, orgID, serviceAccount.RoleNames(), authtypes.MustNewSubject(coretypes.NewResourceServiceAccount(), id.StringValue(), orgID, nil))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -386,7 +387,7 @@ func (module *module) setRole(ctx context.Context, orgID valuer.UUID, id valuer.
|
||||
return err
|
||||
}
|
||||
|
||||
err = module.authz.ModifyGrant(ctx, orgID, serviceAccount.RoleNames(), []string{role.Name}, authtypes.MustNewSubject(authtypes.TypeableServiceAccount, id.String(), orgID, nil))
|
||||
err = module.authz.ModifyGrant(ctx, orgID, serviceAccount.RoleNames(), []string{role.Name}, authtypes.MustNewSubject(coretypes.NewResourceServiceAccount(), id.String(), orgID, nil))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2,9 +2,10 @@ package tracefunnel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
"net/http"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
traceFunnels "github.com/SigNoz/signoz/pkg/types/tracefunneltypes"
|
||||
)
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/modules/user"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
)
|
||||
|
||||
@@ -43,8 +44,8 @@ func NewService(
|
||||
orgGetter: orgGetter,
|
||||
authz: authz,
|
||||
config: config,
|
||||
stopC: make(chan struct{}),
|
||||
healthyC: make(chan struct{}),
|
||||
stopC: make(chan struct{}),
|
||||
healthyC: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -148,7 +149,7 @@ func (s *service) createOrPromoteRootUser(ctx context.Context, orgID valuer.UUID
|
||||
orgID,
|
||||
existingUserRoleNames,
|
||||
[]string{authtypes.SigNozAdminRoleName},
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, existingUser.ID.StringValue(), orgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), existingUser.ID.StringValue(), orgID, nil),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/tokenizer"
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/authtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/coretypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/emailtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/integrationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
@@ -176,7 +177,7 @@ func (module *setter) CreateUser(ctx context.Context, user *types.User, opts ...
|
||||
ctx,
|
||||
user.OrgID,
|
||||
createUserOpts.RoleNames,
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, user.ID.StringValue(), user.OrgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), user.ID.StringValue(), user.OrgID, nil),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -236,15 +237,15 @@ func (module *setter) UpdateUserDeprecated(ctx context.Context, orgID valuer.UUI
|
||||
roleChange := user.Role != "" && user.Role != existingUser.Role
|
||||
|
||||
if roleChange {
|
||||
selectors := []authtypes.Selector{
|
||||
authtypes.MustNewSelector(authtypes.TypeRole, authtypes.SigNozAdminRoleName),
|
||||
selectors := []coretypes.Selector{
|
||||
coretypes.TypeRole.MustSelector(authtypes.SigNozAdminRoleName),
|
||||
}
|
||||
err = module.authz.CheckWithTupleCreation(
|
||||
ctx,
|
||||
claims,
|
||||
valuer.MustNewUUID(claims.OrgID),
|
||||
authtypes.RelationAssignee,
|
||||
authtypes.TypeableRole,
|
||||
authtypes.Relation{Verb: coretypes.VerbAssignee},
|
||||
coretypes.NewResourceRole(),
|
||||
selectors,
|
||||
selectors,
|
||||
)
|
||||
@@ -264,7 +265,7 @@ func (module *setter) UpdateUserDeprecated(ctx context.Context, orgID valuer.UUI
|
||||
orgID,
|
||||
[]string{authtypes.MustGetSigNozManagedRoleFromExistingRole(existingUser.Role)},
|
||||
[]string{authtypes.MustGetSigNozManagedRoleFromExistingRole(user.Role)},
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, id, orgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), id, orgID, nil),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -400,7 +401,7 @@ func (module *setter) DeleteUser(ctx context.Context, orgID valuer.UUID, id stri
|
||||
ctx,
|
||||
orgID,
|
||||
roleNames,
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, id, orgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), id, orgID, nil),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -586,7 +587,7 @@ func (module *setter) UpdatePasswordByResetPasswordToken(ctx context.Context, to
|
||||
ctx,
|
||||
user.OrgID,
|
||||
roleNames,
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, user.ID.StringValue(), user.OrgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), user.ID.StringValue(), user.OrgID, nil),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -719,7 +720,7 @@ func (module *setter) CreateFirstUser(ctx context.Context, organization *types.O
|
||||
return err
|
||||
}
|
||||
|
||||
err = module.CreateUser(ctx, user, root.WithFactorPassword(password), root.WithRoleNames(roleNames))
|
||||
err = module.createUserWithoutGrant(ctx, user, root.WithFactorPassword(password), root.WithRoleNames(roleNames))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -796,7 +797,7 @@ func (module *setter) activatePendingUser(ctx context.Context, user *types.User,
|
||||
ctx,
|
||||
user.OrgID,
|
||||
createUserOpts.RoleNames,
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, user.ID.StringValue(), user.OrgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), user.ID.StringValue(), user.OrgID, nil),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -890,7 +891,7 @@ func (module *setter) AddUserRole(ctx context.Context, orgID, userID valuer.UUID
|
||||
orgID,
|
||||
existingRoles,
|
||||
[]string{roleName},
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, existingUser.ID.StringValue(), existingUser.OrgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), existingUser.ID.StringValue(), existingUser.OrgID, nil),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -953,7 +954,7 @@ func (module *setter) RemoveUserRole(ctx context.Context, orgID, userID valuer.U
|
||||
ctx,
|
||||
orgID,
|
||||
[]string{roleName},
|
||||
authtypes.MustNewSubject(authtypes.TypeableUser, existingUser.ID.StringValue(), existingUser.OrgID, nil),
|
||||
authtypes.MustNewSubject(coretypes.NewResourceUser(), existingUser.ID.StringValue(), existingUser.OrgID, nil),
|
||||
); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -4,9 +4,10 @@ package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
"sync"
|
||||
"unicode"
|
||||
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
)
|
||||
|
||||
// Suppress unused import error
|
||||
|
||||
@@ -4,9 +4,10 @@ package parser
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
"sync"
|
||||
"unicode"
|
||||
|
||||
"github.com/antlr4-go/antlr/v4"
|
||||
)
|
||||
|
||||
// Suppress unused import error
|
||||
|
||||
@@ -2,11 +2,12 @@ package clickhouseprometheus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore/telemetrystoretest"
|
||||
cmock "github.com/srikanthccv/ClickHouse-go-mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/SigNoz/signoz/pkg/telemetrystore"
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
package prometheustest
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/prometheus"
|
||||
"github.com/prometheus/prometheus/model/labels"
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRemoveExtraLabels(t *testing.T) {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"github.com/SigNoz/govaluate"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/errors"
|
||||
"github.com/SigNoz/signoz/pkg/querybuilder"
|
||||
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
)
|
||||
@@ -969,11 +970,19 @@ func (q *querier) prepareFillZeroArgsWithStep(functions []qbtypes.Function, req
|
||||
updatedFunctions := make([]qbtypes.Function, len(functions))
|
||||
copy(updatedFunctions, functions)
|
||||
|
||||
// funcFillZero expects start/end in milliseconds. req.Start/req.End may
|
||||
// arrive in s/ms/μs/ns depending on the caller; normalize via ToNanoSecs
|
||||
// (same pattern used elsewhere in the codebase, e.g. RecommendedStepInterval)
|
||||
// then convert to ms. Without this, an ns payload makes (end-start)/step
|
||||
// 10^6× too large and OOMs the process.
|
||||
startMs := querybuilder.ToNanoSecs(req.Start) / 1_000_000
|
||||
endMs := querybuilder.ToNanoSecs(req.End) / 1_000_000
|
||||
|
||||
for i, fn := range updatedFunctions {
|
||||
if fn.Name == qbtypes.FunctionNameFillZero && len(fn.Args) == 0 {
|
||||
fn.Args = []qbtypes.FunctionArg{
|
||||
{Value: float64(req.Start)},
|
||||
{Value: float64(req.End)},
|
||||
{Value: float64(startMs)},
|
||||
{Value: float64(endMs)},
|
||||
{Value: float64(step)},
|
||||
}
|
||||
updatedFunctions[i] = fn
|
||||
|
||||
@@ -251,10 +251,10 @@ func TestEnhancePromQLError(t *testing.T) {
|
||||
|
||||
t.Run("quoted metric outside braces patterns", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
wantHint bool
|
||||
wantMetricInHint string
|
||||
name string
|
||||
query string
|
||||
wantHint bool
|
||||
wantMetricInHint string
|
||||
}{
|
||||
{
|
||||
name: "quoted metric name followed by selector",
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/types"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
@@ -1342,7 +1343,7 @@ func getLocalTableName(tableName string) string {
|
||||
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
|
||||
func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params *retentiontypes.TTLParams) (*retentiontypes.SetTTLResponseItem, *model.ApiError) {
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
@@ -1377,7 +1378,7 @@ func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params
|
||||
if apiErr != nil {
|
||||
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
|
||||
}
|
||||
if statusItem.Status == constants.StatusPending {
|
||||
if statusItem.Status == retentiontypes.TTLSettingStatusPending {
|
||||
return nil, &model.ApiError{Typ: model.ErrorConflict, Err: fmt.Errorf("TTL is already running")}
|
||||
}
|
||||
}
|
||||
@@ -1425,7 +1426,7 @@ func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params
|
||||
// we will change ttl for only the new parts and not the old ones
|
||||
query += " SETTINGS materialize_ttl_after_modify=0"
|
||||
|
||||
ttl := types.TTLSetting{
|
||||
ttl := retentiontypes.TTLSetting{
|
||||
Identifiable: types.Identifiable{
|
||||
ID: valuer.GenerateUUID(),
|
||||
},
|
||||
@@ -1436,7 +1437,7 @@ func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params
|
||||
TransactionID: uuid,
|
||||
TableName: tableName,
|
||||
TTL: int(params.DelDuration),
|
||||
Status: constants.StatusPending,
|
||||
Status: retentiontypes.TTLSettingStatusPending,
|
||||
ColdStorageTTL: coldStorageDuration,
|
||||
OrgID: orgID,
|
||||
}
|
||||
@@ -1460,9 +1461,9 @@ func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusFailed).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusFailed).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -1480,9 +1481,9 @@ func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusFailed).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusFailed).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -1495,9 +1496,9 @@ func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusSuccess).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusSuccess).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -1507,10 +1508,10 @@ func (r *ClickHouseReader) setTTLLogs(ctx context.Context, orgID string, params
|
||||
}
|
||||
|
||||
}(ttlPayload)
|
||||
return &model.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
|
||||
return &retentiontypes.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
|
||||
func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, params *retentiontypes.TTLParams) (*retentiontypes.SetTTLResponseItem, *model.ApiError) {
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalTraces.StringValue(),
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
@@ -1540,7 +1541,7 @@ func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, param
|
||||
if apiErr != nil {
|
||||
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
|
||||
}
|
||||
if statusItem.Status == constants.StatusPending {
|
||||
if statusItem.Status == retentiontypes.TTLSettingStatusPending {
|
||||
return nil, &model.ApiError{Typ: model.ErrorConflict, Err: fmt.Errorf("TTL is already running")}
|
||||
}
|
||||
}
|
||||
@@ -1563,7 +1564,7 @@ func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, param
|
||||
timestamp = "end"
|
||||
}
|
||||
|
||||
ttl := types.TTLSetting{
|
||||
ttl := retentiontypes.TTLSetting{
|
||||
Identifiable: types.Identifiable{
|
||||
ID: valuer.GenerateUUID(),
|
||||
},
|
||||
@@ -1574,7 +1575,7 @@ func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, param
|
||||
TransactionID: uuid,
|
||||
TableName: tableName,
|
||||
TTL: int(params.DelDuration),
|
||||
Status: constants.StatusPending,
|
||||
Status: retentiontypes.TTLSettingStatusPending,
|
||||
ColdStorageTTL: coldStorageDuration,
|
||||
OrgID: orgID,
|
||||
}
|
||||
@@ -1610,9 +1611,9 @@ func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, param
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusFailed).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusFailed).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -1631,9 +1632,9 @@ func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, param
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusFailed).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusFailed).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -1646,9 +1647,9 @@ func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, param
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusSuccess).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusSuccess).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -1657,7 +1658,7 @@ func (r *ClickHouseReader) setTTLTraces(ctx context.Context, orgID string, param
|
||||
}
|
||||
}(distributedTableName)
|
||||
}
|
||||
return &model.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
|
||||
return &retentiontypes.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) hasCustomRetentionColumn(ctx context.Context) (bool, error) {
|
||||
@@ -1686,7 +1687,7 @@ func (r *ClickHouseReader) hasCustomRetentionColumn(ctx context.Context) (bool,
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *model.CustomRetentionTTLParams) (*model.CustomRetentionTTLResponse, error) {
|
||||
func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *retentiontypes.CustomRetentionTTLParams) (*retentiontypes.CustomRetentionTTLResponse, error) {
|
||||
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalLogs.StringValue(),
|
||||
@@ -1701,7 +1702,7 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
if !hasCustomRetention {
|
||||
r.logger.Info("Custom retention not supported, falling back to standard TTL method", "orgID", orgID)
|
||||
|
||||
ttlParams := &model.TTLParams{
|
||||
ttlParams := &retentiontypes.TTLParams{
|
||||
Type: params.Type,
|
||||
DelDuration: int64(params.DefaultTTLDays * 24 * 3600),
|
||||
}
|
||||
@@ -1722,7 +1723,7 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
return nil, errorsV2.Wrapf(apiErr.Err, errorsV2.TypeInternal, errorsV2.CodeInternal, "failed to set standard TTL")
|
||||
}
|
||||
|
||||
return &model.CustomRetentionTTLResponse{
|
||||
return &retentiontypes.CustomRetentionTTLResponse{
|
||||
Message: fmt.Sprintf("Custom retention not supported, applied standard TTL of %d days. %s", params.DefaultTTLDays, ttlResult.Message),
|
||||
}, nil
|
||||
}
|
||||
@@ -1733,7 +1734,7 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
uuidWithHyphen := valuer.GenerateUUID()
|
||||
uuid := strings.Replace(uuidWithHyphen.String(), "-", "", -1)
|
||||
|
||||
if params.Type != constants.LogsTTL {
|
||||
if params.Type != retentiontypes.LogsTTL {
|
||||
return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "custom retention TTL only supported for logs")
|
||||
}
|
||||
|
||||
@@ -1764,7 +1765,7 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
if apiErr != nil {
|
||||
return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "error in processing custom_retention_ttl_status check sql query")
|
||||
}
|
||||
if statusItem.Status == constants.StatusPending {
|
||||
if statusItem.Status == retentiontypes.TTLSettingStatusPending {
|
||||
return nil, errorsV2.Newf(errorsV2.TypeInternal, errorsV2.CodeInternal, "custom retention TTL is already running")
|
||||
}
|
||||
}
|
||||
@@ -1838,7 +1839,7 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
}
|
||||
|
||||
for tableName, queries := range ttlPayload {
|
||||
customTTL := types.TTLSetting{
|
||||
customTTL := retentiontypes.TTLSetting{
|
||||
Identifiable: types.Identifiable{
|
||||
ID: valuer.GenerateUUID(),
|
||||
},
|
||||
@@ -1850,7 +1851,7 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
TableName: tableName,
|
||||
TTL: params.DefaultTTLDays,
|
||||
Condition: string(ttlConditionsJSON),
|
||||
Status: constants.StatusPending,
|
||||
Status: retentiontypes.TTLSettingStatusPending,
|
||||
ColdStorageTTL: coldStorageDuration,
|
||||
OrgID: orgID,
|
||||
}
|
||||
@@ -1866,7 +1867,7 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
err := r.setColdStorage(ctx, tableName, params.ColdStorageVolume)
|
||||
if err != nil {
|
||||
r.logger.Error("error in setting cold storage", errorsV2.Attr(err))
|
||||
r.updateCustomRetentionTTLStatus(ctx, orgID, tableName, constants.StatusFailed)
|
||||
r.updateCustomRetentionTTLStatus(ctx, orgID, tableName, retentiontypes.TTLSettingStatusFailed)
|
||||
return nil, errorsV2.Wrapf(err.Err, errorsV2.TypeInternal, errorsV2.CodeInternal, "error setting cold storage for table %s", tableName)
|
||||
}
|
||||
}
|
||||
@@ -1875,21 +1876,21 @@ func (r *ClickHouseReader) SetTTLV2(ctx context.Context, orgID string, params *m
|
||||
r.logger.Debug("Executing custom retention TTL request: ", "request", query, "step", i+1)
|
||||
if err := r.db.Exec(ctx, query); err != nil {
|
||||
r.logger.Error("error while setting custom retention ttl", errorsV2.Attr(err))
|
||||
r.updateCustomRetentionTTLStatus(ctx, orgID, tableName, constants.StatusFailed)
|
||||
r.updateCustomRetentionTTLStatus(ctx, orgID, tableName, retentiontypes.TTLSettingStatusFailed)
|
||||
return nil, errorsV2.Wrapf(err, errorsV2.TypeInternal, errorsV2.CodeInternal, "error setting custom retention TTL for table %s, query: %s", tableName, query)
|
||||
}
|
||||
}
|
||||
|
||||
r.updateCustomRetentionTTLStatus(ctx, orgID, tableName, constants.StatusSuccess)
|
||||
r.updateCustomRetentionTTLStatus(ctx, orgID, tableName, retentiontypes.TTLSettingStatusSuccess)
|
||||
}
|
||||
|
||||
return &model.CustomRetentionTTLResponse{
|
||||
return &retentiontypes.CustomRetentionTTLResponse{
|
||||
Message: "custom retention TTL has been successfully set up",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// New method to build multiIf expressions with support for multiple AND conditions
|
||||
func (r *ClickHouseReader) buildMultiIfExpression(ttlConditions []model.CustomRetentionRule, defaultTTLDays int, isResourceTable bool) string {
|
||||
func (r *ClickHouseReader) buildMultiIfExpression(ttlConditions []retentiontypes.CustomRetentionRule, defaultTTLDays int, isResourceTable bool) string {
|
||||
var conditions []string
|
||||
|
||||
for i, rule := range ttlConditions {
|
||||
@@ -1961,7 +1962,7 @@ func (r *ClickHouseReader) buildMultiIfExpression(ttlConditions []model.CustomRe
|
||||
return result
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) GetCustomRetentionTTL(ctx context.Context, orgID string) (*model.GetCustomRetentionTTLResponse, error) {
|
||||
func (r *ClickHouseReader) GetCustomRetentionTTL(ctx context.Context, orgID string) (*retentiontypes.GetCustomRetentionTTLResponse, error) {
|
||||
// Check if V2 (custom retention) is supported
|
||||
hasCustomRetention, err := r.hasCustomRetentionColumn(ctx)
|
||||
if err != nil {
|
||||
@@ -1970,14 +1971,14 @@ func (r *ClickHouseReader) GetCustomRetentionTTL(ctx context.Context, orgID stri
|
||||
hasCustomRetention = false
|
||||
}
|
||||
|
||||
response := &model.GetCustomRetentionTTLResponse{}
|
||||
response := &retentiontypes.GetCustomRetentionTTLResponse{}
|
||||
|
||||
if hasCustomRetention {
|
||||
// V2 - Custom retention is supported
|
||||
response.Version = "v2"
|
||||
|
||||
// Get the latest custom retention TTL setting
|
||||
customTTL := new(types.TTLSetting)
|
||||
customTTL := new(retentiontypes.TTLSetting)
|
||||
err := r.sqlDB.BunDB().NewSelect().
|
||||
Model(customTTL).
|
||||
Where("org_id = ?", orgID).
|
||||
@@ -1993,19 +1994,19 @@ func (r *ClickHouseReader) GetCustomRetentionTTL(ctx context.Context, orgID stri
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
// No V2 configuration found, return defaults
|
||||
response.DefaultTTLDays = 15
|
||||
response.TTLConditions = []model.CustomRetentionRule{}
|
||||
response.Status = constants.StatusSuccess
|
||||
response.DefaultTTLDays = retentiontypes.DefaultLogsRetentionDays
|
||||
response.TTLConditions = []retentiontypes.CustomRetentionRule{}
|
||||
response.Status = retentiontypes.TTLSettingStatusSuccess
|
||||
response.ColdStorageTTLDays = -1
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// Parse TTL conditions from Condition
|
||||
var ttlConditions []model.CustomRetentionRule
|
||||
var ttlConditions []retentiontypes.CustomRetentionRule
|
||||
if customTTL.Condition != "" {
|
||||
if err := json.Unmarshal([]byte(customTTL.Condition), &ttlConditions); err != nil {
|
||||
r.logger.Error("Error parsing TTL conditions", errorsV2.Attr(err))
|
||||
ttlConditions = []model.CustomRetentionRule{}
|
||||
ttlConditions = []retentiontypes.CustomRetentionRule{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2019,8 +2020,8 @@ func (r *ClickHouseReader) GetCustomRetentionTTL(ctx context.Context, orgID stri
|
||||
response.Version = "v1"
|
||||
|
||||
// Get V1 TTL configuration
|
||||
ttlParams := &model.GetTTLParams{
|
||||
Type: constants.LogsTTL,
|
||||
ttlParams := &retentiontypes.GetTTLParams{
|
||||
Type: retentiontypes.LogsTTL,
|
||||
}
|
||||
|
||||
ttlResult, apiErr := r.GetTTL(ctx, orgID, ttlParams)
|
||||
@@ -2040,14 +2041,14 @@ func (r *ClickHouseReader) GetCustomRetentionTTL(ctx context.Context, orgID stri
|
||||
}
|
||||
|
||||
// For V1, we don't have TTL conditions
|
||||
response.TTLConditions = []model.CustomRetentionRule{}
|
||||
response.TTLConditions = []retentiontypes.CustomRetentionRule{}
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) checkCustomRetentionTTLStatusItem(ctx context.Context, orgID string, tableName string) (*types.TTLSetting, error) {
|
||||
ttl := new(types.TTLSetting)
|
||||
func (r *ClickHouseReader) checkCustomRetentionTTLStatusItem(ctx context.Context, orgID string, tableName string) (*retentiontypes.TTLSetting, error) {
|
||||
ttl := new(retentiontypes.TTLSetting)
|
||||
err := r.sqlDB.BunDB().NewSelect().
|
||||
Model(ttl).
|
||||
Where("table_name = ?", tableName).
|
||||
@@ -2068,7 +2069,7 @@ func (r *ClickHouseReader) updateCustomRetentionTTLStatus(ctx context.Context, o
|
||||
statusItem, apiErr := r.checkCustomRetentionTTLStatusItem(ctx, orgID, tableName)
|
||||
if apiErr == nil && statusItem != nil {
|
||||
_, dbErr := r.sqlDB.BunDB().NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", status).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
@@ -2080,7 +2081,7 @@ func (r *ClickHouseReader) updateCustomRetentionTTLStatus(ctx context.Context, o
|
||||
}
|
||||
|
||||
// Enhanced validation function with duplicate detection and efficient key validation
|
||||
func (r *ClickHouseReader) validateTTLConditions(ctx context.Context, ttlConditions []model.CustomRetentionRule) error {
|
||||
func (r *ClickHouseReader) validateTTLConditions(ctx context.Context, ttlConditions []retentiontypes.CustomRetentionRule) error {
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
instrumentationtypes.CodeFunctionName: "validateTTLConditions",
|
||||
@@ -2184,16 +2185,16 @@ func (r *ClickHouseReader) validateTTLConditions(ctx context.Context, ttlConditi
|
||||
// SetTTL sets the TTL for traces or metrics or logs tables.
|
||||
// This is an async API which creates goroutines to set TTL.
|
||||
// Status of TTL update is tracked with ttl_status table in sqlite db.
|
||||
func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
|
||||
func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string, params *retentiontypes.TTLParams) (*retentiontypes.SetTTLResponseItem, *model.ApiError) {
|
||||
// Keep only latest 100 transactions/requests
|
||||
r.deleteTtlTransactions(ctx, orgID, 100)
|
||||
|
||||
switch params.Type {
|
||||
case constants.TraceTTL:
|
||||
case retentiontypes.TraceTTL:
|
||||
return r.setTTLTraces(ctx, orgID, params)
|
||||
case constants.MetricsTTL:
|
||||
case retentiontypes.MetricsTTL:
|
||||
return r.setTTLMetrics(ctx, orgID, params)
|
||||
case constants.LogsTTL:
|
||||
case retentiontypes.LogsTTL:
|
||||
return r.setTTLLogs(ctx, orgID, params)
|
||||
default:
|
||||
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while setting ttl. ttl type should be <metrics|traces>, got %v", params.Type)}
|
||||
@@ -2201,7 +2202,7 @@ func (r *ClickHouseReader) SetTTL(ctx context.Context, orgID string, params *mod
|
||||
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, params *model.TTLParams) (*model.SetTTLResponseItem, *model.ApiError) {
|
||||
func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, params *retentiontypes.TTLParams) (*retentiontypes.SetTTLResponseItem, *model.ApiError) {
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(),
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
@@ -2230,12 +2231,12 @@ func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, para
|
||||
if apiErr != nil {
|
||||
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
|
||||
}
|
||||
if statusItem.Status == constants.StatusPending {
|
||||
if statusItem.Status == retentiontypes.TTLSettingStatusPending {
|
||||
return nil, &model.ApiError{Typ: model.ErrorConflict, Err: fmt.Errorf("TTL is already running")}
|
||||
}
|
||||
}
|
||||
metricTTL := func(tableName string) {
|
||||
ttl := types.TTLSetting{
|
||||
ttl := retentiontypes.TTLSetting{
|
||||
Identifiable: types.Identifiable{
|
||||
ID: valuer.GenerateUUID(),
|
||||
},
|
||||
@@ -2246,7 +2247,7 @@ func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, para
|
||||
TransactionID: uuid,
|
||||
TableName: tableName,
|
||||
TTL: int(params.DelDuration),
|
||||
Status: constants.StatusPending,
|
||||
Status: retentiontypes.TTLSettingStatusPending,
|
||||
ColdStorageTTL: coldStorageDuration,
|
||||
OrgID: orgID,
|
||||
}
|
||||
@@ -2282,9 +2283,9 @@ func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, para
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusFailed).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusFailed).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -2303,9 +2304,9 @@ func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, para
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusFailed).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusFailed).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -2318,9 +2319,9 @@ func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, para
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewUpdate().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Set("updated_at = ?", time.Now()).
|
||||
Set("status = ?", constants.StatusSuccess).
|
||||
Set("status = ?", retentiontypes.TTLSettingStatusSuccess).
|
||||
Where("id = ?", statusItem.ID.StringValue()).
|
||||
Exec(ctx)
|
||||
if dbErr != nil {
|
||||
@@ -2331,7 +2332,7 @@ func (r *ClickHouseReader) setTTLMetrics(ctx context.Context, orgID string, para
|
||||
for _, tableName := range tableNames {
|
||||
go metricTTL(tableName)
|
||||
}
|
||||
return &model.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
|
||||
return &retentiontypes.SetTTLResponseItem{Message: "move ttl has been successfully set up"}, nil
|
||||
}
|
||||
|
||||
func (r *ClickHouseReader) deleteTtlTransactions(ctx context.Context, orgID string, numberOfTransactionsStore int) {
|
||||
@@ -2341,7 +2342,7 @@ func (r *ClickHouseReader) deleteTtlTransactions(ctx context.Context, orgID stri
|
||||
BunDB().
|
||||
NewSelect().
|
||||
Column("transaction_id").
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Where("org_id = ?", orgID).
|
||||
Group("transaction_id").
|
||||
OrderExpr("MAX(created_at) DESC").
|
||||
@@ -2356,7 +2357,7 @@ func (r *ClickHouseReader) deleteTtlTransactions(ctx context.Context, orgID stri
|
||||
sqlDB.
|
||||
BunDB().
|
||||
NewDelete().
|
||||
Model(new(types.TTLSetting)).
|
||||
Model(new(retentiontypes.TTLSetting)).
|
||||
Where("transaction_id NOT IN (?)", bun.In(limitTransactions)).
|
||||
Exec(ctx)
|
||||
if err != nil {
|
||||
@@ -2365,9 +2366,9 @@ func (r *ClickHouseReader) deleteTtlTransactions(ctx context.Context, orgID stri
|
||||
}
|
||||
|
||||
// checkTTLStatusItem checks if ttl_status table has an entry for the given table name
|
||||
func (r *ClickHouseReader) checkTTLStatusItem(ctx context.Context, orgID string, tableName string) (*types.TTLSetting, *model.ApiError) {
|
||||
func (r *ClickHouseReader) checkTTLStatusItem(ctx context.Context, orgID string, tableName string) (*retentiontypes.TTLSetting, *model.ApiError) {
|
||||
r.logger.Info("checkTTLStatusItem query", "tableName", tableName)
|
||||
ttl := new(types.TTLSetting)
|
||||
ttl := new(retentiontypes.TTLSetting)
|
||||
err := r.
|
||||
sqlDB.
|
||||
BunDB().
|
||||
@@ -2388,26 +2389,26 @@ func (r *ClickHouseReader) checkTTLStatusItem(ctx context.Context, orgID string,
|
||||
// getTTLQueryStatus fetches ttl_status table status from DB
|
||||
func (r *ClickHouseReader) getTTLQueryStatus(ctx context.Context, orgID string, tableNameArray []string) (string, *model.ApiError) {
|
||||
failFlag := false
|
||||
status := constants.StatusSuccess
|
||||
status := retentiontypes.TTLSettingStatusSuccess
|
||||
for _, tableName := range tableNameArray {
|
||||
statusItem, apiErr := r.checkTTLStatusItem(ctx, orgID, tableName)
|
||||
emptyStatusStruct := new(types.TTLSetting)
|
||||
emptyStatusStruct := new(retentiontypes.TTLSetting)
|
||||
if statusItem == emptyStatusStruct {
|
||||
return "", nil
|
||||
}
|
||||
if apiErr != nil {
|
||||
return "", &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error in processing ttl_status check sql query")}
|
||||
}
|
||||
if statusItem.Status == constants.StatusPending && statusItem.UpdatedAt.Unix()-time.Now().Unix() < 3600 {
|
||||
status = constants.StatusPending
|
||||
if statusItem.Status == retentiontypes.TTLSettingStatusPending && statusItem.UpdatedAt.Unix()-time.Now().Unix() < 3600 {
|
||||
status = retentiontypes.TTLSettingStatusPending
|
||||
return status, nil
|
||||
}
|
||||
if statusItem.Status == constants.StatusFailed {
|
||||
if statusItem.Status == retentiontypes.TTLSettingStatusFailed {
|
||||
failFlag = true
|
||||
}
|
||||
}
|
||||
if failFlag {
|
||||
status = constants.StatusFailed
|
||||
status = retentiontypes.TTLSettingStatusFailed
|
||||
}
|
||||
|
||||
return status, nil
|
||||
@@ -2460,7 +2461,7 @@ func getLocalTableNameArray(tableNames []string) []string {
|
||||
}
|
||||
|
||||
// GetTTL returns current ttl, expected ttl and past setTTL status for metrics/traces.
|
||||
func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *model.GetTTLParams) (*model.GetTTLResponseItem, *model.ApiError) {
|
||||
func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *retentiontypes.GetTTLParams) (*retentiontypes.GetTTLResponseItem, *model.ApiError) {
|
||||
|
||||
ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
|
||||
instrumentationtypes.CodeNamespace: "clickhouse-reader",
|
||||
@@ -2495,8 +2496,8 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
|
||||
return delTTL, moveTTL
|
||||
}
|
||||
|
||||
getMetricsTTL := func() (*model.DBResponseTTL, *model.ApiError) {
|
||||
var dbResp []model.DBResponseTTL
|
||||
getMetricsTTL := func() (*retentiontypes.DBResponseTTL, *model.ApiError) {
|
||||
var dbResp []retentiontypes.DBResponseTTL
|
||||
|
||||
query := fmt.Sprintf("SELECT engine_full FROM system.tables WHERE name='%v'", signozSampleLocalTableName)
|
||||
|
||||
@@ -2513,8 +2514,8 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
|
||||
}
|
||||
}
|
||||
|
||||
getTracesTTL := func() (*model.DBResponseTTL, *model.ApiError) {
|
||||
var dbResp []model.DBResponseTTL
|
||||
getTracesTTL := func() (*retentiontypes.DBResponseTTL, *model.ApiError) {
|
||||
var dbResp []retentiontypes.DBResponseTTL
|
||||
|
||||
query := fmt.Sprintf("SELECT engine_full FROM system.tables WHERE name='%v' AND database='%v'", r.traceLocalTableName, signozTraceDBName)
|
||||
|
||||
@@ -2531,8 +2532,8 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
|
||||
}
|
||||
}
|
||||
|
||||
getLogsTTL := func() (*model.DBResponseTTL, *model.ApiError) {
|
||||
var dbResp []model.DBResponseTTL
|
||||
getLogsTTL := func() (*retentiontypes.DBResponseTTL, *model.ApiError) {
|
||||
var dbResp []retentiontypes.DBResponseTTL
|
||||
|
||||
query := fmt.Sprintf("SELECT engine_full FROM system.tables WHERE name='%v' AND database='%v'", r.logsLocalTableName, r.logsDB)
|
||||
|
||||
@@ -2550,7 +2551,7 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
|
||||
}
|
||||
|
||||
switch ttlParams.Type {
|
||||
case constants.TraceTTL:
|
||||
case retentiontypes.TraceTTL:
|
||||
tableNameArray := []string{
|
||||
r.TraceDB + "." + r.traceTableName,
|
||||
r.TraceDB + "." + r.traceResourceTableV3,
|
||||
@@ -2578,9 +2579,9 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
|
||||
}
|
||||
|
||||
delTTL, moveTTL := parseTTL(dbResp.EngineFull)
|
||||
return &model.GetTTLResponseItem{TracesTime: delTTL, TracesMoveTime: moveTTL, ExpectedTracesTime: ttlQuery.TTL, ExpectedTracesMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
|
||||
return &retentiontypes.GetTTLResponseItem{TracesTime: delTTL, TracesMoveTime: moveTTL, ExpectedTracesTime: ttlQuery.TTL, ExpectedTracesMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
|
||||
|
||||
case constants.MetricsTTL:
|
||||
case retentiontypes.MetricsTTL:
|
||||
tableNameArray := []string{signozMetricDBName + "." + signozSampleTableName}
|
||||
tableNameArray = getLocalTableNameArray(tableNameArray)
|
||||
status, apiErr := r.getTTLQueryStatus(ctx, orgID, tableNameArray)
|
||||
@@ -2601,9 +2602,9 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
|
||||
}
|
||||
|
||||
delTTL, moveTTL := parseTTL(dbResp.EngineFull)
|
||||
return &model.GetTTLResponseItem{MetricsTime: delTTL, MetricsMoveTime: moveTTL, ExpectedMetricsTime: ttlQuery.TTL, ExpectedMetricsMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
|
||||
return &retentiontypes.GetTTLResponseItem{MetricsTime: delTTL, MetricsMoveTime: moveTTL, ExpectedMetricsTime: ttlQuery.TTL, ExpectedMetricsMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
|
||||
|
||||
case constants.LogsTTL:
|
||||
case retentiontypes.LogsTTL:
|
||||
tableNameArray := []string{r.logsDB + "." + r.logsTableName}
|
||||
tableNameArray = getLocalTableNameArray(tableNameArray)
|
||||
status, apiErr := r.getTTLQueryStatus(ctx, orgID, tableNameArray)
|
||||
@@ -2624,7 +2625,7 @@ func (r *ClickHouseReader) GetTTL(ctx context.Context, orgID string, ttlParams *
|
||||
}
|
||||
|
||||
delTTL, moveTTL := parseTTL(dbResp.EngineFull)
|
||||
return &model.GetTTLResponseItem{LogsTime: delTTL, LogsMoveTime: moveTTL, ExpectedLogsTime: ttlQuery.TTL, ExpectedLogsMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
|
||||
return &retentiontypes.GetTTLResponseItem{LogsTime: delTTL, LogsMoveTime: moveTTL, ExpectedLogsTime: ttlQuery.TTL, ExpectedLogsMoveTime: ttlQuery.ColdStorageTTL, Status: status}, nil
|
||||
|
||||
default:
|
||||
return nil, &model.ApiError{Typ: model.ErrorExec, Err: fmt.Errorf("error while getting ttl. ttl type should be metrics|traces, got %v",
|
||||
|
||||
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/cloudintegrations/services"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/integrations"
|
||||
"github.com/SigNoz/signoz/pkg/signoz"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -1677,7 +1678,7 @@ func (aH *APIHandler) setCustomRetentionTTL(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
var params model.CustomRetentionTTLParams
|
||||
var params retentiontypes.CustomRetentionTTLParams
|
||||
if err := json.NewDecoder(r.Body).Decode(¶ms); err != nil {
|
||||
render.Error(w, errorsV2.Newf(errorsV2.TypeInvalidInput, errorsV2.CodeInvalidInput, "Invalid data"))
|
||||
return
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/query-service/app/metrics/v4/helpers"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/common"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/constants"
|
||||
@@ -17,7 +19,6 @@ import (
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/postprocess"
|
||||
"github.com/SigNoz/signoz/pkg/types/ctxtypes"
|
||||
"log/slog"
|
||||
|
||||
"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
|
||||
"github.com/SigNoz/signoz/pkg/valuer"
|
||||
|
||||
@@ -2,6 +2,7 @@ package helpers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ import (
|
||||
"github.com/SigNoz/signoz/pkg/query-service/postprocess"
|
||||
"github.com/SigNoz/signoz/pkg/query-service/utils"
|
||||
querytemplate "github.com/SigNoz/signoz/pkg/query-service/utils/queryTemplate"
|
||||
"github.com/SigNoz/signoz/pkg/types/retentiontypes"
|
||||
chVariables "github.com/SigNoz/signoz/pkg/variables/clickhouse"
|
||||
)
|
||||
|
||||
@@ -419,7 +420,7 @@ func parseTime(param string, r *http.Request) (*time.Time, error) {
|
||||
|
||||
}
|
||||
|
||||
func parseTTLParams(r *http.Request) (*model.TTLParams, error) {
|
||||
func parseTTLParams(r *http.Request) (*retentiontypes.TTLParams, error) {
|
||||
|
||||
// make sure either of the query params are present
|
||||
typeTTL := r.URL.Query().Get("type")
|
||||
@@ -432,7 +433,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) {
|
||||
}
|
||||
|
||||
// Validate the type parameter
|
||||
if typeTTL != baseconstants.TraceTTL && typeTTL != baseconstants.MetricsTTL && typeTTL != baseconstants.LogsTTL {
|
||||
if typeTTL != retentiontypes.TraceTTL && typeTTL != retentiontypes.MetricsTTL && typeTTL != retentiontypes.LogsTTL {
|
||||
return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL)
|
||||
}
|
||||
|
||||
@@ -455,7 +456,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) {
|
||||
}
|
||||
}
|
||||
|
||||
return &model.TTLParams{
|
||||
return &retentiontypes.TTLParams{
|
||||
Type: typeTTL,
|
||||
DelDuration: int64(durationParsed.Seconds()),
|
||||
ColdStorageVolume: coldStorage,
|
||||
@@ -463,7 +464,7 @@ func parseTTLParams(r *http.Request) (*model.TTLParams, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseGetTTL(r *http.Request) (*model.GetTTLParams, error) {
|
||||
func parseGetTTL(r *http.Request) (*retentiontypes.GetTTLParams, error) {
|
||||
|
||||
typeTTL := r.URL.Query().Get("type")
|
||||
|
||||
@@ -471,12 +472,12 @@ func parseGetTTL(r *http.Request) (*model.GetTTLParams, error) {
|
||||
return nil, fmt.Errorf("type param cannot be empty from the query")
|
||||
} else {
|
||||
// Validate the type parameter
|
||||
if typeTTL != baseconstants.TraceTTL && typeTTL != baseconstants.MetricsTTL && typeTTL != baseconstants.LogsTTL {
|
||||
if typeTTL != retentiontypes.TraceTTL && typeTTL != retentiontypes.MetricsTTL && typeTTL != retentiontypes.LogsTTL {
|
||||
return nil, fmt.Errorf("type param should be metrics|traces|logs, got %v", typeTTL)
|
||||
}
|
||||
}
|
||||
|
||||
return &model.GetTTLParams{Type: typeTTL}, nil
|
||||
return &retentiontypes.GetTTLParams{Type: typeTTL}, nil
|
||||
}
|
||||
|
||||
func parseAggregateAttributeRequest(r *http.Request) (*v3.AggregateAttributeRequest, error) {
|
||||
|
||||
@@ -3,9 +3,9 @@ package querier
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strings"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@ package v2
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strings"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user