Compare commits

...

17 Commits

Author SHA1 Message Date
nikhilmantri0902
ee21f180d7 chore: goroutines in container monitoring 2026-07-06 14:47:21 +05:30
Nikhil Mantri
1a51f22679 Merge branch 'main' into infraM/kubernetes_container_monitoring 2026-07-02 20:39:26 +05:30
nikhilmantri0902
700e4485b9 chore: integration tests 2026-07-02 20:34:08 +05:30
Nikhil Mantri
4d73a24e3a Merge branch 'main' into infraM/kubernetes_container_monitoring 2026-07-02 20:21:19 +05:30
nikhilmantri0902
7c8b2a922c chore: added integration tests 2026-07-02 20:20:48 +05:30
nikhilmantri0902
89524c4c67 chore: containers query modified 2026-07-02 19:00:48 +05:30
nikhilmantri0902
83bb1993fe chore: added changes to the checks API for new kube containers section 2026-07-02 18:55:44 +05:30
nikhilmantri0902
df05f33c06 chore: corrected metrics list for metadata lookup 2026-07-02 18:05:01 +05:30
nikhilmantri0902
df83080d7c chore: add metrics to the containers list for metadata and earliest time 2026-07-02 17:40:36 +05:30
nikhilmantri0902
42b38b54ac chore: pass orgID to getMetadata for containers (adapt to rebased main) 2026-07-02 17:15:27 +05:30
nikhilmantri0902
11d56a1c4a chore: use recency for container status.reason as well 2026-07-02 17:10:32 +05:30
nikhilmantri0902
c3e7e0925b chore: endpoint and variable rename 2026-07-02 17:10:32 +05:30
nikhilmantri0902
1ea1b489f9 chore: added open api spec 2026-07-02 17:10:32 +05:30
nikhilmantri0902
246e01c89a chore: added wiring 2026-07-02 17:10:32 +05:30
nikhilmantri0902
76791cbd7e chore: added helper queries 2026-07-02 17:10:32 +05:30
nikhilmantri0902
926125df22 chore: added querier query and constants 2026-07-02 17:10:32 +05:30
nikhilmantri0902
ce664e3c85 chore: added types for containers 2026-07-02 17:10:32 +05:30
24 changed files with 2630 additions and 22 deletions

View File

@@ -4162,6 +4162,7 @@ components:
- namespaces
- clusters
- volumes
- kube_containers
type: string
InframonitoringtypesChecks:
properties:
@@ -4264,6 +4265,158 @@ components:
- total
- endTimeBeforeRetention
type: object
InframonitoringtypesContainerCountsByReady:
properties:
notReady:
type: integer
ready:
type: integer
required:
- ready
- notReady
type: object
InframonitoringtypesContainerCountsByStatus:
properties:
completed:
type: integer
containerCannotRun:
type: integer
containerCreating:
type: integer
crashLoopBackOff:
type: integer
createContainerConfigError:
type: integer
errImagePull:
type: integer
error:
type: integer
imagePullBackOff:
type: integer
oomKilled:
type: integer
running:
type: integer
terminated:
type: integer
unknown:
type: integer
waiting:
type: integer
required:
- running
- waiting
- terminated
- crashLoopBackOff
- imagePullBackOff
- errImagePull
- createContainerConfigError
- containerCreating
- oomKilled
- completed
- error
- containerCannotRun
- unknown
type: object
InframonitoringtypesContainerReady:
enum:
- ready
- not_ready
- no_data
type: string
InframonitoringtypesContainerRecord:
properties:
containerCountsByReady:
$ref: '#/components/schemas/InframonitoringtypesContainerCountsByReady'
containerCountsByStatus:
$ref: '#/components/schemas/InframonitoringtypesContainerCountsByStatus'
containerName:
type: string
cpu:
format: double
type: number
cpuLimitUtilization:
format: double
type: number
cpuRequestUtilization:
format: double
type: number
memory:
format: double
type: number
memoryLimitUtilization:
format: double
type: number
memoryRequestUtilization:
format: double
type: number
meta:
additionalProperties:
type: string
nullable: true
type: object
podUID:
type: string
ready:
$ref: '#/components/schemas/InframonitoringtypesContainerReady'
restarts:
format: int64
type: integer
status:
$ref: '#/components/schemas/InframonitoringtypesContainerStatus'
required:
- podUID
- containerName
- status
- containerCountsByStatus
- ready
- containerCountsByReady
- restarts
- cpu
- cpuRequestUtilization
- cpuLimitUtilization
- memory
- memoryRequestUtilization
- memoryLimitUtilization
- meta
type: object
InframonitoringtypesContainerStatus:
enum:
- running
- waiting
- terminated
- crashloopbackoff
- imagepullbackoff
- errimagepull
- createcontainerconfigerror
- containercreating
- oomkilled
- completed
- error
- containercannotrun
- unknown
- no_data
type: string
InframonitoringtypesContainers:
properties:
endTimeBeforeRetention:
type: boolean
records:
items:
$ref: '#/components/schemas/InframonitoringtypesContainerRecord'
type: array
total:
type: integer
type:
$ref: '#/components/schemas/InframonitoringtypesResponseType'
warning:
$ref: '#/components/schemas/Querybuildertypesv5QueryWarnData'
required:
- type
- records
- total
- endTimeBeforeRetention
type: object
InframonitoringtypesDaemonSetRecord:
properties:
currentNodes:
@@ -4833,6 +4986,32 @@ components:
- end
- limit
type: object
InframonitoringtypesPostableContainers:
properties:
end:
format: int64
type: integer
filter:
$ref: '#/components/schemas/Querybuildertypesv5Filter'
groupBy:
items:
$ref: '#/components/schemas/Querybuildertypesv5GroupByKey'
nullable: true
type: array
limit:
type: integer
offset:
type: integer
orderBy:
$ref: '#/components/schemas/Querybuildertypesv5OrderBy'
start:
format: int64
type: integer
required:
- start
- end
- limit
type: object
InframonitoringtypesPostableDaemonSets:
properties:
end:
@@ -15692,6 +15871,86 @@ paths:
summary: List Jobs for Infra Monitoring
tags:
- inframonitoring
/api/v2/infra_monitoring/kube_containers:
post:
deprecated: false
description: 'Returns a paginated list of Kubernetes containers with key kubeletstats
metrics: CPU usage (cores), CPU request/limit utilization, memory working
set, and memory request/limit utilization. Each container also reports health
signals from the k8s_cluster receiver: status (kubectl-style display status
derived from k8s.container.status.state + k8s.container.status.reason), restarts
(absolute count from k8s.container.restarts), and ready (ready/not_ready from
k8s.container.ready). The row identity is (k8s.pod.uid, k8s.container.name),
stable across container restarts. Each container includes metadata attributes
(k8s.container.name, k8s.pod.name, container.image.name, container.image.tag,
k8s.namespace.name, k8s.node.name, k8s.cluster.name, and workload owner such
as deployment/statefulset/daemonset/job). The response type is ''list'' for
the default (k8s.pod.uid, k8s.container.name) grouping (each row is one container
with its current status and ready state) or ''grouped_list'' for custom groupBy
keys (each row aggregates containers in the group with per-status counts under
containerCountsByStatus, per-readiness counts under containerCountsByReady,
and restarts as the group sum). Status requires the optional k8s.container.status.state
and k8s.container.status.reason metrics; when either is missing, status is
omitted and a warning is returned while restarts and ready are still computed.
Supports filtering via a filter expression, custom groupBy, ordering by any
of the six metrics (cpu, cpu_request, cpu_limit, memory, memory_request, memory_limit),
and pagination via offset/limit. Also reports whether the requested time range
falls before the data retention boundary. Numeric metric fields (cpu, cpuRequestUtilization,
cpuLimitUtilization, memory, memoryRequestUtilization, memoryLimitUtilization)
and restarts return -1 as a sentinel when no data is available for that field.'
operationId: ListContainers
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/InframonitoringtypesPostableContainers'
responses:
"200":
content:
application/json:
schema:
properties:
data:
$ref: '#/components/schemas/InframonitoringtypesContainers'
status:
type: string
required:
- status
- data
type: object
description: OK
"400":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Bad Request
"401":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Unauthorized
"403":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Forbidden
"500":
content:
application/json:
schema:
$ref: '#/components/schemas/RenderErrorResponse'
description: Internal Server Error
security:
- api_key:
- VIEWER
- tokenizer:
- VIEWER
summary: List Kubernetes Containers for Infra Monitoring
tags:
- inframonitoring
/api/v2/infra_monitoring/namespaces:
post:
deprecated: false

View File

@@ -21,6 +21,7 @@ import type {
GetChecks200,
GetChecksParams,
InframonitoringtypesPostableClustersDTO,
InframonitoringtypesPostableContainersDTO,
InframonitoringtypesPostableDaemonSetsDTO,
InframonitoringtypesPostableDeploymentsDTO,
InframonitoringtypesPostableHostsDTO,
@@ -31,6 +32,7 @@ import type {
InframonitoringtypesPostableStatefulSetsDTO,
InframonitoringtypesPostableVolumesDTO,
ListClusters200,
ListContainers200,
ListDaemonSets200,
ListDeployments200,
ListHosts200,
@@ -548,6 +550,89 @@ export const useListJobs = <
> => {
return useMutation(getListJobsMutationOptions(options));
};
/**
* Returns a paginated list of Kubernetes containers with key kubeletstats metrics: CPU usage (cores), CPU request/limit utilization, memory working set, and memory request/limit utilization. Each container also reports health signals from the k8s_cluster receiver: status (kubectl-style display status derived from k8s.container.status.state + k8s.container.status.reason), restarts (absolute count from k8s.container.restarts), and ready (ready/not_ready from k8s.container.ready). The row identity is (k8s.pod.uid, k8s.container.name), stable across container restarts. Each container includes metadata attributes (k8s.container.name, k8s.pod.name, container.image.name, container.image.tag, k8s.namespace.name, k8s.node.name, k8s.cluster.name, and workload owner such as deployment/statefulset/daemonset/job). The response type is 'list' for the default (k8s.pod.uid, k8s.container.name) grouping (each row is one container with its current status and ready state) or 'grouped_list' for custom groupBy keys (each row aggregates containers in the group with per-status counts under containerCountsByStatus, per-readiness counts under containerCountsByReady, and restarts as the group sum). Status requires the optional k8s.container.status.state and k8s.container.status.reason metrics; when either is missing, status is omitted and a warning is returned while restarts and ready are still computed. Supports filtering via a filter expression, custom groupBy, ordering by any of the six metrics (cpu, cpu_request, cpu_limit, memory, memory_request, memory_limit), and pagination via offset/limit. Also reports whether the requested time range falls before the data retention boundary. Numeric metric fields (cpu, cpuRequestUtilization, cpuLimitUtilization, memory, memoryRequestUtilization, memoryLimitUtilization) and restarts return -1 as a sentinel when no data is available for that field.
* @summary List Kubernetes Containers for Infra Monitoring
*/
export const listContainers = (
inframonitoringtypesPostableContainersDTO?: BodyType<InframonitoringtypesPostableContainersDTO>,
signal?: AbortSignal,
) => {
return GeneratedAPIInstance<ListContainers200>({
url: `/api/v2/infra_monitoring/kube_containers`,
method: 'POST',
headers: { 'Content-Type': 'application/json' },
data: inframonitoringtypesPostableContainersDTO,
signal,
});
};
export const getListContainersMutationOptions = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof listContainers>>,
TError,
{ data?: BodyType<InframonitoringtypesPostableContainersDTO> },
TContext
>;
}): UseMutationOptions<
Awaited<ReturnType<typeof listContainers>>,
TError,
{ data?: BodyType<InframonitoringtypesPostableContainersDTO> },
TContext
> => {
const mutationKey = ['listContainers'];
const { mutation: mutationOptions } = options
? options.mutation &&
'mutationKey' in options.mutation &&
options.mutation.mutationKey
? options
: { ...options, mutation: { ...options.mutation, mutationKey } }
: { mutation: { mutationKey } };
const mutationFn: MutationFunction<
Awaited<ReturnType<typeof listContainers>>,
{ data?: BodyType<InframonitoringtypesPostableContainersDTO> }
> = (props) => {
const { data } = props ?? {};
return listContainers(data);
};
return { mutationFn, ...mutationOptions };
};
export type ListContainersMutationResult = NonNullable<
Awaited<ReturnType<typeof listContainers>>
>;
export type ListContainersMutationBody =
| BodyType<InframonitoringtypesPostableContainersDTO>
| undefined;
export type ListContainersMutationError = ErrorType<RenderErrorResponseDTO>;
/**
* @summary List Kubernetes Containers for Infra Monitoring
*/
export const useListContainers = <
TError = ErrorType<RenderErrorResponseDTO>,
TContext = unknown,
>(options?: {
mutation?: UseMutationOptions<
Awaited<ReturnType<typeof listContainers>>,
TError,
{ data?: BodyType<InframonitoringtypesPostableContainersDTO> },
TContext
>;
}): UseMutationResult<
Awaited<ReturnType<typeof listContainers>>,
TError,
{ data?: BodyType<InframonitoringtypesPostableContainersDTO> },
TContext
> => {
return useMutation(getListContainersMutationOptions(options));
};
/**
* Returns a paginated list of Kubernetes namespaces with key aggregated pod metrics: CPU usage and memory working set (summed across pods in the group), plus per-group podCountsByPhase ({ pending, running, succeeded, failed, unknown } from each pod's latest k8s.pod.phase value in the window). Each namespace includes metadata attributes (k8s.namespace.name, k8s.cluster.name). The response type is 'list' for the default k8s.namespace.name grouping or 'grouped_list' for custom groupBy keys; in both modes every row aggregates pods in the group. Supports filtering via a filter expression, custom groupBy, ordering by cpu / memory, and pagination via offset/limit. Also reports whether the requested time range falls before the data retention boundary. Numeric metric fields (namespaceCPU, namespaceMemory) return -1 as a sentinel when no data is available for that field.
* @summary List Namespaces for Infra Monitoring

View File

@@ -5553,6 +5553,7 @@ export enum InframonitoringtypesCheckTypeDTO {
namespaces = 'namespaces',
clusters = 'clusters',
volumes = 'volumes',
kube_containers = 'kube_containers',
}
export interface InframonitoringtypesMissingMetricsComponentEntryDTO {
associatedComponent: InframonitoringtypesAssociatedComponentDTO;
@@ -5756,6 +5757,174 @@ export interface InframonitoringtypesClustersDTO {
warning?: Querybuildertypesv5QueryWarnDataDTO;
}
export interface InframonitoringtypesContainerCountsByReadyDTO {
/**
* @type integer
*/
notReady: number;
/**
* @type integer
*/
ready: number;
}
export interface InframonitoringtypesContainerCountsByStatusDTO {
/**
* @type integer
*/
completed: number;
/**
* @type integer
*/
containerCannotRun: number;
/**
* @type integer
*/
containerCreating: number;
/**
* @type integer
*/
crashLoopBackOff: number;
/**
* @type integer
*/
createContainerConfigError: number;
/**
* @type integer
*/
errImagePull: number;
/**
* @type integer
*/
error: number;
/**
* @type integer
*/
imagePullBackOff: number;
/**
* @type integer
*/
oomKilled: number;
/**
* @type integer
*/
running: number;
/**
* @type integer
*/
terminated: number;
/**
* @type integer
*/
unknown: number;
/**
* @type integer
*/
waiting: number;
}
export enum InframonitoringtypesContainerReadyDTO {
ready = 'ready',
not_ready = 'not_ready',
no_data = 'no_data',
}
export type InframonitoringtypesContainerRecordDTOMetaAnyOf = {
[key: string]: string;
};
/**
* @nullable
*/
export type InframonitoringtypesContainerRecordDTOMeta =
InframonitoringtypesContainerRecordDTOMetaAnyOf | null;
export enum InframonitoringtypesContainerStatusDTO {
running = 'running',
waiting = 'waiting',
terminated = 'terminated',
crashloopbackoff = 'crashloopbackoff',
imagepullbackoff = 'imagepullbackoff',
errimagepull = 'errimagepull',
createcontainerconfigerror = 'createcontainerconfigerror',
containercreating = 'containercreating',
oomkilled = 'oomkilled',
completed = 'completed',
error = 'error',
containercannotrun = 'containercannotrun',
unknown = 'unknown',
no_data = 'no_data',
}
export interface InframonitoringtypesContainerRecordDTO {
containerCountsByReady: InframonitoringtypesContainerCountsByReadyDTO;
containerCountsByStatus: InframonitoringtypesContainerCountsByStatusDTO;
/**
* @type string
*/
containerName: string;
/**
* @type number
* @format double
*/
cpu: number;
/**
* @type number
* @format double
*/
cpuLimitUtilization: number;
/**
* @type number
* @format double
*/
cpuRequestUtilization: number;
/**
* @type number
* @format double
*/
memory: number;
/**
* @type number
* @format double
*/
memoryLimitUtilization: number;
/**
* @type number
* @format double
*/
memoryRequestUtilization: number;
/**
* @type object,null
*/
meta: InframonitoringtypesContainerRecordDTOMeta;
/**
* @type string
*/
podUID: string;
ready: InframonitoringtypesContainerReadyDTO;
/**
* @type integer
* @format int64
*/
restarts: number;
status: InframonitoringtypesContainerStatusDTO;
}
export interface InframonitoringtypesContainersDTO {
/**
* @type boolean
*/
endTimeBeforeRetention: boolean;
/**
* @type array
*/
records: InframonitoringtypesContainerRecordDTO[];
/**
* @type integer
*/
total: number;
type: InframonitoringtypesResponseTypeDTO;
warning?: Querybuildertypesv5QueryWarnDataDTO;
}
export type InframonitoringtypesDaemonSetRecordDTOMetaAnyOf = {
[key: string]: string;
};
@@ -6305,6 +6474,33 @@ export interface InframonitoringtypesPostableClustersDTO {
start: number;
}
export interface InframonitoringtypesPostableContainersDTO {
/**
* @type integer
* @format int64
*/
end: number;
filter?: Querybuildertypesv5FilterDTO;
/**
* @type array,null
*/
groupBy?: Querybuildertypesv5GroupByKeyDTO[] | null;
/**
* @type integer
*/
limit: number;
/**
* @type integer
*/
offset?: number;
orderBy?: Querybuildertypesv5OrderByDTO;
/**
* @type integer
* @format int64
*/
start: number;
}
export interface InframonitoringtypesPostableDaemonSetsDTO {
/**
* @type integer
@@ -10843,6 +11039,14 @@ export type ListJobs200 = {
status: string;
};
export type ListContainers200 = {
data: InframonitoringtypesContainersDTO;
/**
* @type string
*/
status: string;
};
export type ListNamespaces200 = {
data: InframonitoringtypesNamespacesDTO;
/**

View File

@@ -48,6 +48,25 @@ func (provider *provider) addInfraMonitoringRoutes(router *mux.Router) error {
return err
}
if err := router.Handle("/api/v2/infra_monitoring/kube_containers", handler.New(
provider.authzMiddleware.ViewAccess(provider.infraMonitoringHandler.ListContainers),
handler.OpenAPIDef{
ID: "ListContainers",
Tags: []string{"inframonitoring"},
Summary: "List Kubernetes Containers for Infra Monitoring",
Description: "Returns a paginated list of Kubernetes containers with key kubeletstats metrics: CPU usage (cores), CPU request/limit utilization, memory working set, and memory request/limit utilization. Each container also reports health signals from the k8s_cluster receiver: status (kubectl-style display status derived from k8s.container.status.state + k8s.container.status.reason), restarts (absolute count from k8s.container.restarts), and ready (ready/not_ready from k8s.container.ready). The row identity is (k8s.pod.uid, k8s.container.name), stable across container restarts. Each container includes metadata attributes (k8s.container.name, k8s.pod.name, container.image.name, container.image.tag, k8s.namespace.name, k8s.node.name, k8s.cluster.name, and workload owner such as deployment/statefulset/daemonset/job). The response type is 'list' for the default (k8s.pod.uid, k8s.container.name) grouping (each row is one container with its current status and ready state) or 'grouped_list' for custom groupBy keys (each row aggregates containers in the group with per-status counts under containerCountsByStatus, per-readiness counts under containerCountsByReady, and restarts as the group sum). Status requires the optional k8s.container.status.state and k8s.container.status.reason metrics; when either is missing, status is omitted and a warning is returned while restarts and ready are still computed. Supports filtering via a filter expression, custom groupBy, ordering by any of the six metrics (cpu, cpu_request, cpu_limit, memory, memory_request, memory_limit), and pagination via offset/limit. Also reports whether the requested time range falls before the data retention boundary. Numeric metric fields (cpu, cpuRequestUtilization, cpuLimitUtilization, memory, memoryRequestUtilization, memoryLimitUtilization) and restarts return -1 as a sentinel when no data is available for that field.",
Request: new(inframonitoringtypes.PostableContainers),
RequestContentType: "application/json",
Response: new(inframonitoringtypes.Containers),
ResponseContentType: "application/json",
SuccessStatusCode: http.StatusOK,
ErrorStatusCodes: []int{http.StatusBadRequest, http.StatusUnauthorized},
Deprecated: false,
SecuritySchemes: newSecuritySchemes(types.RoleViewer),
})).Methods(http.MethodPost).GetError(); err != nil {
return err
}
if err := router.Handle("/api/v2/infra_monitoring/nodes", handler.New(
provider.authzMiddleware.ViewAccess(provider.infraMonitoringHandler.ListNodes),
handler.OpenAPIDef{

View File

@@ -49,17 +49,18 @@ var (
// checkSpecs is the single lookup table the module consults for a type's
// readiness contract. Every CheckType value must have an entry here.
var checkSpecs = map[inframonitoringtypes.CheckType]checkSpec{
inframonitoringtypes.CheckTypeHosts: hostsSpec,
inframonitoringtypes.CheckTypeProcesses: processesSpec,
inframonitoringtypes.CheckTypePods: podsSpec,
inframonitoringtypes.CheckTypeNodes: nodesSpec,
inframonitoringtypes.CheckTypeDeployments: deploymentsSpec,
inframonitoringtypes.CheckTypeDaemonsets: daemonsetsSpec,
inframonitoringtypes.CheckTypeStatefulsets: statefulsetsSpec,
inframonitoringtypes.CheckTypeJobs: jobsSpec,
inframonitoringtypes.CheckTypeNamespaces: namespacesSpec,
inframonitoringtypes.CheckTypeClusters: clustersSpec,
inframonitoringtypes.CheckTypeVolumes: volumesSpec,
inframonitoringtypes.CheckTypeHosts: hostsSpec,
inframonitoringtypes.CheckTypeProcesses: processesSpec,
inframonitoringtypes.CheckTypePods: podsSpec,
inframonitoringtypes.CheckTypeNodes: nodesSpec,
inframonitoringtypes.CheckTypeDeployments: deploymentsSpec,
inframonitoringtypes.CheckTypeDaemonsets: daemonsetsSpec,
inframonitoringtypes.CheckTypeStatefulsets: statefulsetsSpec,
inframonitoringtypes.CheckTypeJobs: jobsSpec,
inframonitoringtypes.CheckTypeNamespaces: namespacesSpec,
inframonitoringtypes.CheckTypeClusters: clustersSpec,
inframonitoringtypes.CheckTypeVolumes: volumesSpec,
inframonitoringtypes.CheckTypeKubeContainers: kubeContainersSpec,
}
// Per-type specs. Every metric and attribute is spelled out in its own spec
@@ -100,6 +101,42 @@ var processesSpec = checkSpec{
},
}
var kubeContainersSpec = checkSpec{
Buckets: []checkComponentBucket{
{
Component: componentKubeletStatsReceiver,
DefaultMetrics: []string{
"container.cpu.usage",
"container.memory.working_set",
},
OptionalMetrics: []string{
"k8s.container.cpu_request_utilization",
"k8s.container.cpu_limit_utilization",
"k8s.container.memory_request_utilization",
"k8s.container.memory_limit_utilization",
},
DocumentationLink: docLinkKubeletStatsReceiver,
},
{
Component: componentK8sClusterReceiver,
DefaultMetrics: []string{
"k8s.container.restarts",
"k8s.container.ready",
},
OptionalMetrics: []string{
"k8s.container.status.state",
"k8s.container.status.reason",
},
DocumentationLink: docLinkK8sClusterReceiver,
},
{
Component: componentK8sAttributesProcessor,
RequiredAttrs: []string{"k8s.pod.uid", "k8s.container.name"},
DocumentationLink: docLinkK8sAttributesProcessor,
},
},
}
var podsSpec = checkSpec{
Buckets: []checkComponentBucket{
{

View File

@@ -0,0 +1,808 @@
package implinframonitoring
import (
"context"
"fmt"
"slices"
"strings"
"github.com/SigNoz/signoz/pkg/querybuilder"
"github.com/SigNoz/signoz/pkg/telemetrymetrics"
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/valuer"
"github.com/huandu/go-sqlbuilder"
)
// buildContainerRecords assembles the page records, merging kubeletstats
// metrics (A-F from the querier) with the k8scluster health signals (status,
// restarts, ready). In list mode (isContainerRowInGroupBy=true) each group is a
// single container, so exactly one status/ready bucket is 1 and the single
// Status/Ready fields are derived from it; otherwise they stay NoData and only
// the per-group counts are populated.
func buildContainerRecords(
isContainerNameAndPodUIDInGroupBy bool,
resp *qbtypes.QueryRangeResponse,
pageGroups []map[string]string,
groupBy []qbtypes.GroupByKey,
metadataMap map[string]map[string]string,
statusCounts map[string]containerStatusCounts,
restartCounts map[string]int64,
readyCounts map[string]containerReadyCounts,
) []inframonitoringtypes.ContainerRecord {
metricsMap := parseFullQueryResponse(resp, groupBy)
records := make([]inframonitoringtypes.ContainerRecord, 0, len(pageGroups))
for _, labels := range pageGroups {
compositeKey := compositeKeyFromLabels(labels, groupBy)
record := inframonitoringtypes.ContainerRecord{ // initialize with default values
PodUID: labels[podUIDAttrKey],
ContainerName: labels[containerNameAttrKey],
Status: inframonitoringtypes.ContainerStatusNoData,
Ready: inframonitoringtypes.ContainerReadyNoData,
Restarts: -1,
CPU: -1,
CPURequestUtilization: -1,
CPULimitUtilization: -1,
Memory: -1,
MemoryRequestUtilization: -1,
MemoryLimitUtilization: -1,
Meta: map[string]string{},
}
if metrics, ok := metricsMap[compositeKey]; ok {
if v, exists := metrics["A"]; exists {
record.CPU = v
}
if v, exists := metrics["B"]; exists {
record.CPURequestUtilization = v
}
if v, exists := metrics["C"]; exists {
record.CPULimitUtilization = v
}
if v, exists := metrics["D"]; exists {
record.Memory = v
}
if v, exists := metrics["E"]; exists {
record.MemoryRequestUtilization = v
}
if v, exists := metrics["F"]; exists {
record.MemoryLimitUtilization = v
}
}
if statusCountsForGroup, ok := statusCounts[compositeKey]; ok {
record.ContainerCountsByStatus = containerStatusCountsToResponse(statusCountsForGroup)
// In list mode each group is one container; the count==1 bucket identifies the status.
if isContainerNameAndPodUIDInGroupBy {
switch {
case statusCountsForGroup.Running == 1:
record.Status = inframonitoringtypes.ContainerStatusRunning
case statusCountsForGroup.Waiting == 1:
record.Status = inframonitoringtypes.ContainerStatusWaiting
case statusCountsForGroup.Terminated == 1:
record.Status = inframonitoringtypes.ContainerStatusTerminated
case statusCountsForGroup.CrashLoopBackOff == 1:
record.Status = inframonitoringtypes.ContainerStatusCrashLoopBackOff
case statusCountsForGroup.ImagePullBackOff == 1:
record.Status = inframonitoringtypes.ContainerStatusImagePullBackOff
case statusCountsForGroup.ErrImagePull == 1:
record.Status = inframonitoringtypes.ContainerStatusErrImagePull
case statusCountsForGroup.CreateContainerConfigError == 1:
record.Status = inframonitoringtypes.ContainerStatusCreateContainerConfigError
case statusCountsForGroup.ContainerCreating == 1:
record.Status = inframonitoringtypes.ContainerStatusContainerCreating
case statusCountsForGroup.OOMKilled == 1:
record.Status = inframonitoringtypes.ContainerStatusOOMKilled
case statusCountsForGroup.Completed == 1:
record.Status = inframonitoringtypes.ContainerStatusCompleted
case statusCountsForGroup.Error == 1:
record.Status = inframonitoringtypes.ContainerStatusError
case statusCountsForGroup.ContainerCannotRun == 1:
record.Status = inframonitoringtypes.ContainerStatusContainerCannotRun
case statusCountsForGroup.Unknown == 1:
record.Status = inframonitoringtypes.ContainerStatusUnknown
}
}
}
// Restart count: container's own count (list mode) or group sum (grouped mode).
if restartCountForGroup, ok := restartCounts[compositeKey]; ok {
record.Restarts = restartCountForGroup
}
if readyCountsForGroup, ok := readyCounts[compositeKey]; ok {
record.ContainerCountsByReady = containerReadyCountsToResponse(readyCountsForGroup)
// In list mode each group is one container; exactly one of ready/not_ready is 1.
if isContainerNameAndPodUIDInGroupBy {
switch {
case readyCountsForGroup.Ready == 1:
record.Ready = inframonitoringtypes.ContainerReadyReady
case readyCountsForGroup.NotReady == 1:
record.Ready = inframonitoringtypes.ContainerReadyNotReady
}
}
}
if attrs, ok := metadataMap[compositeKey]; ok {
for k, v := range attrs {
record.Meta[k] = v
}
}
records = append(records, record)
}
return records
}
func (m *module) getTopContainerGroups(
ctx context.Context,
orgID valuer.UUID,
req *inframonitoringtypes.PostableContainers,
metadataMap map[string]map[string]string,
) ([]map[string]string, error) {
orderByKey := req.OrderBy.Key.Name
if orderByKey == inframonitoringtypes.ContainerNameAttrKey {
return inframonitoringtypes.PaginateMetadataByName(metadataMap, req.GroupBy, req.OrderBy.Direction, req.Offset, req.Limit, inframonitoringtypes.ContainerNameAttrKey), nil
}
queryNamesForOrderBy := orderByToContainersQueryNames[orderByKey]
rankingQueryName := queryNamesForOrderBy[len(queryNamesForOrderBy)-1]
topReq := &qbtypes.QueryRangeRequest{
Start: uint64(req.Start),
End: uint64(req.End),
RequestType: qbtypes.RequestTypeScalar,
CompositeQuery: qbtypes.CompositeQuery{
Queries: make([]qbtypes.QueryEnvelope, 0, len(queryNamesForOrderBy)),
},
}
for _, envelope := range m.newContainersTableListQuery().CompositeQuery.Queries {
if !slices.Contains(queryNamesForOrderBy, envelope.GetQueryName()) {
continue
}
copied := envelope
if copied.Type == qbtypes.QueryTypeBuilder {
existingExpr := ""
if f := copied.GetFilter(); f != nil {
existingExpr = f.Expression
}
reqFilterExpr := ""
if req.Filter != nil {
reqFilterExpr = req.Filter.Expression
}
merged := mergeFilterExpressions(existingExpr, reqFilterExpr)
copied.SetFilter(&qbtypes.Filter{Expression: merged})
copied.SetGroupBy(req.GroupBy)
}
topReq.CompositeQuery.Queries = append(topReq.CompositeQuery.Queries, copied)
}
resp, err := m.querier.QueryRange(ctx, orgID, topReq)
if err != nil {
return nil, err
}
allMetricGroups := parseAndSortGroups(resp, rankingQueryName, req.GroupBy, req.OrderBy.Direction)
return paginateWithBackfill(allMetricGroups, metadataMap, req.GroupBy, req.Offset, req.Limit), nil
}
func (m *module) getContainersTableMetadata(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableContainers) (map[string]map[string]string, error) {
var nonGroupByAttrs []string
for _, key := range containerAttrKeysForMetadata {
if !isKeyInGroupByAttrs(req.GroupBy, key) {
nonGroupByAttrs = append(nonGroupByAttrs, key)
}
}
return m.getMetadata(ctx, orgID, containersTableMetricNamesList, req.GroupBy, nonGroupByAttrs, req.Filter, req.Start, req.End)
}
// getPerGroupContainerStatusCountsWithReqMetricChecks gates
// getPerGroupContainerStatusCounts on the required metrics being present. If
// either of containerStatusMetricNamesList (k8s.container.status.state /
// k8s.container.status.reason) has never been reported, it skips the query and
// returns a warning instead (the status derivation needs both). Otherwise it
// runs the query. The returned counts map is empty (never nil) when gated off.
func (m *module) getPerGroupContainerStatusCountsWithReqMetricChecks(
ctx context.Context,
start, end int64,
filter *qbtypes.Filter,
groupBy []qbtypes.GroupByKey,
pageGroups []map[string]string,
) (map[string]containerStatusCounts, *qbtypes.QueryWarnData, error) {
present, err := m.getMetricsExistence(ctx, containerStatusMetricNamesList)
if err != nil {
return nil, nil, err
}
var missing []string
for _, name := range containerStatusMetricNamesList {
if !present[name] {
missing = append(missing, name)
}
}
if len(missing) > 0 {
warning := &qbtypes.QueryWarnData{
Message: fmt.Sprintf(
"Container status could not be computed: required metric(s) not found: %s. "+
"Enable the optional k8s.container.status.state and k8s.container.status.reason "+
"metrics in the k8s_cluster receiver to see container statuses.",
strings.Join(missing, ", "),
),
Url: docLinkK8sClusterReceiver,
}
return map[string]containerStatusCounts{}, warning, nil
}
counts, err := m.getPerGroupContainerStatusCounts(ctx, start, end, filter, groupBy, pageGroups)
if err != nil {
return nil, nil, err
}
return counts, nil, nil
}
// getPerGroupContainerStatusCounts computes per-group counts of distinct
// containers bucketed by their latest kubectl-style display status in window.
// Caller must ensure the required metrics exist
// (getPerGroupContainerStatusCountsWithReqMetricChecks).
//
// Row identity is (pod_uid, container_name). Pipeline:
//
// state_fps / container_state: current k8s.container.status.state per container
// (argMaxIf(state, unix_milli, value=1) — safe because a
// container always has exactly one active state).
// reason_fps / container_reason: current active k8s.container.status.reason per container
// (two-level: HAVING is_active=1 excludes stale reasons of
// recovered containers, then recency picks the latest).
// container_status: display status per container (reason > state fallback).
// countContainersPerStatus: per-group uniqExactIf over distinct (pod_uid, container_name).
//
// Groups absent from the result map have implicit zero counts (caller default).
func (m *module) getPerGroupContainerStatusCounts(
ctx context.Context,
start, end int64,
filter *qbtypes.Filter,
groupBy []qbtypes.GroupByKey,
pageGroups []map[string]string,
) (map[string]containerStatusCounts, error) {
if len(pageGroups) == 0 || len(groupBy) == 0 {
return map[string]containerStatusCounts{}, nil
}
userFilterExpr := ""
if filter != nil {
userFilterExpr = filter.Expression
}
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
// Built once; identical across the two fps CTEs (buildFilterClause hits the
// metadata store + parses the expression). AddWhereClause only reads it.
var (
filterClause *sqlbuilder.WhereClause
err error
)
if mergedFilterExpr != "" {
filterClause, err = m.buildFilterClause(ctx, &qbtypes.Filter{Expression: mergedFilterExpr}, start, end)
if err != nil {
return nil, err
}
}
// ----- state_fps (carries groupBy cols) -----
stateFps := sqlbuilder.NewSelectBuilder()
stateFpsCols := []string{
"fingerprint",
fmt.Sprintf("JSONExtractString(labels, %s) AS pod_uid", stateFps.Var(podUIDAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", stateFps.Var(containerNameAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS state", stateFps.Var(containerStatusStateAttrKey)),
}
for _, key := range groupBy {
stateFpsCols = append(stateFpsCols,
fmt.Sprintf("JSONExtractString(labels, %s) AS %s", stateFps.Var(key.Name), quoteIdentifier(key.Name)),
)
}
stateFps.Select(stateFpsCols...)
stateFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
stateFps.Where(
stateFps.E("metric_name", containerStatusStateMetricName),
stateFps.GE("unix_milli", tsAdjustedStart),
stateFps.LE("unix_milli", flooredEndMs),
)
if filterClause != nil {
stateFps.AddWhereClause(filterClause)
}
stateFpsGroupBy := []string{"fingerprint", "pod_uid", "container_name", "state"}
for _, key := range groupBy {
stateFpsGroupBy = append(stateFpsGroupBy, quoteIdentifier(key.Name))
}
stateFps.GroupBy(stateFpsGroupBy...)
stateFpsSQL, stateFpsArgs := stateFps.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- container_state (current state per container; single-pass argMaxIf) -----
containerState := sqlbuilder.NewSelectBuilder()
containerStateCols := []string{
"fps.pod_uid AS pod_uid",
"fps.container_name AS container_name",
}
for _, key := range groupBy {
col := quoteIdentifier(key.Name)
containerStateCols = append(containerStateCols, fmt.Sprintf("argMax(fps.%s, samples.unix_milli) AS %s", col, col))
}
containerStateCols = append(containerStateCols,
fmt.Sprintf("argMaxIf(fps.state, samples.unix_milli, samples.%s = 1) AS state", valueCol),
)
containerState.Select(containerStateCols...)
containerState.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN state_fps AS fps ON samples.fingerprint = fps.fingerprint",
telemetrymetrics.DBName, distributedSamplesTable,
))
containerState.Where(
containerState.E("samples.metric_name", containerStatusStateMetricName),
containerState.GE("samples.unix_milli", samplesStartMs),
containerState.L("samples.unix_milli", flooredEndMs),
"fps.pod_uid != ''",
)
containerState.GroupBy("fps.pod_uid", "fps.container_name")
containerStateSQL, containerStateArgs := containerState.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- reason_fps -----
reasonFps := sqlbuilder.NewSelectBuilder()
reasonFps.Select(
"fingerprint",
fmt.Sprintf("JSONExtractString(labels, %s) AS pod_uid", reasonFps.Var(podUIDAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", reasonFps.Var(containerNameAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS reason", reasonFps.Var(containerStatusReasonAttrKey)),
)
reasonFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
reasonFps.Where(
reasonFps.E("metric_name", containerStatusReasonMetricName),
reasonFps.GE("unix_milli", tsAdjustedStart),
reasonFps.LE("unix_milli", flooredEndMs),
)
if filterClause != nil {
reasonFps.AddWhereClause(filterClause)
}
reasonFps.GroupBy("fingerprint", "pod_uid", "container_name", "reason")
reasonFpsSQL, reasonFpsArgs := reasonFps.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- container_reason -----
// Inner: latest value per (pod, container, reason) -> kills stale fingerprints
// from old container incarnations; keep only active (=1). Outer: the current
// incarnation's reason wins via recency (argMax by last_active_ms). We do NOT
// priority-rank reasons at container grain -- "worst wins" is a pod-level concept
// (aggregating multiple containers into one status); a single container has one
// reason at a time, and multiple only appear active via container.id churn on
// restart, where the latest (current) incarnation is the right answer.
reasonInner := sqlbuilder.NewSelectBuilder()
reasonInner.Select(
"fps.pod_uid AS pod_uid",
"fps.container_name AS container_name",
"fps.reason AS reason",
fmt.Sprintf("argMax(samples.%s, samples.unix_milli) AS is_active", valueCol),
"max(samples.unix_milli) AS last_active_ms",
)
reasonInner.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN reason_fps AS fps ON samples.fingerprint = fps.fingerprint",
telemetrymetrics.DBName, distributedSamplesTable,
))
reasonInner.Where(
reasonInner.E("samples.metric_name", containerStatusReasonMetricName),
reasonInner.GE("samples.unix_milli", samplesStartMs),
reasonInner.L("samples.unix_milli", flooredEndMs),
"fps.pod_uid != ''",
)
reasonInner.GroupBy("fps.pod_uid", "fps.container_name", "fps.reason")
reasonInner.Having("is_active = 1")
reasonInnerSQL, reasonInnerArgs := reasonInner.BuildWithFlavor(sqlbuilder.ClickHouse)
containerReasonSQL := fmt.Sprintf(
"SELECT pod_uid, container_name, argMax(reason, last_active_ms) AS active_reason FROM (%s) GROUP BY pod_uid, container_name",
reasonInnerSQL,
)
// ----- container_status (display status per container) -----
// container reason > state fallback (running/terminated/waiting).
displayStatusExpr := `multiIf(
cr.active_reason != '', cr.active_reason,
st.state = 'running', 'Running',
st.state = 'terminated', 'Terminated',
st.state = 'waiting', 'Waiting',
'Unknown')`
containerStatusSelectCols := []string{
"st.pod_uid AS pod_uid",
"st.container_name AS container_name",
}
for _, key := range groupBy {
col := quoteIdentifier(key.Name)
containerStatusSelectCols = append(containerStatusSelectCols, fmt.Sprintf("st.%s AS %s", col, col))
}
containerStatusSelectCols = append(containerStatusSelectCols, displayStatusExpr+" AS display_status")
containerStatusSQL := fmt.Sprintf(
"SELECT %s FROM container_state AS st LEFT JOIN container_reason AS cr ON st.pod_uid = cr.pod_uid AND st.container_name = cr.container_name",
strings.Join(containerStatusSelectCols, ", "),
)
// ----- countContainersPerStatus (outer SELECT) -----
// Fixed status order; MUST match the containerStatusCounts assignment below.
statusCountCols := []string{
"uniqExactIf((pod_uid, container_name), display_status = 'Running') AS running_count",
"uniqExactIf((pod_uid, container_name), display_status = 'Waiting') AS waiting_count",
"uniqExactIf((pod_uid, container_name), display_status = 'Terminated') AS terminated_count",
"uniqExactIf((pod_uid, container_name), display_status = 'CrashLoopBackOff') AS crash_loop_back_off_count",
"uniqExactIf((pod_uid, container_name), display_status = 'ImagePullBackOff') AS image_pull_back_off_count",
"uniqExactIf((pod_uid, container_name), display_status = 'ErrImagePull') AS err_image_pull_count",
"uniqExactIf((pod_uid, container_name), display_status = 'CreateContainerConfigError') AS create_container_config_error_count",
"uniqExactIf((pod_uid, container_name), display_status = 'ContainerCreating') AS container_creating_count",
"uniqExactIf((pod_uid, container_name), display_status = 'OOMKilled') AS oom_killed_count",
"uniqExactIf((pod_uid, container_name), display_status = 'Completed') AS completed_count",
"uniqExactIf((pod_uid, container_name), display_status = 'Error') AS error_count",
"uniqExactIf((pod_uid, container_name), display_status = 'ContainerCannotRun') AS container_cannot_run_count",
"uniqExactIf((pod_uid, container_name), display_status = 'Unknown') AS unknown_count",
}
countSelectCols := make([]string, 0, len(groupBy)+len(statusCountCols))
countGroupBy := make([]string, 0, len(groupBy))
for _, key := range groupBy {
col := quoteIdentifier(key.Name)
countSelectCols = append(countSelectCols, col)
countGroupBy = append(countGroupBy, col)
}
countSelectCols = append(countSelectCols, statusCountCols...)
countSQL := fmt.Sprintf(
"SELECT %s FROM container_status GROUP BY %s",
strings.Join(countSelectCols, ", "),
strings.Join(countGroupBy, ", "),
)
// Combine CTEs + outer. Arg order mirrors CTE declaration order.
cteFragments := []string{
fmt.Sprintf("state_fps AS (%s)", stateFpsSQL),
fmt.Sprintf("container_state AS (%s)", containerStateSQL),
fmt.Sprintf("reason_fps AS (%s)", reasonFpsSQL),
fmt.Sprintf("container_reason AS (%s)", containerReasonSQL),
fmt.Sprintf("container_status AS (%s)", containerStatusSQL),
}
finalSQL := querybuilder.CombineCTEs(cteFragments) + countSQL
finalArgs := querybuilder.PrependArgs([][]any{
stateFpsArgs, containerStateArgs, reasonFpsArgs, reasonInnerArgs,
}, nil)
rows, err := m.telemetryStore.ClickhouseDB().Query(ctx, finalSQL, finalArgs...)
if err != nil {
return nil, err
}
defer rows.Close()
result := make(map[string]containerStatusCounts)
for rows.Next() {
groupVals := make([]string, len(groupBy))
counts := make([]uint64, len(statusCountCols))
scanPtrs := make([]any, 0, len(groupBy)+len(statusCountCols))
for i := range groupVals {
scanPtrs = append(scanPtrs, &groupVals[i])
}
for i := range counts {
scanPtrs = append(scanPtrs, &counts[i])
}
if err := rows.Scan(scanPtrs...); err != nil {
return nil, err
}
result[compositeKeyFromList(groupVals)] = containerStatusCounts{
Running: int(counts[0]),
Waiting: int(counts[1]),
Terminated: int(counts[2]),
CrashLoopBackOff: int(counts[3]),
ImagePullBackOff: int(counts[4]),
ErrImagePull: int(counts[5]),
CreateContainerConfigError: int(counts[6]),
ContainerCreating: int(counts[7]),
OOMKilled: int(counts[8]),
Completed: int(counts[9]),
Error: int(counts[10]),
ContainerCannotRun: int(counts[11]),
Unknown: int(counts[12]),
}
}
if err := rows.Err(); err != nil {
return nil, err
}
return result, nil
}
// getPerGroupContainerRestartCounts computes the absolute container restart count
// per group from k8s.container.restarts (default-enabled, so no existence gate).
// In list mode (groupBy contains the row identity) each group is one container ->
// its restart count; in grouped mode it's the summed restarts across all
// containers in the group. argMax(value, unix_milli) per (pod, container) takes
// the current cumulative count from the latest incarnation, then sum.
//
// restart_fps: fp ↔ (pod_uid, container_name, groupBy cols) from time_series.
// container_restarts: INNER JOIN samples, latest restartCount per (pod, container).
// (outer): per-group sum(restart_count).
//
// Groups absent from the result map have no data (caller default).
func (m *module) getPerGroupContainerRestartCounts(
ctx context.Context,
start, end int64,
filter *qbtypes.Filter,
groupBy []qbtypes.GroupByKey,
pageGroups []map[string]string,
) (map[string]int64, error) {
if len(pageGroups) == 0 || len(groupBy) == 0 {
return map[string]int64{}, nil
}
userFilterExpr := ""
if filter != nil {
userFilterExpr = filter.Expression
}
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
var (
filterClause *sqlbuilder.WhereClause
err error
)
if mergedFilterExpr != "" {
filterClause, err = m.buildFilterClause(ctx, &qbtypes.Filter{Expression: mergedFilterExpr}, start, end)
if err != nil {
return nil, err
}
}
// ----- restart_fps (carries groupBy cols) -----
restartFps := sqlbuilder.NewSelectBuilder()
restartFpsCols := []string{
"fingerprint",
fmt.Sprintf("JSONExtractString(labels, %s) AS pod_uid", restartFps.Var(podUIDAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", restartFps.Var(containerNameAttrKey)),
}
for _, key := range groupBy {
restartFpsCols = append(restartFpsCols,
fmt.Sprintf("JSONExtractString(labels, %s) AS %s", restartFps.Var(key.Name), quoteIdentifier(key.Name)),
)
}
restartFps.Select(restartFpsCols...)
restartFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
restartFps.Where(
restartFps.E("metric_name", containerRestartsMetricName),
restartFps.GE("unix_milli", tsAdjustedStart),
restartFps.LE("unix_milli", flooredEndMs),
)
if filterClause != nil {
restartFps.AddWhereClause(filterClause)
}
restartFpsGroupBy := []string{"fingerprint", "pod_uid", "container_name"}
for _, key := range groupBy {
restartFpsGroupBy = append(restartFpsGroupBy, quoteIdentifier(key.Name))
}
restartFps.GroupBy(restartFpsGroupBy...)
restartFpsSQL, restartFpsArgs := restartFps.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- container_restarts (latest cumulative count per container) -----
containerRestarts := sqlbuilder.NewSelectBuilder()
containerRestartsCols := []string{
"fps.pod_uid AS pod_uid",
"fps.container_name AS container_name",
}
for _, key := range groupBy {
col := quoteIdentifier(key.Name)
containerRestartsCols = append(containerRestartsCols, fmt.Sprintf("argMax(fps.%s, samples.unix_milli) AS %s", col, col))
}
containerRestartsCols = append(containerRestartsCols, fmt.Sprintf("argMax(samples.%s, samples.unix_milli) AS restart_count", valueCol))
containerRestarts.Select(containerRestartsCols...)
containerRestarts.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN restart_fps AS fps ON samples.fingerprint = fps.fingerprint",
telemetrymetrics.DBName, distributedSamplesTable,
))
containerRestarts.Where(
containerRestarts.E("samples.metric_name", containerRestartsMetricName),
containerRestarts.GE("samples.unix_milli", samplesStartMs),
containerRestarts.L("samples.unix_milli", flooredEndMs),
"fps.pod_uid != ''",
)
containerRestarts.GroupBy("fps.pod_uid", "fps.container_name")
containerRestartsSQL, containerRestartsArgs := containerRestarts.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- outer: per-group sum across containers -----
sumSelectCols := make([]string, 0, len(groupBy)+1)
sumGroupBy := make([]string, 0, len(groupBy))
for _, key := range groupBy {
col := quoteIdentifier(key.Name)
sumSelectCols = append(sumSelectCols, col)
sumGroupBy = append(sumGroupBy, col)
}
sumSelectCols = append(sumSelectCols, "sum(restart_count) AS total_restarts")
sumSQL := fmt.Sprintf(
"SELECT %s FROM container_restarts GROUP BY %s",
strings.Join(sumSelectCols, ", "),
strings.Join(sumGroupBy, ", "),
)
cteFragments := []string{
fmt.Sprintf("restart_fps AS (%s)", restartFpsSQL),
fmt.Sprintf("container_restarts AS (%s)", containerRestartsSQL),
}
finalSQL := querybuilder.CombineCTEs(cteFragments) + sumSQL
finalArgs := querybuilder.PrependArgs([][]any{restartFpsArgs, containerRestartsArgs}, nil)
rows, err := m.telemetryStore.ClickhouseDB().Query(ctx, finalSQL, finalArgs...)
if err != nil {
return nil, err
}
defer rows.Close()
result := make(map[string]int64)
for rows.Next() {
groupVals := make([]string, len(groupBy))
var totalRestarts float64
scanPtrs := make([]any, 0, len(groupBy)+1)
for i := range groupVals {
scanPtrs = append(scanPtrs, &groupVals[i])
}
scanPtrs = append(scanPtrs, &totalRestarts)
if err := rows.Scan(scanPtrs...); err != nil {
return nil, err
}
result[compositeKeyFromList(groupVals)] = int64(totalRestarts)
}
if err := rows.Err(); err != nil {
return nil, err
}
return result, nil
}
// getPerGroupContainerReadyCounts computes per-group counts of distinct
// containers bucketed by their latest readiness (k8s.container.ready, a single
// 0/1 gauge per container — default-enabled, so no existence gate). ready = latest
// value 1; not_ready = latest value < 1.
//
// ready_fps: fp ↔ (pod_uid, container_name, groupBy cols) from time_series.
// container_ready: INNER JOIN samples, latest ready value per (pod, container).
// (outer): per-group uniqExactIf over distinct (pod_uid, container_name).
//
// Groups absent from the result map have no data (caller default).
func (m *module) getPerGroupContainerReadyCounts(
ctx context.Context,
start, end int64,
filter *qbtypes.Filter,
groupBy []qbtypes.GroupByKey,
pageGroups []map[string]string,
) (map[string]containerReadyCounts, error) {
if len(pageGroups) == 0 || len(groupBy) == 0 {
return map[string]containerReadyCounts{}, nil
}
userFilterExpr := ""
if filter != nil {
userFilterExpr = filter.Expression
}
mergedFilterExpr := mergeFilterExpressions(userFilterExpr, buildPageGroupsFilterExpr(pageGroups))
samplesStartMs, flooredEndMs, tsAdjustedStart, _, localTimeSeriesTable, distributedSamplesTable, _ := alignedMetricWindow(start, end)
valueCol := telemetrymetrics.ValueColumnForSamplesTable(distributedSamplesTable)
var (
filterClause *sqlbuilder.WhereClause
err error
)
if mergedFilterExpr != "" {
filterClause, err = m.buildFilterClause(ctx, &qbtypes.Filter{Expression: mergedFilterExpr}, start, end)
if err != nil {
return nil, err
}
}
// ----- ready_fps (carries groupBy cols) -----
readyFps := sqlbuilder.NewSelectBuilder()
readyFpsCols := []string{
"fingerprint",
fmt.Sprintf("JSONExtractString(labels, %s) AS pod_uid", readyFps.Var(podUIDAttrKey)),
fmt.Sprintf("JSONExtractString(labels, %s) AS container_name", readyFps.Var(containerNameAttrKey)),
}
for _, key := range groupBy {
readyFpsCols = append(readyFpsCols,
fmt.Sprintf("JSONExtractString(labels, %s) AS %s", readyFps.Var(key.Name), quoteIdentifier(key.Name)),
)
}
readyFps.Select(readyFpsCols...)
readyFps.From(fmt.Sprintf("%s.%s", telemetrymetrics.DBName, localTimeSeriesTable))
readyFps.Where(
readyFps.E("metric_name", containerReadyMetricName),
readyFps.GE("unix_milli", tsAdjustedStart),
readyFps.LE("unix_milli", flooredEndMs),
)
if filterClause != nil {
readyFps.AddWhereClause(filterClause)
}
readyFpsGroupBy := []string{"fingerprint", "pod_uid", "container_name"}
for _, key := range groupBy {
readyFpsGroupBy = append(readyFpsGroupBy, quoteIdentifier(key.Name))
}
readyFps.GroupBy(readyFpsGroupBy...)
readyFpsSQL, readyFpsArgs := readyFps.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- container_ready (latest 0/1 per container) -----
containerReady := sqlbuilder.NewSelectBuilder()
containerReadyCols := []string{
"fps.pod_uid AS pod_uid",
"fps.container_name AS container_name",
}
for _, key := range groupBy {
col := quoteIdentifier(key.Name)
containerReadyCols = append(containerReadyCols, fmt.Sprintf("argMax(fps.%s, samples.unix_milli) AS %s", col, col))
}
containerReadyCols = append(containerReadyCols, fmt.Sprintf("argMax(samples.%s, samples.unix_milli) AS ready_value", valueCol))
containerReady.Select(containerReadyCols...)
containerReady.From(fmt.Sprintf(
"%s.%s AS samples INNER JOIN ready_fps AS fps ON samples.fingerprint = fps.fingerprint",
telemetrymetrics.DBName, distributedSamplesTable,
))
containerReady.Where(
containerReady.E("samples.metric_name", containerReadyMetricName),
containerReady.GE("samples.unix_milli", samplesStartMs),
containerReady.L("samples.unix_milli", flooredEndMs),
"fps.pod_uid != ''",
)
containerReady.GroupBy("fps.pod_uid", "fps.container_name")
containerReadySQL, containerReadyArgs := containerReady.BuildWithFlavor(sqlbuilder.ClickHouse)
// ----- outer: per-group ready / not_ready counts over distinct containers -----
countSelectCols := make([]string, 0, len(groupBy)+2)
countGroupBy := make([]string, 0, len(groupBy))
for _, key := range groupBy {
col := quoteIdentifier(key.Name)
countSelectCols = append(countSelectCols, col)
countGroupBy = append(countGroupBy, col)
}
countSelectCols = append(countSelectCols,
"uniqExactIf((pod_uid, container_name), ready_value = 1) AS ready",
"uniqExactIf((pod_uid, container_name), ready_value < 1) AS not_ready",
)
countSQL := fmt.Sprintf(
"SELECT %s FROM container_ready GROUP BY %s",
strings.Join(countSelectCols, ", "),
strings.Join(countGroupBy, ", "),
)
cteFragments := []string{
fmt.Sprintf("ready_fps AS (%s)", readyFpsSQL),
fmt.Sprintf("container_ready AS (%s)", containerReadySQL),
}
finalSQL := querybuilder.CombineCTEs(cteFragments) + countSQL
finalArgs := querybuilder.PrependArgs([][]any{readyFpsArgs, containerReadyArgs}, nil)
rows, err := m.telemetryStore.ClickhouseDB().Query(ctx, finalSQL, finalArgs...)
if err != nil {
return nil, err
}
defer rows.Close()
result := make(map[string]containerReadyCounts)
for rows.Next() {
groupVals := make([]string, len(groupBy))
var ready, notReady uint64
scanPtrs := make([]any, 0, len(groupBy)+2)
for i := range groupVals {
scanPtrs = append(scanPtrs, &groupVals[i])
}
scanPtrs = append(scanPtrs, &ready, &notReady)
if err := rows.Scan(scanPtrs...); err != nil {
return nil, err
}
result[compositeKeyFromList(groupVals)] = containerReadyCounts{
Ready: int(ready),
NotReady: int(notReady),
}
}
if err := rows.Err(); err != nil {
return nil, err
}
return result, nil
}

View File

@@ -0,0 +1,208 @@
package implinframonitoring
import (
"github.com/SigNoz/signoz/pkg/types/inframonitoringtypes"
"github.com/SigNoz/signoz/pkg/types/metrictypes"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)
const containerNameAttrKey = inframonitoringtypes.ContainerNameAttrKey
const (
containerStatusStateAttrKey = "k8s.container.status.state"
containerStatusReasonAttrKey = "k8s.container.status.reason"
)
const (
containerStatusStateMetricName = "k8s.container.status.state"
containerStatusReasonMetricName = "k8s.container.status.reason"
containerReadyMetricName = "k8s.container.ready"
containerRestartsMetricName = "k8s.container.restarts"
)
// containerStatusMetricNamesList are the metrics required to derive the
// kubectl-style container display status. Gated by
// getPerGroupContainerStatusCountsWithReqMetricChecks — if either is missing,
// status is skipped and a warning is returned.
var containerStatusMetricNamesList = []string{
containerStatusStateMetricName,
containerStatusReasonMetricName,
}
var containerNameGroupByKey = qbtypes.GroupByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: containerNameAttrKey,
FieldContext: telemetrytypes.FieldContextResource,
FieldDataType: telemetrytypes.FieldDataTypeString,
},
}
// containerRowGroupBy is the default row-identity groupBy for the containers
// list: (k8s.pod.uid, k8s.container.name). Stable across container restarts.
// podUIDGroupByKey is shared with the pods module (pods_constants.go).
var containerRowGroupBy = []qbtypes.GroupByKey{podUIDGroupByKey, containerNameGroupByKey}
// containersTableMetricNamesList are the metrics that carry the container
// attributes; used for metadata resolution and the retention check.
var containersTableMetricNamesList = []string{
"container.cpu.usage",
"k8s.container.cpu_request_utilization",
"k8s.container.cpu_limit_utilization",
"container.memory.working_set",
"k8s.container.memory_request_utilization",
"k8s.container.memory_limit_utilization",
"k8s.container.restarts",
"k8s.container.ready",
"k8s.container.status.reason",
"k8s.container.status.state",
}
var containerAttrKeysForMetadata = []string{
"k8s.pod.uid",
"k8s.container.name",
"k8s.pod.name",
"container.image.name",
"container.image.tag",
"k8s.namespace.name",
"k8s.node.name",
"k8s.deployment.name",
"k8s.statefulset.name",
"k8s.daemonset.name",
"k8s.job.name",
"k8s.cronjob.name",
"k8s.cluster.name",
}
var orderByToContainersQueryNames = map[string][]string{
inframonitoringtypes.ContainersOrderByCPU: {"A"},
inframonitoringtypes.ContainersOrderByCPURequest: {"B"},
inframonitoringtypes.ContainersOrderByCPULimit: {"C"},
inframonitoringtypes.ContainersOrderByMemory: {"D"},
inframonitoringtypes.ContainersOrderByMemoryRequest: {"E"},
inframonitoringtypes.ContainersOrderByMemoryLimit: {"F"},
}
// newContainersTableListQuery builds the composite QB v5 request for the
// containers list (kubeletstats usage/utilization). Status, restarts and ready
// come from k8sclusterreceiver via dedicated queries (works for both list and
// grouped_list modes), so not included here.
func (m *module) newContainersTableListQuery() *qbtypes.QueryRangeRequest {
queries := []qbtypes.QueryEnvelope{
// Query A: CPU usage (cores)
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Name: "A",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "container.cpu.usage",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
ReduceTo: qbtypes.ReduceToAvg,
},
},
GroupBy: containerRowGroupBy,
Disabled: false,
},
},
// Query B: CPU request utilization
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Name: "B",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "k8s.container.cpu_request_utilization",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationAvg,
ReduceTo: qbtypes.ReduceToAvg,
},
},
GroupBy: containerRowGroupBy,
Disabled: false,
},
},
// Query C: CPU limit utilization
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Name: "C",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "k8s.container.cpu_limit_utilization",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationAvg,
ReduceTo: qbtypes.ReduceToAvg,
},
},
GroupBy: containerRowGroupBy,
Disabled: false,
},
},
// Query D: Memory working set
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Name: "D",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "container.memory.working_set",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationSum,
ReduceTo: qbtypes.ReduceToAvg,
},
},
GroupBy: containerRowGroupBy,
Disabled: false,
},
},
// Query E: Memory request utilization
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Name: "E",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "k8s.container.memory_request_utilization",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationAvg,
ReduceTo: qbtypes.ReduceToAvg,
},
},
GroupBy: containerRowGroupBy,
Disabled: false,
},
},
// Query F: Memory limit utilization
{
Type: qbtypes.QueryTypeBuilder,
Spec: qbtypes.QueryBuilderQuery[qbtypes.MetricAggregation]{
Name: "F",
Signal: telemetrytypes.SignalMetrics,
Aggregations: []qbtypes.MetricAggregation{
{
MetricName: "k8s.container.memory_limit_utilization",
TimeAggregation: metrictypes.TimeAggregationAvg,
SpaceAggregation: metrictypes.SpaceAggregationAvg,
ReduceTo: qbtypes.ReduceToAvg,
},
},
GroupBy: containerRowGroupBy,
Disabled: false,
},
},
}
return &qbtypes.QueryRangeRequest{
RequestType: qbtypes.RequestTypeScalar,
CompositeQuery: qbtypes.CompositeQuery{
Queries: queries,
},
}
}

View File

@@ -94,6 +94,30 @@ func (h *handler) ListPods(rw http.ResponseWriter, req *http.Request) {
render.Success(rw, http.StatusOK, result)
}
func (h *handler) ListContainers(rw http.ResponseWriter, req *http.Request) {
claims, err := authtypes.ClaimsFromContext(req.Context())
if err != nil {
render.Error(rw, err)
return
}
orgID := valuer.MustNewUUID(claims.OrgID)
var parsedReq inframonitoringtypes.PostableContainers
if err := binding.JSON.BindBody(req.Body, &parsedReq); err != nil {
render.Error(rw, err)
return
}
result, err := h.module.ListContainers(req.Context(), orgID, &parsedReq)
if err != nil {
render.Error(rw, err)
return
}
render.Success(rw, http.StatusOK, result)
}
func (h *handler) ListNodes(rw http.ResponseWriter, req *http.Request) {
claims, err := authtypes.ClaimsFromContext(req.Context())
if err != nil {

View File

@@ -738,3 +738,26 @@ func (m *module) getMetadata(
return result, nil
}
// mergeQueryWarnings combines multiple query warnings into one; nil inputs are
// skipped. The first non-nil warning is the primary; subsequent ones are folded
// into its Warnings list.
func mergeQueryWarnings(warnings ...*qbtypes.QueryWarnData) *qbtypes.QueryWarnData {
var merged *qbtypes.QueryWarnData
for _, w := range warnings {
if w == nil {
continue
}
if merged == nil {
// Copy so we don't mutate the caller's warning.
primary := *w
merged = &primary
continue
}
if w.Message != "" {
merged.Warnings = append(merged.Warnings, qbtypes.QueryWarnDataAdditional{Message: w.Message})
}
merged.Warnings = append(merged.Warnings, w.Warnings...)
}
return merged
}

View File

@@ -78,3 +78,61 @@ func (s checkSpec) getAllAttrs() []string {
}
return out
}
// containerStatusCounts holds per-group container counts bucketed by latest
// kubectl-style display status in window. Mirrors inframonitoringtypes.ContainerCountsByStatus.
type containerStatusCounts struct {
// State fallback.
Running int
Waiting int
Terminated int
// Container-level reasons.
CrashLoopBackOff int
ImagePullBackOff int
ErrImagePull int
CreateContainerConfigError int
ContainerCreating int
OOMKilled int
Completed int
Error int
ContainerCannotRun int
Unknown int
}
// containerStatusCountsToResponse copies the internal per-group status counts
// into the public response struct.
func containerStatusCountsToResponse(c containerStatusCounts) inframonitoringtypes.ContainerCountsByStatus {
return inframonitoringtypes.ContainerCountsByStatus{
Running: c.Running,
Waiting: c.Waiting,
Terminated: c.Terminated,
CrashLoopBackOff: c.CrashLoopBackOff,
ImagePullBackOff: c.ImagePullBackOff,
ErrImagePull: c.ErrImagePull,
CreateContainerConfigError: c.CreateContainerConfigError,
ContainerCreating: c.ContainerCreating,
OOMKilled: c.OOMKilled,
Completed: c.Completed,
Error: c.Error,
ContainerCannotRun: c.ContainerCannotRun,
Unknown: c.Unknown,
}
}
// containerReadyCounts holds per-group container counts bucketed by latest
// readiness in window. Mirrors inframonitoringtypes.ContainerCountsByReady.
type containerReadyCounts struct {
Ready int
NotReady int
}
// containerReadyCountsToResponse copies the internal per-group ready counts
// into the public response struct.
func containerReadyCountsToResponse(c containerReadyCounts) inframonitoringtypes.ContainerCountsByReady {
return inframonitoringtypes.ContainerCountsByReady{
Ready: c.Ready,
NotReady: c.NotReady,
}
}

View File

@@ -17,6 +17,7 @@ import (
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
"github.com/SigNoz/signoz/pkg/valuer"
"golang.org/x/sync/errgroup"
)
type module struct {
@@ -316,6 +317,110 @@ func (m *module) ListPods(ctx context.Context, orgID valuer.UUID, req *inframoni
return resp, nil
}
func (m *module) ListContainers(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableContainers) (*inframonitoringtypes.Containers, error) {
ctx = m.withInfraMonitoringContext(ctx, "ListContainers")
if err := req.Validate(); err != nil {
return nil, err
}
resp := &inframonitoringtypes.Containers{}
if req.OrderBy == nil {
req.OrderBy = &qbtypes.OrderBy{
Key: qbtypes.OrderByKey{
TelemetryFieldKey: telemetrytypes.TelemetryFieldKey{
Name: inframonitoringtypes.ContainersOrderByCPU,
},
},
Direction: qbtypes.OrderDirectionDesc,
}
}
if len(req.GroupBy) == 0 {
req.GroupBy = containerRowGroupBy
resp.Type = inframonitoringtypes.ResponseTypeList
} else {
resp.Type = inframonitoringtypes.ResponseTypeGroupedList
}
minFirstReportedUnixMilli, err := m.getEarliestMetricTime(ctx, containersTableMetricNamesList)
if err != nil {
return nil, err
}
if req.End < int64(minFirstReportedUnixMilli) {
resp.EndTimeBeforeRetention = true
resp.Records = []inframonitoringtypes.ContainerRecord{}
resp.Total = 0
return resp, nil
}
metadataMap, err := m.getContainersTableMetadata(ctx, orgID, req)
if err != nil {
return nil, err
}
resp.Total = len(metadataMap)
pageGroups, err := m.getTopContainerGroups(ctx, orgID, req, metadataMap)
if err != nil {
return nil, err
}
if len(pageGroups) == 0 {
resp.Records = []inframonitoringtypes.ContainerRecord{}
return resp, nil
}
filterExpr := ""
if req.Filter != nil {
filterExpr = req.Filter.Expression
}
fullQueryReq := buildFullQueryRequest(req.Start, req.End, filterExpr, req.GroupBy, pageGroups, m.newContainersTableListQuery())
var (
queryResp *qbtypes.QueryRangeResponse
statusCounts map[string]containerStatusCounts
statusWarning *qbtypes.QueryWarnData
restartCounts map[string]int64
readyCounts map[string]containerReadyCounts
)
g, gCtx := errgroup.WithContext(ctx)
g.Go(func() error {
var err error
queryResp, err = m.querier.QueryRange(gCtx, orgID, fullQueryReq)
return err
})
g.Go(func() error {
var err error
statusCounts, statusWarning, err = m.getPerGroupContainerStatusCountsWithReqMetricChecks(gCtx, req.Start, req.End, req.Filter, req.GroupBy, pageGroups)
return err
})
g.Go(func() error {
var err error
restartCounts, err = m.getPerGroupContainerRestartCounts(gCtx, req.Start, req.End, req.Filter, req.GroupBy, pageGroups)
return err
})
g.Go(func() error {
var err error
readyCounts, err = m.getPerGroupContainerReadyCounts(gCtx, req.Start, req.End, req.Filter, req.GroupBy, pageGroups)
return err
})
if err := g.Wait(); err != nil {
return nil, err
}
isContainerNameAndPodUIDInGroupBy := isKeyInGroupByAttrs(req.GroupBy, containerNameAttrKey) && isKeyInGroupByAttrs(req.GroupBy, podUIDAttrKey)
resp.Records = buildContainerRecords(isContainerNameAndPodUIDInGroupBy, queryResp, pageGroups, req.GroupBy, metadataMap, statusCounts, restartCounts, readyCounts)
resp.Warning = mergeQueryWarnings(queryResp.Warning, statusWarning)
return resp, nil
}
func (m *module) ListNodes(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableNodes) (*inframonitoringtypes.Nodes, error) {
ctx = m.withInfraMonitoringContext(ctx, "ListNodes")

View File

@@ -12,6 +12,7 @@ import (
type Handler interface {
ListHosts(http.ResponseWriter, *http.Request)
ListPods(http.ResponseWriter, *http.Request)
ListContainers(http.ResponseWriter, *http.Request)
ListNodes(http.ResponseWriter, *http.Request)
ListNamespaces(http.ResponseWriter, *http.Request)
ListClusters(http.ResponseWriter, *http.Request)
@@ -27,6 +28,7 @@ type Module interface {
statsreporter.StatsCollector
ListHosts(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableHosts) (*inframonitoringtypes.Hosts, error)
ListPods(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostablePods) (*inframonitoringtypes.Pods, error)
ListContainers(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableContainers) (*inframonitoringtypes.Containers, error)
ListNodes(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableNodes) (*inframonitoringtypes.Nodes, error)
ListNamespaces(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableNamespaces) (*inframonitoringtypes.Namespaces, error)
ListClusters(ctx context.Context, orgID valuer.UUID, req *inframonitoringtypes.PostableClusters) (*inframonitoringtypes.Clusters, error)

View File

@@ -9,17 +9,18 @@ type CheckType struct {
}
var (
CheckTypeHosts = CheckType{valuer.NewString("hosts")}
CheckTypeProcesses = CheckType{valuer.NewString("processes")}
CheckTypePods = CheckType{valuer.NewString("pods")}
CheckTypeNodes = CheckType{valuer.NewString("nodes")}
CheckTypeDeployments = CheckType{valuer.NewString("deployments")}
CheckTypeDaemonsets = CheckType{valuer.NewString("daemonsets")}
CheckTypeStatefulsets = CheckType{valuer.NewString("statefulsets")}
CheckTypeJobs = CheckType{valuer.NewString("jobs")}
CheckTypeNamespaces = CheckType{valuer.NewString("namespaces")}
CheckTypeClusters = CheckType{valuer.NewString("clusters")}
CheckTypeVolumes = CheckType{valuer.NewString("volumes")}
CheckTypeHosts = CheckType{valuer.NewString("hosts")}
CheckTypeProcesses = CheckType{valuer.NewString("processes")}
CheckTypePods = CheckType{valuer.NewString("pods")}
CheckTypeNodes = CheckType{valuer.NewString("nodes")}
CheckTypeDeployments = CheckType{valuer.NewString("deployments")}
CheckTypeDaemonsets = CheckType{valuer.NewString("daemonsets")}
CheckTypeStatefulsets = CheckType{valuer.NewString("statefulsets")}
CheckTypeJobs = CheckType{valuer.NewString("jobs")}
CheckTypeNamespaces = CheckType{valuer.NewString("namespaces")}
CheckTypeClusters = CheckType{valuer.NewString("clusters")}
CheckTypeVolumes = CheckType{valuer.NewString("volumes")}
CheckTypeKubeContainers = CheckType{valuer.NewString("kube_containers")}
)
func (CheckType) Enum() []any {
@@ -35,6 +36,7 @@ func (CheckType) Enum() []any {
CheckTypeNamespaces,
CheckTypeClusters,
CheckTypeVolumes,
CheckTypeKubeContainers,
}
}
@@ -50,6 +52,7 @@ var ValidCheckTypes = []CheckType{
CheckTypeNamespaces,
CheckTypeClusters,
CheckTypeVolumes,
CheckTypeKubeContainers,
}
// CheckComponentType tags each AssociatedComponent as either a receiver or a processor.

View File

@@ -84,6 +84,11 @@ func TestPostableChecks_Validate(t *testing.T) {
req: &PostableChecks{Type: CheckTypeVolumes},
wantErr: false,
},
{
name: "kube_containers",
req: &PostableChecks{Type: CheckTypeKubeContainers},
wantErr: false,
},
}
for _, tt := range tests {

View File

@@ -0,0 +1,148 @@
package inframonitoringtypes
import (
"encoding/json"
"slices"
"github.com/SigNoz/signoz/pkg/errors"
qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
)
type Containers struct {
Type ResponseType `json:"type" required:"true"`
Records []ContainerRecord `json:"records" required:"true" nullable:"false"`
Total int `json:"total" required:"true"`
EndTimeBeforeRetention bool `json:"endTimeBeforeRetention" required:"true"`
Warning *qbtypes.QueryWarnData `json:"warning,omitempty"`
}
// ContainerCountsByStatus buckets container counts by their latest kubectl-style
// display status in the time window (see ContainerStatus). One field per
// derivable status. Populated in both list and grouped_list modes.
type ContainerCountsByStatus struct {
// State fallback.
Running int `json:"running" required:"true"`
Waiting int `json:"waiting" required:"true"`
Terminated int `json:"terminated" required:"true"`
// Container-level reasons (k8s.container.status.reason allowlist).
CrashLoopBackOff int `json:"crashLoopBackOff" required:"true"`
ImagePullBackOff int `json:"imagePullBackOff" required:"true"`
ErrImagePull int `json:"errImagePull" required:"true"`
CreateContainerConfigError int `json:"createContainerConfigError" required:"true"`
ContainerCreating int `json:"containerCreating" required:"true"`
OOMKilled int `json:"oomKilled" required:"true"`
Completed int `json:"completed" required:"true"`
Error int `json:"error" required:"true"`
ContainerCannotRun int `json:"containerCannotRun" required:"true"`
Unknown int `json:"unknown" required:"true"`
}
// ContainerCountsByReady buckets container counts by their latest readiness
// (k8s.container.ready) in the time window. Populated in both modes.
type ContainerCountsByReady struct {
Ready int `json:"ready" required:"true"`
NotReady int `json:"notReady" required:"true"`
}
type ContainerRecord struct {
// Row identity: (k8s.pod.uid, k8s.container.name). Stable across container
// restarts (unlike container.id), globally unique, present on both receivers.
PodUID string `json:"podUID" required:"true"`
ContainerName string `json:"containerName" required:"true"`
// Health (k8sclusterreceiver). Single value in list mode; counts in grouped_list mode.
Status ContainerStatus `json:"status" required:"true"`
ContainerCountsByStatus ContainerCountsByStatus `json:"containerCountsByStatus" required:"true"`
Ready ContainerReady `json:"ready" required:"true"`
ContainerCountsByReady ContainerCountsByReady `json:"containerCountsByReady" required:"true"`
// Absolute restart count: own count (list mode) or group sum (grouped mode). -1 when no data.
Restarts int64 `json:"restarts" required:"true"`
// Usage / utilization (kubeletstats). -1 when no data.
CPU float64 `json:"cpu" required:"true"` // container.cpu.usage (cores)
CPURequestUtilization float64 `json:"cpuRequestUtilization" required:"true"` // k8s.container.cpu_request_utilization
CPULimitUtilization float64 `json:"cpuLimitUtilization" required:"true"` // k8s.container.cpu_limit_utilization
Memory float64 `json:"memory" required:"true"` // container.memory.working_set
MemoryRequestUtilization float64 `json:"memoryRequestUtilization" required:"true"` // k8s.container.memory_request_utilization
MemoryLimitUtilization float64 `json:"memoryLimitUtilization" required:"true"` // k8s.container.memory_limit_utilization
Meta map[string]string `json:"meta" required:"true"`
}
// PostableContainers is the request body for the v2 containers list API.
type PostableContainers struct {
Start int64 `json:"start" required:"true"`
End int64 `json:"end" required:"true"`
Filter *qbtypes.Filter `json:"filter"`
GroupBy []qbtypes.GroupByKey `json:"groupBy"`
OrderBy *qbtypes.OrderBy `json:"orderBy"`
Offset int `json:"offset"`
Limit int `json:"limit" required:"true"`
}
// Validate ensures PostableContainers contains acceptable values.
func (req *PostableContainers) Validate() error {
if req == nil {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "request is nil")
}
if req.Start <= 0 {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
"invalid start time %d: start must be greater than 0",
req.Start,
)
}
if req.End <= 0 {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
"invalid end time %d: end must be greater than 0",
req.End,
)
}
if req.Start >= req.End {
return errors.NewInvalidInputf(
errors.CodeInvalidInput,
"invalid time range: start (%d) must be less than end (%d)",
req.Start,
req.End,
)
}
if req.Limit < 1 || req.Limit > 5000 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "limit must be between 1 and 5000")
}
if req.Offset < 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "offset cannot be negative")
}
if req.OrderBy != nil {
if !slices.Contains(ContainersValidOrderByKeys, req.OrderBy.Key.Name) {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order by key: %s", req.OrderBy.Key.Name)
}
if req.OrderBy.Direction != qbtypes.OrderDirectionAsc && req.OrderBy.Direction != qbtypes.OrderDirectionDesc {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid order by direction: %s", req.OrderBy.Direction)
}
if req.OrderBy.Key.Name == ContainerNameAttrKey && len(req.GroupBy) > 0 {
return errors.NewInvalidInputf(errors.CodeInvalidInput, "order by '%s' is only allowed when groupBy is empty", ContainerNameAttrKey)
}
}
return nil
}
// UnmarshalJSON validates input immediately after decoding.
func (req *PostableContainers) UnmarshalJSON(data []byte) error {
type raw PostableContainers
var decoded raw
if err := json.Unmarshal(data, &decoded); err != nil {
return err
}
*req = PostableContainers(decoded)
return req.Validate()
}

View File

@@ -0,0 +1,92 @@
package inframonitoringtypes
import "github.com/SigNoz/signoz/pkg/valuer"
// ContainerStatus is the kubectl-style display status of a container, derived
// from k8s.container.status.state (base) + k8s.container.status.reason (overlay).
type ContainerStatus struct {
valuer.String
}
var (
// State fallback (from k8s.container.status.state).
ContainerStatusRunning = ContainerStatus{valuer.NewString("Running")}
ContainerStatusWaiting = ContainerStatus{valuer.NewString("Waiting")}
ContainerStatusTerminated = ContainerStatus{valuer.NewString("Terminated")}
// Reasons (from k8s.container.status.reason allowlist).
ContainerStatusCrashLoopBackOff = ContainerStatus{valuer.NewString("CrashLoopBackOff")}
ContainerStatusImagePullBackOff = ContainerStatus{valuer.NewString("ImagePullBackOff")}
ContainerStatusErrImagePull = ContainerStatus{valuer.NewString("ErrImagePull")}
ContainerStatusCreateContainerConfigError = ContainerStatus{valuer.NewString("CreateContainerConfigError")}
ContainerStatusContainerCreating = ContainerStatus{valuer.NewString("ContainerCreating")}
ContainerStatusOOMKilled = ContainerStatus{valuer.NewString("OOMKilled")}
ContainerStatusCompleted = ContainerStatus{valuer.NewString("Completed")}
ContainerStatusError = ContainerStatus{valuer.NewString("Error")}
ContainerStatusContainerCannotRun = ContainerStatus{valuer.NewString("ContainerCannotRun")}
ContainerStatusUnknown = ContainerStatus{valuer.NewString("Unknown")}
// ContainerStatusNoData is the record default: no status data / metrics disabled.
ContainerStatusNoData = ContainerStatus{valuer.NewString("no_data")}
)
func (ContainerStatus) Enum() []any {
return []any{
ContainerStatusRunning,
ContainerStatusWaiting,
ContainerStatusTerminated,
ContainerStatusCrashLoopBackOff,
ContainerStatusImagePullBackOff,
ContainerStatusErrImagePull,
ContainerStatusCreateContainerConfigError,
ContainerStatusContainerCreating,
ContainerStatusOOMKilled,
ContainerStatusCompleted,
ContainerStatusError,
ContainerStatusContainerCannotRun,
ContainerStatusUnknown,
ContainerStatusNoData,
}
}
// ContainerReady is the latest readiness of a container (k8s.container.ready).
type ContainerReady struct {
valuer.String
}
var (
ContainerReadyReady = ContainerReady{valuer.NewString("ready")}
ContainerReadyNotReady = ContainerReady{valuer.NewString("not_ready")}
// ContainerReadyNoData is the record default: no readiness data.
ContainerReadyNoData = ContainerReady{valuer.NewString("no_data")}
)
func (ContainerReady) Enum() []any {
return []any{
ContainerReadyReady,
ContainerReadyNotReady,
ContainerReadyNoData,
}
}
const ContainerNameAttrKey = "k8s.container.name"
const (
ContainersOrderByCPU = "cpu"
ContainersOrderByCPURequest = "cpu_request"
ContainersOrderByCPULimit = "cpu_limit"
ContainersOrderByMemory = "memory"
ContainersOrderByMemoryRequest = "memory_request"
ContainersOrderByMemoryLimit = "memory_limit"
)
var ContainersValidOrderByKeys = []string{
ContainersOrderByCPU,
ContainersOrderByCPURequest,
ContainersOrderByCPULimit,
ContainersOrderByMemory,
ContainersOrderByMemoryRequest,
ContainersOrderByMemoryLimit,
ContainerNameAttrKey,
}

View File

@@ -0,0 +1,28 @@
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "crun-uid", "k8s.container.name": "app", "k8s.pod.name": "crun", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.3, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "crun-uid", "k8s.container.name": "app", "k8s.pod.name": "crun", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 3000000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "crun-uid", "k8s.container.name": "app", "k8s.pod.name": "crun", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a", "k8s.container.status.state": "running"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "crun-uid", "k8s.container.name": "app", "k8s.pod.name": "crun", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "crun-uid", "k8s.container.name": "app", "k8s.pod.name": "crun", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "cnr-uid", "k8s.container.name": "app", "k8s.pod.name": "cnr", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.2, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "cnr-uid", "k8s.container.name": "app", "k8s.pod.name": "cnr", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 2000000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "cnr-uid", "k8s.container.name": "app", "k8s.pod.name": "cnr", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a", "k8s.container.status.state": "running"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "cnr-uid", "k8s.container.name": "app", "k8s.pod.name": "cnr", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "cnr-uid", "k8s.container.name": "app", "k8s.pod.name": "cnr", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "cclo-uid", "k8s.container.name": "app", "k8s.pod.name": "cclo", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "cclo-uid", "k8s.container.name": "app", "k8s.pod.name": "cclo", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1000000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "cclo-uid", "k8s.container.name": "app", "k8s.pod.name": "cclo", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a", "k8s.container.status.state": "waiting"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "cclo-uid", "k8s.container.name": "app", "k8s.pod.name": "cclo", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a", "k8s.container.status.reason": "CrashLoopBackOff"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "cclo-uid", "k8s.container.name": "app", "k8s.pod.name": "cclo", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "cclo-uid", "k8s.container.name": "app", "k8s.pod.name": "cclo", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 5, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "cimg-uid", "k8s.container.name": "app", "k8s.pod.name": "cimg", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a", "k8s.container.status.state": "waiting"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "cimg-uid", "k8s.container.name": "app", "k8s.pod.name": "cimg", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a", "k8s.container.status.reason": "ImagePullBackOff"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "cimg-uid", "k8s.container.name": "app", "k8s.pod.name": "cimg", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "cimg-uid", "k8s.container.name": "app", "k8s.pod.name": "cimg", "k8s.namespace.name": "ns-a", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-a"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "ccomp-uid", "k8s.container.name": "app", "k8s.pod.name": "ccomp", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b", "k8s.container.status.state": "terminated"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "ccomp-uid", "k8s.container.name": "app", "k8s.pod.name": "ccomp", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b", "k8s.container.status.reason": "Completed"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "ccomp-uid", "k8s.container.name": "app", "k8s.pod.name": "ccomp", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "ccomp-uid", "k8s.container.name": "app", "k8s.pod.name": "ccomp", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "coom-uid", "k8s.container.name": "app", "k8s.pod.name": "coom", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b", "k8s.container.status.state": "terminated"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "coom-uid", "k8s.container.name": "app", "k8s.pod.name": "coom", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b", "k8s.container.status.reason": "OOMKilled"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "coom-uid", "k8s.container.name": "app", "k8s.pod.name": "coom", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "coom-uid", "k8s.container.name": "app", "k8s.pod.name": "coom", "k8s.namespace.name": "ns-b", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-b"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 3, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}

View File

@@ -0,0 +1,4 @@
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "cwarn-uid", "k8s.container.name": "app", "k8s.pod.name": "cwarn", "k8s.namespace.name": "ns-warn", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-warn"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.15, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "cwarn-uid", "k8s.container.name": "app", "k8s.pod.name": "cwarn", "k8s.namespace.name": "ns-warn", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-warn"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1500000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "cwarn-uid", "k8s.container.name": "app", "k8s.pod.name": "cwarn", "k8s.namespace.name": "ns-warn", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-warn"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "cwarn-uid", "k8s.container.name": "app", "k8s.pod.name": "cwarn", "k8s.namespace.name": "ns-warn", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-warn"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 2, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}

View File

@@ -0,0 +1,16 @@
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.state": "waiting"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.state": "waiting"}, "timestamp": "2025-01-10T10:01:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.state": "running"}, "timestamp": "2025-01-10T10:02:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.state": "running"}, "timestamp": "2025-01-10T10:03:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.reason": "CrashLoopBackOff"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.reason": "CrashLoopBackOff"}, "timestamp": "2025-01-10T10:01:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.reason": "CrashLoopBackOff"}, "timestamp": "2025-01-10T10:02:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec", "k8s.container.status.reason": "CrashLoopBackOff"}, "timestamp": "2025-01-10T10:03:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:01:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:02:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:03:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 5, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:01:00+00:00", "value": 5, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:02:00+00:00", "value": 5, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "crec-uid", "k8s.container.name": "app", "k8s.pod.name": "crec", "k8s.namespace.name": "ns-rec", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-rec"}, "timestamp": "2025-01-10T10:03:00+00:00", "value": 5, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}

View File

@@ -0,0 +1,12 @@
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "cterm-uid", "k8s.container.name": "app", "k8s.pod.name": "cterm", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.05, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "cterm-uid", "k8s.container.name": "app", "k8s.pod.name": "cterm", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 500000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "cterm-uid", "k8s.container.name": "app", "k8s.pod.name": "cterm", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb", "k8s.container.status.state": "terminated"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "cterm-uid", "k8s.container.name": "app", "k8s.pod.name": "cterm", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb", "k8s.container.status.reason": "Completed"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "cterm-uid", "k8s.container.name": "app", "k8s.pod.name": "cterm", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "cterm-uid", "k8s.container.name": "app", "k8s.pod.name": "cterm", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "cwait-uid", "k8s.container.name": "app", "k8s.pod.name": "cwait", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.05, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "cwait-uid", "k8s.container.name": "app", "k8s.pod.name": "cwait", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 500000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.state", "labels": {"k8s.pod.uid": "cwait-uid", "k8s.container.name": "app", "k8s.pod.name": "cwait", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb", "k8s.container.status.state": "waiting"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 1, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.status.reason", "labels": {"k8s.pod.uid": "cwait-uid", "k8s.container.name": "app", "k8s.pod.name": "cwait", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb", "k8s.container.status.reason": "ContainerCreating"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.ready", "labels": {"k8s.pod.uid": "cwait-uid", "k8s.container.name": "app", "k8s.pod.name": "cwait", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.restarts", "labels": {"k8s.pod.uid": "cwait-uid", "k8s.container.name": "app", "k8s.pod.name": "cwait", "k8s.namespace.name": "ns-fb", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-fb"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}

View File

@@ -0,0 +1,12 @@
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "acc1-uid", "k8s.container.name": "app", "k8s.pod.name": "acc1", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.25, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.cpu_request_utilization", "labels": {"k8s.pod.uid": "acc1-uid", "k8s.container.name": "app", "k8s.pod.name": "acc1", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.5, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.cpu_limit_utilization", "labels": {"k8s.pod.uid": "acc1-uid", "k8s.container.name": "app", "k8s.pod.name": "acc1", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.3, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "acc1-uid", "k8s.container.name": "app", "k8s.pod.name": "acc1", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 2000000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.memory_request_utilization", "labels": {"k8s.pod.uid": "acc1-uid", "k8s.container.name": "app", "k8s.pod.name": "acc1", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.6, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.memory_limit_utilization", "labels": {"k8s.pod.uid": "acc1-uid", "k8s.container.name": "app", "k8s.pod.name": "acc1", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.4, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.cpu.usage", "labels": {"k8s.pod.uid": "acc2-uid", "k8s.container.name": "app", "k8s.pod.name": "acc2", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.75, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.cpu_request_utilization", "labels": {"k8s.pod.uid": "acc2-uid", "k8s.container.name": "app", "k8s.pod.name": "acc2", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.9, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.cpu_limit_utilization", "labels": {"k8s.pod.uid": "acc2-uid", "k8s.container.name": "app", "k8s.pod.name": "acc2", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.7, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "container.memory.working_set", "labels": {"k8s.pod.uid": "acc2-uid", "k8s.container.name": "app", "k8s.pod.name": "acc2", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 5000000, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.memory_request_utilization", "labels": {"k8s.pod.uid": "acc2-uid", "k8s.container.name": "app", "k8s.pod.name": "acc2", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.8, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}
{"metric_name": "k8s.container.memory_limit_utilization", "labels": {"k8s.pod.uid": "acc2-uid", "k8s.container.name": "app", "k8s.pod.name": "acc2", "k8s.namespace.name": "ns-acc", "k8s.node.name": "node-a", "k8s.cluster.name": "cluster-x", "container.image.name": "nginx", "container.image.tag": "1.27", "k8s.deployment.name": "dep-acc"}, "timestamp": "2025-01-10T10:00:00+00:00", "value": 0.5, "temporality": "Unspecified", "type_": "Gauge", "is_monotonic": false}

View File

@@ -0,0 +1,22 @@
{
"records": [
{
"podName": "acc1",
"cpu": 0.25,
"cpuRequestUtilization": 0.5,
"cpuLimitUtilization": 0.3,
"memory": 2000000,
"memoryRequestUtilization": 0.6,
"memoryLimitUtilization": 0.4
},
{
"podName": "acc2",
"cpu": 0.75,
"cpuRequestUtilization": 0.9,
"cpuLimitUtilization": 0.7,
"memory": 5000000,
"memoryRequestUtilization": 0.8,
"memoryLimitUtilization": 0.5
}
]
}

View File

@@ -105,6 +105,14 @@ SPECS = {
"optional": {},
"attrs": {KAP: ["k8s.persistentvolumeclaim.name", "k8s.namespace.name"], RDP: ["k8s.cluster.name"]},
},
"kube_containers": {
"default": {KSR: ["container.cpu.usage", "container.memory.working_set"], KCR: ["k8s.container.restarts", "k8s.container.ready"]},
"optional": {
KSR: ["k8s.container.cpu_request_utilization", "k8s.container.cpu_limit_utilization", "k8s.container.memory_request_utilization", "k8s.container.memory_limit_utilization"],
KCR: ["k8s.container.status.state", "k8s.container.status.reason"],
},
"attrs": {KAP: ["k8s.pod.uid", "k8s.container.name"]},
},
}
ALL_TYPES = list(SPECS.keys())

View File

@@ -0,0 +1,426 @@
"""Integration tests for the v2 infra-monitoring kube_containers endpoint."""
import json
from datetime import UTC, datetime, timedelta
from http import HTTPStatus
import pytest
import requests
from fixtures import types
from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD
from fixtures.fs import get_testdata_file_path
from fixtures.metrics import Metrics
from fixtures.querier import compare_values, get_all_warnings
ENDPOINT = "/api/v2/infra_monitoring/kube_containers"
def _post(signoz: types.SigNoz, token: str, body: dict):
return requests.post(
signoz.self.host_configs["8080"].get(ENDPOINT),
headers={"authorization": f"Bearer {token}"},
json=body,
timeout=5,
)
def test_kube_containers_accuracy(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""Exact per-container usage/utilization values against precomputed expected
output. (Status metrics are intentionally absent here, so status is no_data
and a status warning is surfaced — this test asserts only the numeric fields.)"""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_value_accuracy.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
with open(
get_testdata_file_path("inframonitoring/kube_containers_value_accuracy_expected.json"),
encoding="utf-8",
) as f:
expected = json.load(f)
exp_by_name = {r["podName"]: r for r in expected["records"]}
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"filter": {"expression": "k8s.namespace.name = 'ns-acc'"},
"limit": 50,
},
)
assert response.status_code == HTTPStatus.OK, response.text
data = response.json()["data"]
assert data["total"] == len(expected["records"])
assert {r["meta"]["k8s.pod.name"] for r in data["records"]} == set(exp_by_name.keys())
for record in data["records"]:
exp = exp_by_name[record["meta"]["k8s.pod.name"]]
for field in (
"cpu",
"cpuRequestUtilization",
"cpuLimitUtilization",
"memory",
"memoryRequestUtilization",
"memoryLimitUtilization",
):
assert compare_values(record[field], exp[field], 1e-9), f"{record['meta']['k8s.pod.name']}.{field}: got {record[field]}, expected {exp[field]}"
def test_kube_containers_status_health_and_base_set(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""List mode: per-container status / ready / restarts derived from the
k8s_cluster health metrics, AND the base-set union — containers with only
health metrics and NO kubeletstats usage (ImagePullBackOff / Completed /
OOMKilled) still appear, with cpu = -1 sentinel."""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_dataset.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
# pod name -> (status, ready, restarts, has_usage)
expected = {
"crun": ("running", "ready", 0, True),
"cnr": ("running", "not_ready", 0, True),
"cclo": ("crashloopbackoff", "not_ready", 5, True),
"cimg": ("imagepullbackoff", "not_ready", 0, False),
"ccomp": ("completed", "not_ready", 0, False),
"coom": ("oomkilled", "not_ready", 3, False),
}
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"limit": 50,
},
)
assert response.status_code == HTTPStatus.OK, response.text
data = response.json()["data"]
by_pod = {r["meta"]["k8s.pod.name"]: r for r in data["records"]}
assert set(expected).issubset(set(by_pod)), f"missing containers: {set(expected) - set(by_pod)}"
for pod, (status, ready, restarts, has_usage) in expected.items():
rec = by_pod[pod]
assert rec["status"] == status, f"{pod}.status: got {rec['status']}, expected {status}"
assert rec["ready"] == ready, f"{pod}.ready: got {rec['ready']}, expected {ready}"
assert rec["restarts"] == restarts, f"{pod}.restarts: got {rec['restarts']}, expected {restarts}"
if has_usage:
assert rec["cpu"] != -1, f"{pod}: expected cpu populated, got {rec['cpu']}"
else:
assert rec["cpu"] == -1, f"{pod}: expected cpu -1 sentinel (base-set-only), got {rec['cpu']}"
def test_kube_containers_status_counts_grouped_mode(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""Grouped mode (groupBy namespace): per-group counts of distinct containers
by status and readiness."""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_dataset.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"groupBy": [{"name": "k8s.namespace.name", "fieldDataType": "string", "fieldContext": "resource"}],
"limit": 50,
},
)
assert response.status_code == HTTPStatus.OK, response.text
data = response.json()["data"]
assert data["type"] == "grouped_list"
by_ns = {r["meta"]["k8s.namespace.name"]: r for r in data["records"]}
assert {"ns-a", "ns-b"}.issubset(set(by_ns))
ns_a = by_ns["ns-a"]["containerCountsByStatus"]
assert ns_a["running"] == 2
assert ns_a["crashLoopBackOff"] == 1
assert ns_a["imagePullBackOff"] == 1
assert by_ns["ns-a"]["containerCountsByReady"] == {"ready": 1, "notReady": 3}
ns_b = by_ns["ns-b"]["containerCountsByStatus"]
assert ns_b["completed"] == 1
assert ns_b["oomKilled"] == 1
assert by_ns["ns-b"]["containerCountsByReady"] == {"ready": 0, "notReady": 2}
def test_kube_containers_status_recency(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""Status derivation is by recency, not priority. A container that
CrashLoopBackOff'd earlier in the window (restarts=5) but is Running now
(state flipped to running, reason series flipped to 0) must report the
CURRENT status 'running' — while restarts=5 preserves the crash history.
A priority/argMax-on-reason bug would wrongly surface 'crashloopbackoff'."""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_recency.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"limit": 50,
},
)
assert response.status_code == HTTPStatus.OK, response.text
data = response.json()["data"]
by_pod = {r["meta"]["k8s.pod.name"]: r for r in data["records"]}
assert "crec" in by_pod, f"recency container missing: {list(by_pod)}"
rec = by_pod["crec"]
assert rec["status"] == "running", f"recency: expected current status running, got {rec['status']}"
assert rec["ready"] == "ready", f"recency: expected ready, got {rec['ready']}"
assert rec["restarts"] == 5, f"recency: expected restarts=5 (history preserved), got {rec['restarts']}"
def test_kube_containers_status_state_only_fallback(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""When a container has a state but NO active reason, display status falls
through to the state itself: state='terminated' -> 'terminated',
state='waiting' -> 'waiting'. Exercises the state-fallback branches of the
multiIf and the container_reason LEFT-JOIN miss (containers.go:410)."""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_state_fallback.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"limit": 50,
},
)
assert response.status_code == HTTPStatus.OK, response.text
data = response.json()["data"]
by_pod = {r["meta"]["k8s.pod.name"]: r for r in data["records"]}
assert {"cterm", "cwait"}.issubset(set(by_pod)), f"fallback containers missing: {list(by_pod)}"
assert by_pod["cterm"]["status"] == "terminated", f"terminated fallback: got {by_pod['cterm']['status']}"
assert by_pod["cwait"]["status"] == "waiting", f"waiting fallback: got {by_pod['cwait']['status']}"
def test_kube_containers_status_warning_missing_metrics(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""Status metrics (k8s.container.status.state/reason) never ingested: the
container still appears (usage present), status gates to 'no_data' with a
non-blocking warning naming the missing metrics, while the independent
ready / restarts health signals are still computed correctly."""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_missing_status.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"limit": 50,
},
)
assert response.status_code == HTTPStatus.OK, response.text
body = response.json()
data = body["data"]
by_pod = {r["meta"]["k8s.pod.name"]: r for r in data["records"]}
assert "cwarn" in by_pod, f"container missing: {list(by_pod)}"
rec = by_pod["cwarn"]
assert rec["status"] == "no_data", f"expected status no_data (metrics disabled), got {rec['status']}"
assert rec["ready"] == "ready", f"ready should still compute, got {rec['ready']}"
assert rec["restarts"] == 2, f"restarts should still compute, got {rec['restarts']}"
assert rec["cpu"] != -1, f"usage present, cpu should be populated, got {rec['cpu']}"
warnings = get_all_warnings(body)
assert any("status.state" in w["message"] and "status.reason" in w["message"] for w in warnings), f"status warning naming the missing metrics not surfaced: {warnings!r}"
def test_kube_containers_filter(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""Filter expression narrows the result set by resource attribute."""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_dataset.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
def pods_for(expr: str) -> set[str]:
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"filter": {"expression": expr},
"limit": 50,
},
)
assert response.status_code == HTTPStatus.OK, response.text
data = response.json()["data"]
pods = {r["meta"]["k8s.pod.name"] for r in data["records"]}
assert data["total"] == len(pods)
return pods
assert pods_for("k8s.namespace.name = 'ns-a'") == {"crun", "cnr", "cclo", "cimg"}
assert pods_for("k8s.namespace.name = 'ns-b'") == {"ccomp", "coom"}
assert pods_for("k8s.pod.name = 'crun'") == {"crun"}
def test_kube_containers_orderby_and_pagination(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
insert_metrics,
) -> None:
"""orderBy cpu desc ranks usage containers by cpu (crun>cnr>cclo) with the
usage-less base-set (cpu=-1) backfilled last; pagination walks the full set
(total=6) in non-overlapping pages preserving that order."""
now = datetime.now(tz=UTC).replace(microsecond=0)
insert_metrics(
Metrics.load_from_file(
get_testdata_file_path("inframonitoring/kube_containers_dataset.jsonl"),
base_time=now - timedelta(minutes=4),
)
)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
def page(limit: int, offset: int) -> dict:
response = _post(
signoz,
token,
{
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"orderBy": {"key": {"name": "cpu"}, "direction": "desc"},
"limit": limit,
"offset": offset,
},
)
assert response.status_code == HTTPStatus.OK, response.text
return response.json()["data"]
full = page(50, 0)
assert full["total"] == 6
order = [r["meta"]["k8s.pod.name"] for r in full["records"]]
assert order[:3] == ["crun", "cnr", "cclo"], f"cpu-desc ranking wrong: {order}"
assert set(order) == {"crun", "cnr", "cclo", "cimg", "ccomp", "coom"}
# Paginate 2 at a time; pages must be disjoint, cover everything, keep order.
seen: list[str] = []
for off in (0, 2, 4):
p = page(2, off)
assert p["total"] == 6, f"total must stay 6 at offset {off}, got {p['total']}"
pods = [r["meta"]["k8s.pod.name"] for r in p["records"]]
assert len(pods) == 2, f"offset {off}: expected 2 records, got {pods}"
seen.extend(pods)
assert seen == order, f"paginated sequence {seen} != full-page order {order}"
@pytest.mark.parametrize(
("payload_override", "err_substr"),
[
pytest.param({"limit": 0}, "limit must be between", id="limit_zero"),
pytest.param({"limit": 5001}, "limit must be between", id="limit_too_large"),
pytest.param({"offset": -1}, "offset cannot be negative", id="offset_negative"),
pytest.param(
{"orderBy": {"key": {"name": "bogus"}, "direction": "desc"}},
"invalid order by key",
id="orderby_invalid_key",
),
pytest.param(
{"orderBy": {"key": {"name": "cpu"}, "direction": "up"}},
"invalid order by direction",
id="orderby_invalid_direction",
),
pytest.param(
{
"orderBy": {"key": {"name": "k8s.container.name"}, "direction": "asc"},
"groupBy": [{"name": "k8s.namespace.name", "fieldDataType": "string", "fieldContext": "resource"}],
},
"is only allowed when groupBy is empty",
id="orderby_container_name_with_groupby",
),
],
)
def test_kube_containers_validation_errors(
signoz: types.SigNoz,
create_user_admin: None, # pylint: disable=unused-argument
get_token,
payload_override: dict,
err_substr: str,
) -> None:
"""PostableContainers.Validate() rules reject with 400 + descriptive error."""
now = datetime.now(tz=UTC).replace(microsecond=0)
body: dict = {
"start": int((now - timedelta(minutes=5)).timestamp() * 1000),
"end": int(now.timestamp() * 1000),
"limit": 50,
}
body.update(payload_override)
token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)
response = _post(signoz, token, body)
assert response.status_code == HTTPStatus.BAD_REQUEST, response.text
error = response.json()["error"]
assert error["code"] == "invalid_input"
assert err_substr.lower() in error["message"].lower(), f"expected {err_substr!r} in: {error['message']!r}"