diff --git a/.gitignore b/.gitignore
index cde0123..eb35bfe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,2 @@
-
+.venv
dist/
diff --git a/.jsfh-conf.yaml b/.jsfh-conf.yaml
new file mode 100644
index 0000000..39334fd
--- /dev/null
+++ b/.jsfh-conf.yaml
@@ -0,0 +1,8 @@
+---
+template_name: md
+template_md_options:
+ properties_table_columns:
+ - Property
+ - Type
+ - Title/Description
+footer_show_time: false
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a4b0770..482b0fc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,5 +1,10 @@
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
+default_install_hook_types:
+- pre-commit
+- pre-push
+default_stages:
+- pre-commit
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.4.0
@@ -9,22 +14,24 @@ repos:
- id: check-yaml
exclude: ^charts/kube-transition-metrics/templates/
- id: check-added-large-files
-- repo: https://github.com/golangci/golangci-lint
- rev: v1.52.2
- hooks:
- - id: golangci-lint
- name: golangci-lint
- description: Fast linters runner for Go.
- entry: golangci-lint run --timeout=2m --fix --out-format checkstyle
- types: [go]
- language: golang
- pass_filenames: false
- repo: https://github.com/dnephin/pre-commit-golang
rev: v0.5.1
hooks:
- id: go-fmt
- id: golangci-lint
+ args:
+ - --timeout=5m
- id: go-mod-tidy
+- repo: local
+ hooks:
+ - id: json-schema-for-humans
+ name: json-schema-for-humans
+ description: Generate doc/SCHEMA.md with https://github.com/coveooss/json-schema-for-humans
+ entry: bash -c 'python -m venv .venv && source .venv/bin/activate && pip install --quiet json-schema-for-humans && generate-schema-doc --config-file .jsfh-conf.yaml internal/logging/schemas doc/SCHEMA.md'
+ language: system
+ files: internal/logging/schemas
+ types:
+ - json
- repo: local
hooks:
- id: trufflehog
@@ -32,11 +39,13 @@ repos:
description: Detect secrets in your data.
entry: bash -c 'docker run --rm -v "$PWD:/src" ghcr.io/trufflesecurity/trufflehog:latest git --branch=HEAD file:///src --fail --only-verified'
language: system
- stages: ["commit", "push"]
+ stages:
+ - pre-push
- repo: local
hooks:
- id: semgrep
name: semgrep
entry: bash -c 'docker run --rm -v "$PWD:/src" returntocorp/semgrep semgrep --config p/ci --error --skip-unknown-extensions'
language: system
- stages: ["commit", "push"]
+ stages:
+ - pre-push
diff --git a/README.md b/README.md
index 4594d5c..c0912f5 100644
--- a/README.md
+++ b/README.md
@@ -68,8 +68,105 @@ podAnnotations:
## Available metrics
-For a detailed overview of available metrics, see
-[internal/statistics/README.md](internal/statistics/README.md).
+This Pod life-cycle statistics are emitted in JSON format to `stdout`.
+These logs can be integrated with log processing pipelines like ELK or DataDog.
+Distinguish metric logs from debug and informative logs by the presence of a
+top-level `kube_transition_metrics` key.
+
+### Log Structure
+
+All non-metric logs include a top-level `level` key, which can be: `debug`,
+`info`, `warn`, `error`, or `panic`.
+Non-metric logs are sent to `stderr`, whereas life-cycle metrics are sent to
+`stdout`.
+
+### Examples:
+
+A complete pod record:
+
+```json
+{
+ "kube_transition_metrics": {
+ "type": "pod",
+ "kube_namespace": "default",
+ "pod_name": "flat-earth",
+ "pod": {
+ "creation_timestamp": "2024-06-08T11:14:00+02:00",
+ "scheduled_timestamp": "2024-06-08T11:14:00+02:00",
+ "creation_to_scheduled_seconds": 0,
+ "initialized_timestamp": "2024-06-08T11:14:01+02:00",
+ "creation_to_initialized_seconds": 1,
+ "scheduled_to_initialized_seconds": 1,
+ "ready_timestamp": "2024-06-08T11:14:02+02:00",
+ "creation_to_ready_seconds": 2,
+ "initialized_to_ready_seconds": 1
+ }
+ },
+ "time": "2024-06-08T11:14:06+02:00"
+}
+```
+
+A complete non-init container record:
+```json
+{
+ "kube_transition_metrics": {
+ "type": "container",
+ "kube_namespace": "default",
+ "pod_name": "flat-earth",
+ "container": {
+ "name": "consipire",
+ "init_container": false,
+ "initialized_to_running_seconds": 2.785652,
+ "running_timestamp": "2024-06-08T11:14:02+02:00",
+ "started_timestamp": "2024-06-08T11:14:02+02:00",
+ "running_to_started_seconds": 0,
+ "ready_timestamp": "2024-06-08T11:14:02+02:00",
+ "running_to_ready_seconds": 0,
+ "started_to_ready_seconds": 0
+ }
+ },
+ "time": "2024-06-08T11:14:06+02:00"
+}
+```
+
+A complete init container record:
+```json
+{
+ "kube_transition_metrics": {
+ "type": "container",
+ "kube_namespace": "default",
+ "pod_name": "flat-earth",
+ "container": {
+ "name": "subliminal-messaging",
+ "init_container": true,
+ "ready_timestamp": "2024-06-08T11:14:01+02:00"
+ }
+ },
+ "time": "2024-06-08T11:14:06+02:00"
+}
+```
+
+An image pull record:
+```json
+{
+ "kube_transition_metrics": {
+ "type": "image_pull",
+ "image_pull": {
+ "container_name": "conspire",
+ "already_present": true,
+ "started_timestamp": "2024-06-08T11:14:01+02:00",
+ "finished_timestamp": "2024-06-08T11:14:01+02:00",
+ "duration_seconds": 0
+ },
+ "kube_namespace": "default",
+ "pod_name": "flat-earth"
+ },
+ "time": "2024-06-08T11:14:01+02:00",
+ "message": "Container image \"docker.io/library/nginx:latest\" already present on machine"
+}
+```
+
+For a detailed overview of available metrics, see [doc/SCHEMA.md](doc/SCHEMA.md).
## Contributing
diff --git a/cmd/kube-transition-metrics/main.go b/cmd/kube-transition-metrics/main.go
index 91bb816..73f70c7 100644
--- a/cmd/kube-transition-metrics/main.go
+++ b/cmd/kube-transition-metrics/main.go
@@ -6,12 +6,11 @@ import (
_ "net/http/pprof"
"os"
+ "github.com/BackMarket-oss/kube-transition-metrics/internal/logging"
"github.com/BackMarket-oss/kube-transition-metrics/internal/options"
"github.com/BackMarket-oss/kube-transition-metrics/internal/prommetrics"
"github.com/BackMarket-oss/kube-transition-metrics/internal/statistics"
- "github.com/BackMarket-oss/kube-transition-metrics/internal/zerologhttp"
"github.com/prometheus/client_golang/prometheus/promhttp"
- "github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
@@ -61,10 +60,11 @@ func getKubeconfig(options *options.Options) *rest.Config {
}
func main() {
+ logging.Configure()
prommetrics.Register()
options := options.Parse()
- zerolog.SetGlobalLevel(options.LogLevel)
+ logging.SetOptions(options)
config := getKubeconfig(options)
clientset, err := kubernetes.NewForConfig(config)
@@ -80,7 +80,7 @@ func main() {
go podCollector.Run(clientset)
http.Handle("/metrics", promhttp.Handler())
- handler := zerologhttp.NewHandler(http.DefaultServeMux)
+ handler := logging.NewHTTPHandler(http.DefaultServeMux)
// No timeouts can be set, but that's OK for us as this HTTP server will not be
// exposed publicly.
//nolint:gosec
diff --git a/doc/SCHEMA.md b/doc/SCHEMA.md
new file mode 100644
index 0000000..88df72b
--- /dev/null
+++ b/doc/SCHEMA.md
@@ -0,0 +1,552 @@
+# Metric Record
+
+- [1. Property `Metric Record > kube_transition_metrics`](#kube_transition_metrics)
+ - [1.1. Property `Metric Record > kube_transition_metrics > allOf > item 0`](#kube_transition_metrics_allOf_i0)
+ - [1.1.1. The following properties are required](#autogenerated_heading_2)
+ - [1.2. Property `Metric Record > kube_transition_metrics > allOf > item 1`](#kube_transition_metrics_allOf_i1)
+ - [1.2.1. Property `Metric Record > kube_transition_metrics > allOf > item 1 > oneOf > item 0`](#kube_transition_metrics_allOf_i1_oneOf_i0)
+ - [1.2.1.1. The following properties are required](#autogenerated_heading_3)
+ - [1.2.2. Property `Metric Record > kube_transition_metrics > allOf > item 1 > oneOf > item 1`](#kube_transition_metrics_allOf_i1_oneOf_i1)
+ - [1.2.2.1. The following properties are required](#autogenerated_heading_4)
+ - [1.2.3. Property `Metric Record > kube_transition_metrics > allOf > item 1 > oneOf > item 2`](#kube_transition_metrics_allOf_i1_oneOf_i2)
+ - [1.2.3.1. The following properties are required](#autogenerated_heading_5)
+ - [1.3. Property `Metric Record > kube_transition_metrics > type`](#kube_transition_metrics_type)
+ - [1.4. Property `Metric Record > kube_transition_metrics > kube_namespace`](#kube_transition_metrics_kube_namespace)
+ - [1.5. Property `Metric Record > kube_transition_metrics > pod_name`](#kube_transition_metrics_pod_name)
+ - [1.6. Property `Metric Record > kube_transition_metrics > pod`](#kube_transition_metrics_pod)
+ - [1.6.1. Property `Metric Record > kube_transition_metrics > pod > creation_timestamp`](#kube_transition_metrics_pod_creation_timestamp)
+ - [1.6.2. Property `Metric Record > kube_transition_metrics > pod > scheduled_timestamp`](#kube_transition_metrics_pod_scheduled_timestamp)
+ - [1.6.3. Property `Metric Record > kube_transition_metrics > pod > creation_to_scheduled_seconds`](#kube_transition_metrics_pod_creation_to_scheduled_seconds)
+ - [1.6.4. Property `Metric Record > kube_transition_metrics > pod > initialized_timestamp`](#kube_transition_metrics_pod_initialized_timestamp)
+ - [1.6.5. Property `Metric Record > kube_transition_metrics > pod > creation_to_initialized_seconds`](#kube_transition_metrics_pod_creation_to_initialized_seconds)
+ - [1.6.6. Property `Metric Record > kube_transition_metrics > pod > scheduled_to_initialized_seconds`](#kube_transition_metrics_pod_scheduled_to_initialized_seconds)
+ - [1.6.7. Property `Metric Record > kube_transition_metrics > pod > ready_timestamp`](#kube_transition_metrics_pod_ready_timestamp)
+ - [1.6.8. Property `Metric Record > kube_transition_metrics > pod > creation_to_ready_seconds`](#kube_transition_metrics_pod_creation_to_ready_seconds)
+ - [1.6.9. Property `Metric Record > kube_transition_metrics > pod > initialized_to_ready_seconds`](#kube_transition_metrics_pod_initialized_to_ready_seconds)
+ - [1.7. Property `Metric Record > kube_transition_metrics > container`](#kube_transition_metrics_container)
+ - [1.7.1. Property `Metric Record > kube_transition_metrics > container > name`](#kube_transition_metrics_container_name)
+ - [1.7.2. Property `Metric Record > kube_transition_metrics > container > init_container`](#kube_transition_metrics_container_init_container)
+ - [1.7.3. Property `Metric Record > kube_transition_metrics > container > previous_to_running_seconds`](#kube_transition_metrics_container_previous_to_running_seconds)
+ - [1.7.4. Property `Metric Record > kube_transition_metrics > container > initialized_to_running_seconds`](#kube_transition_metrics_container_initialized_to_running_seconds)
+ - [1.7.5. Property `Metric Record > kube_transition_metrics > container > running_timestamp`](#kube_transition_metrics_container_running_timestamp)
+ - [1.7.6. Property `Metric Record > kube_transition_metrics > container > started_timestamp`](#kube_transition_metrics_container_started_timestamp)
+ - [1.7.7. Property `Metric Record > kube_transition_metrics > container > running_to_started_seconds`](#kube_transition_metrics_container_running_to_started_seconds)
+ - [1.7.8. Property `Metric Record > kube_transition_metrics > container > ready_timestamp`](#kube_transition_metrics_container_ready_timestamp)
+ - [1.7.9. Property `Metric Record > kube_transition_metrics > container > running_to_ready_seconds`](#kube_transition_metrics_container_running_to_ready_seconds)
+ - [1.7.10. Property `Metric Record > kube_transition_metrics > container > started_to_ready_seconds`](#kube_transition_metrics_container_started_to_ready_seconds)
+ - [1.8. Property `Metric Record > kube_transition_metrics > image_pull`](#kube_transition_metrics_image_pull)
+ - [1.8.1. Property `Metric Record > kube_transition_metrics > image_pull > container_name`](#kube_transition_metrics_image_pull_container_name)
+ - [1.8.2. Property `Metric Record > kube_transition_metrics > image_pull > already_present`](#kube_transition_metrics_image_pull_already_present)
+ - [1.8.3. Property `Metric Record > kube_transition_metrics > image_pull > started_timestamp`](#kube_transition_metrics_image_pull_started_timestamp)
+ - [1.8.4. Property `Metric Record > kube_transition_metrics > image_pull > finished_timestamp`](#kube_transition_metrics_image_pull_finished_timestamp)
+ - [1.8.5. Property `Metric Record > kube_transition_metrics > image_pull > duration_seconds`](#kube_transition_metrics_image_pull_duration_seconds)
+- [2. Property `Metric Record > time`](#time)
+- [3. Property `Metric Record > message`](#message)
+
+**Title:** Metric Record
+
+| | |
+| ------------------------- | ------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Not allowed]](# "Additional Properties not allowed.") |
+
+**Description:** JSON schema for metric logs emitted by the kube-transition-metrics controller
+
+| Property | Type | Title/Description |
+| ------------------------------------------------------ | ----------- | ----------------- |
+| + [kube_transition_metrics](#kube_transition_metrics ) | Combination | Metrics |
+| + [time](#time ) | string | Metric Timestamp |
+| - [message](#message ) | string | Message |
+
+## 1. Property `Metric Record > kube_transition_metrics`
+
+**Title:** Metrics
+
+| | |
+| ------------------------- | ------------------------------------------------------- |
+| **Type** | `combining` |
+| **Required** | Yes |
+| **Additional properties** | [[Not allowed]](# "Additional Properties not allowed.") |
+
+**Description:** The metrics pertaining to pod_name
+
+| Property | Type | Title/Description |
+| ------------------------------------------------------------ | ---------------- | ------------------------- |
+| - [type](#kube_transition_metrics_type ) | enum (of string) | Metric type |
+| - [kube_namespace](#kube_transition_metrics_kube_namespace ) | string | Kubernetes Namespace name |
+| - [pod_name](#kube_transition_metrics_pod_name ) | string | Kubernetes Pod name |
+| - [pod](#kube_transition_metrics_pod ) | object | Pod Metrics |
+| - [container](#kube_transition_metrics_container ) | object | Container Metrics |
+| - [image_pull](#kube_transition_metrics_image_pull ) | object | Image Pull Metrics |
+
+| All of(Requirement) |
+| ------------------------------------------- |
+| [item 0](#kube_transition_metrics_allOf_i0) |
+| [item 1](#kube_transition_metrics_allOf_i1) |
+
+### 1.1. Property `Metric Record > kube_transition_metrics > allOf > item 0`
+
+| | |
+| ------------------------- | ------------------------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Any type: allowed]](# "Additional Properties of any type are allowed.") |
+
+#### 1.1.1. The following properties are required
+* kube_namespace
+* pod_name
+* type
+
+### 1.2. Property `Metric Record > kube_transition_metrics > allOf > item 1`
+
+| | |
+| ------------------------- | ------------------------------------------------------------------------- |
+| **Type** | `combining` |
+| **Required** | No |
+| **Additional properties** | [[Any type: allowed]](# "Additional Properties of any type are allowed.") |
+
+| One of(Option) |
+| ---------------------------------------------------- |
+| [item 0](#kube_transition_metrics_allOf_i1_oneOf_i0) |
+| [item 1](#kube_transition_metrics_allOf_i1_oneOf_i1) |
+| [item 2](#kube_transition_metrics_allOf_i1_oneOf_i2) |
+
+#### 1.2.1. Property `Metric Record > kube_transition_metrics > allOf > item 1 > oneOf > item 0`
+
+| | |
+| ------------------------- | ------------------------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Any type: allowed]](# "Additional Properties of any type are allowed.") |
+
+##### 1.2.1.1. The following properties are required
+* pod
+
+#### 1.2.2. Property `Metric Record > kube_transition_metrics > allOf > item 1 > oneOf > item 1`
+
+| | |
+| ------------------------- | ------------------------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Any type: allowed]](# "Additional Properties of any type are allowed.") |
+
+##### 1.2.2.1. The following properties are required
+* container
+
+#### 1.2.3. Property `Metric Record > kube_transition_metrics > allOf > item 1 > oneOf > item 2`
+
+| | |
+| ------------------------- | ------------------------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Any type: allowed]](# "Additional Properties of any type are allowed.") |
+
+##### 1.2.3.1. The following properties are required
+* image_pull
+
+### 1.3. Property `Metric Record > kube_transition_metrics > type`
+
+**Title:** Metric type
+
+| | |
+| ------------ | ------------------ |
+| **Type** | `enum (of string)` |
+| **Required** | No |
+
+**Description:** The type of metric included in kube_transition_metrics
+
+Must be one of:
+* "pod"
+* "container"
+* "image_pull"
+
+### 1.4. Property `Metric Record > kube_transition_metrics > kube_namespace`
+
+**Title:** Kubernetes Namespace name
+
+| | |
+| ------------ | -------- |
+| **Type** | `string` |
+| **Required** | No |
+
+**Description:** The name of the Kubernetes Namespace containing the pod
+
+### 1.5. Property `Metric Record > kube_transition_metrics > pod_name`
+
+**Title:** Kubernetes Pod name
+
+| | |
+| ------------ | -------- |
+| **Type** | `string` |
+| **Required** | No |
+
+**Description:** The name of the Kubernetes Pod to which metrics pertain
+
+### 1.6. Property `Metric Record > kube_transition_metrics > pod`
+
+**Title:** Pod Metrics
+
+| | |
+| ------------------------- | ------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Not allowed]](# "Additional Properties not allowed.") |
+
+**Description:** Included if kube_transition_metric_type is equal to "pod".
+
+| Property | Type | Title/Description |
+| ---------------------------------------------------------------------------------------------------- | ------ | ---------------------------- |
+| + [creation_timestamp](#kube_transition_metrics_pod_creation_timestamp ) | string | Running Timestamp |
+| - [scheduled_timestamp](#kube_transition_metrics_pod_scheduled_timestamp ) | string | Scheduled Timestamp |
+| - [creation_to_scheduled_seconds](#kube_transition_metrics_pod_creation_to_scheduled_seconds ) | number | Pod Creation to Scheduled |
+| - [initialized_timestamp](#kube_transition_metrics_pod_initialized_timestamp ) | string | initialized Timestamp |
+| - [creation_to_initialized_seconds](#kube_transition_metrics_pod_creation_to_initialized_seconds ) | number | Pod Creation to Initialized |
+| - [scheduled_to_initialized_seconds](#kube_transition_metrics_pod_scheduled_to_initialized_seconds ) | number | Pod Scheduled to Initialized |
+| - [ready_timestamp](#kube_transition_metrics_pod_ready_timestamp ) | string | Ready Timestamp |
+| - [creation_to_ready_seconds](#kube_transition_metrics_pod_creation_to_ready_seconds ) | number | Pod Creation to Ready |
+| - [initialized_to_ready_seconds](#kube_transition_metrics_pod_initialized_to_ready_seconds ) | number | Pod Initialized to Ready |
+
+#### 1.6.1. Property `Metric Record > kube_transition_metrics > pod > creation_timestamp`
+
+**Title:** Running Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | Yes |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the Pod was created.
+
+#### 1.6.2. Property `Metric Record > kube_transition_metrics > pod > scheduled_timestamp`
+
+**Title:** Scheduled Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | No |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the Pod was scheduled (Pending->Initializing state).
+
+#### 1.6.3. Property `Metric Record > kube_transition_metrics > pod > creation_to_scheduled_seconds`
+
+**Title:** Pod Creation to Scheduled
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds it took to schedule the Pod.
+
+#### 1.6.4. Property `Metric Record > kube_transition_metrics > pod > initialized_timestamp`
+
+**Title:** initialized Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | No |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the Pod first entered Running state (all init containers exited successfuly and images are pulled). In the event of a pod restart this time is not reset.
+
+#### 1.6.5. Property `Metric Record > kube_transition_metrics > pod > creation_to_initialized_seconds`
+
+**Title:** Pod Creation to Initialized
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the pod creation to when it was initialized.
+
+#### 1.6.6. Property `Metric Record > kube_transition_metrics > pod > scheduled_to_initialized_seconds`
+
+**Title:** Pod Scheduled to Initialized
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the pod was scheduled to when it was initialized (Initializing->Running state).
+
+#### 1.6.7. Property `Metric Record > kube_transition_metrics > pod > ready_timestamp`
+
+**Title:** Ready Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | No |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the Pod first became Ready (all containers had readinessProbe success). In the event of a pod restart this time is not reset.
+
+#### 1.6.8. Property `Metric Record > kube_transition_metrics > pod > creation_to_ready_seconds`
+
+**Title:** Pod Creation to Ready
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the pod creation to becoming Ready.
+
+#### 1.6.9. Property `Metric Record > kube_transition_metrics > pod > initialized_to_ready_seconds`
+
+**Title:** Pod Initialized to Ready
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the pod was initialized (Running state) to when it first bacame Ready.
+
+### 1.7. Property `Metric Record > kube_transition_metrics > container`
+
+**Title:** Container Metrics
+
+| | |
+| ------------------------- | ------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Not allowed]](# "Additional Properties not allowed.") |
+
+**Description:** Included if kube_transition_metric_type is equal to "container".
+
+| Property | Type | Title/Description |
+| ------------------------------------------------------------------------------------------------------ | ------- | -------------------------------------- |
+| + [name](#kube_transition_metrics_container_name ) | string | Container name |
+| + [init_container](#kube_transition_metrics_container_init_container ) | boolean | Init Container |
+| - [previous_to_running_seconds](#kube_transition_metrics_container_previous_to_running_seconds ) | number | Previous Container Finished to Running |
+| - [initialized_to_running_seconds](#kube_transition_metrics_container_initialized_to_running_seconds ) | number | Pod Initialized to Running |
+| - [running_timestamp](#kube_transition_metrics_container_running_timestamp ) | string | Running Timestamp |
+| - [started_timestamp](#kube_transition_metrics_container_started_timestamp ) | string | Started Timestamp |
+| - [running_to_started_seconds](#kube_transition_metrics_container_running_to_started_seconds ) | number | Running to Started |
+| - [ready_timestamp](#kube_transition_metrics_container_ready_timestamp ) | string | Started Timestamp |
+| - [running_to_ready_seconds](#kube_transition_metrics_container_running_to_ready_seconds ) | number | Running to Ready |
+| - [started_to_ready_seconds](#kube_transition_metrics_container_started_to_ready_seconds ) | number | Running to Ready |
+
+#### 1.7.1. Property `Metric Record > kube_transition_metrics > container > name`
+
+**Title:** Container name
+
+| | |
+| ------------ | -------- |
+| **Type** | `string` |
+| **Required** | Yes |
+
+**Description:** The name of the container to which metrics pertain
+
+#### 1.7.2. Property `Metric Record > kube_transition_metrics > container > init_container`
+
+**Title:** Init Container
+
+| | |
+| ------------ | --------- |
+| **Type** | `boolean` |
+| **Required** | Yes |
+
+**Description:** True if the container is an init container, otherwise false.
+
+#### 1.7.3. Property `Metric Record > kube_transition_metrics > container > previous_to_running_seconds`
+
+**Title:** Previous Container Finished to Running
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the previous init container becoming Ready (exited 0) to this container running. Only set for init containers, absent for the first init container.
+
+#### 1.7.4. Property `Metric Record > kube_transition_metrics > container > initialized_to_running_seconds`
+
+**Title:** Pod Initialized to Running
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the Pod becoming initialized (all init containers exited 0) to this container running. Only set for non-init containers.
+
+#### 1.7.5. Property `Metric Record > kube_transition_metrics > container > running_timestamp`
+
+**Title:** Running Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | No |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the container first entered Running state (first fork(2)/execve(2) in container environment). In the event of a pod restart, this timestamp is NOT updated.
+
+#### 1.7.6. Property `Metric Record > kube_transition_metrics > container > started_timestamp`
+
+**Title:** Started Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | No |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the container first started state (startupProbe success). In the event of a pod restart, this timestamp is NOT updated. Only set for non-init containers.
+
+#### 1.7.7. Property `Metric Record > kube_transition_metrics > container > running_to_started_seconds`
+
+**Title:** Running to Started
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the container becoming running to this container started. Only set for non-init containers.
+
+#### 1.7.8. Property `Metric Record > kube_transition_metrics > container > ready_timestamp`
+
+**Title:** Started Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | No |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the container first ready state (readinessProbe success). In the event of a pod restart, this timestamp is NOT updated.
+
+#### 1.7.9. Property `Metric Record > kube_transition_metrics > container > running_to_ready_seconds`
+
+**Title:** Running to Ready
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the container becoming running to this container ready. In init containers, this is the time the container exited with a successful status.
+
+#### 1.7.10. Property `Metric Record > kube_transition_metrics > container > started_to_ready_seconds`
+
+**Title:** Running to Ready
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The time in seconds from the container becoming started to this container ready. Only set for non-init containers.
+
+### 1.8. Property `Metric Record > kube_transition_metrics > image_pull`
+
+**Title:** Image Pull Metrics
+
+| | |
+| ------------------------- | ------------------------------------------------------- |
+| **Type** | `object` |
+| **Required** | No |
+| **Additional properties** | [[Not allowed]](# "Additional Properties not allowed.") |
+
+**Description:** Included if kube_transition_metric_type is equal to "image_pull". Note that these metrics are only emitted in the event that an image pull occurs, if imagePullPolicy is set to IfNotPresent this will only occur if the image is not already present on the node.
+
+| Property | Type | Title/Description |
+| ------------------------------------------------------------------------------- | ------- | ------------------ |
+| + [container_name](#kube_transition_metrics_image_pull_container_name ) | string | Container name |
+| - [already_present](#kube_transition_metrics_image_pull_already_present ) | boolean | Already Present |
+| + [started_timestamp](#kube_transition_metrics_image_pull_started_timestamp ) | string | Started Timestamp |
+| - [finished_timestamp](#kube_transition_metrics_image_pull_finished_timestamp ) | string | Finished Timestamp |
+| - [duration_seconds](#kube_transition_metrics_image_pull_duration_seconds ) | number | Duration |
+
+#### 1.8.1. Property `Metric Record > kube_transition_metrics > image_pull > container_name`
+
+**Title:** Container name
+
+| | |
+| ------------ | -------- |
+| **Type** | `string` |
+| **Required** | Yes |
+
+**Description:** The name of the container which initiated the image pull
+
+#### 1.8.2. Property `Metric Record > kube_transition_metrics > image_pull > already_present`
+
+**Title:** Already Present
+
+| | |
+| ------------ | --------- |
+| **Type** | `boolean` |
+| **Required** | No |
+
+**Description:** true if the image was already present on the machine, otherwise false.
+
+#### 1.8.3. Property `Metric Record > kube_transition_metrics > image_pull > started_timestamp`
+
+**Title:** Started Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | Yes |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the image pull was first initiated. This is obtained from the Event emitted by the Kubelet and may not be 100% accurate. In the event of ImagePullFailed this time is not reset for subsequent attempts.
+
+#### 1.8.4. Property `Metric Record > kube_transition_metrics > image_pull > finished_timestamp`
+
+**Title:** Finished Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | No |
+| **Format** | `date-time` |
+
+**Description:** The timestamp for when the image pull was finished. This is obtained from the Event emitted by the Kubelet and may not be 100% accurate.
+
+#### 1.8.5. Property `Metric Record > kube_transition_metrics > image_pull > duration_seconds`
+
+**Title:** Duration
+
+| | |
+| ------------ | -------- |
+| **Type** | `number` |
+| **Required** | No |
+
+**Description:** The duration in seconds to complete the image pull successfully. This is based purely off the started_timestamp and finished_timestamp, which themselves are based on Event timestamps which are rounded to seconds. The duration here may not match perfectly the duration seen in the kubelet image pull message, due to slight latency in reporting of image pull Events and truncation of timestamps to seconds.
+
+## 2. Property `Metric Record > time`
+
+**Title:** Metric Timestamp
+
+| | |
+| ------------ | ----------- |
+| **Type** | `string` |
+| **Required** | Yes |
+| **Format** | `date-time` |
+
+**Description:** The time at which this metric was emitted.
+
+## 3. Property `Metric Record > message`
+
+**Title:** Message
+
+| | |
+| ------------ | -------- |
+| **Type** | `string` |
+| **Required** | No |
+
+**Description:** An additional message emitted along with metrics.
+
+----------------------------------------------------------------------------------------------------------------------------
+Generated using [json-schema-for-humans](https://github.com/coveooss/json-schema-for-humans)
diff --git a/go.mod b/go.mod
index f59205e..dbb7ddb 100644
--- a/go.mod
+++ b/go.mod
@@ -5,6 +5,7 @@ go 1.20
require (
github.com/prometheus/client_golang v1.16.0
github.com/rs/zerolog v1.30.0
+ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.4
k8s.io/api v0.28.2
diff --git a/go.sum b/go.sum
index a65e89c..7d5c3a2 100644
--- a/go.sum
+++ b/go.sum
@@ -90,6 +90,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c=
github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4=
+github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
diff --git a/internal/zerologhttp/http.go b/internal/logging/http.go
similarity index 74%
rename from internal/zerologhttp/http.go
rename to internal/logging/http.go
index 4993368..c3c8fc7 100644
--- a/internal/zerologhttp/http.go
+++ b/internal/logging/http.go
@@ -1,4 +1,4 @@
-package zerologhttp
+package logging
import (
"fmt"
@@ -9,18 +9,18 @@ import (
"github.com/rs/zerolog/log"
)
-type responseLogger struct {
+type httpResponseLogger struct {
responseWriter http.ResponseWriter
statusCode int
bodyBytesSent int64
}
-func (rl *responseLogger) WriteHeader(code int) {
+func (rl *httpResponseLogger) WriteHeader(code int) {
rl.statusCode = code
rl.responseWriter.WriteHeader(code)
}
-func (rl *responseLogger) Write(data []byte) (int, error) {
+func (rl *httpResponseLogger) Write(data []byte) (int, error) {
length, err := rl.responseWriter.Write(data)
rl.bodyBytesSent += int64(length)
@@ -31,21 +31,21 @@ func (rl *responseLogger) Write(data []byte) (int, error) {
return length, err
}
-func (rl *responseLogger) Header() http.Header {
+func (rl *httpResponseLogger) Header() http.Header {
return rl.responseWriter.Header()
}
-// Handler is a custom request logger middleware.
-type Handler struct {
+// HTTPHandler is a custom request logger middleware.
+type HTTPHandler struct {
handler http.Handler
}
-// NewHandler creates a new Handler middleware.
-func NewHandler(handler http.Handler) *Handler {
- return &Handler{handler: handler}
+// NewHTTPHandler creates a new Handler middleware.
+func NewHTTPHandler(handler http.Handler) *HTTPHandler {
+ return &HTTPHandler{handler: handler}
}
-func (c *Handler) logger() *zerolog.Logger {
+func (c *HTTPHandler) logger() *zerolog.Logger {
logger := log.With().
Str("subsystem", "http").
Logger()
@@ -53,14 +53,14 @@ func (c *Handler) logger() *zerolog.Logger {
return &logger
}
-func (c *Handler) ServeHTTP(
+func (c *HTTPHandler) ServeHTTP(
writer http.ResponseWriter,
req *http.Request,
) {
startTime := time.Now()
logger := c.logger()
- responseLogger := &responseLogger{responseWriter: writer}
+ responseLogger := &httpResponseLogger{responseWriter: writer}
// Call the next handler in the chain
c.handler.ServeHTTP(responseLogger, req)
diff --git a/internal/logging/logging.go b/internal/logging/logging.go
new file mode 100644
index 0000000..999679f
--- /dev/null
+++ b/internal/logging/logging.go
@@ -0,0 +1,19 @@
+package logging
+
+import (
+ "time"
+
+ "github.com/BackMarket-oss/kube-transition-metrics/internal/options"
+ "github.com/rs/zerolog"
+)
+
+// Configure configures zerolog with the required global settings.
+func Configure() {
+ zerolog.DurationFieldInteger = false
+ zerolog.DurationFieldUnit = time.Second
+}
+
+// SetOptions configures zerolog global settings based on user-configured options.
+func SetOptions(options *options.Options) {
+ zerolog.SetGlobalLevel(options.LogLevel)
+}
diff --git a/internal/logging/schemas/kube_transition_metrics.schema.json b/internal/logging/schemas/kube_transition_metrics.schema.json
new file mode 100644
index 0000000..b1ff085
--- /dev/null
+++ b/internal/logging/schemas/kube_transition_metrics.schema.json
@@ -0,0 +1,212 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "schemas/kube_transition_metrics.schema.json",
+ "title": "Metric Record",
+ "description": "JSON schema for metric logs emitted by the kube-transition-metrics controller",
+ "type": "object",
+ "properties": {
+ "kube_transition_metrics": {
+ "title": "Metrics",
+ "description": "The metrics pertaining to pod_name",
+ "type": "object",
+ "properties": {
+ "type": {
+ "title": "Metric type",
+ "description": "The type of metric included in kube_transition_metrics",
+ "type": "string",
+ "enum": ["pod", "container", "image_pull"]
+ },
+ "kube_namespace": {
+ "title": "Kubernetes Namespace name",
+ "description": "The name of the Kubernetes Namespace containing the pod",
+ "type": "string"
+ },
+ "pod_name": {
+ "title": "Kubernetes Pod name",
+ "description": "The name of the Kubernetes Pod to which metrics pertain",
+ "type": "string"
+ },
+ "pod": {
+ "title": "Pod Metrics",
+ "description": "Included if kube_transition_metric_type is equal to \"pod\".",
+ "type": "object",
+ "properties": {
+ "creation_timestamp": {
+ "title": "Running Timestamp",
+ "description": "The timestamp for when the Pod was created.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "scheduled_timestamp": {
+ "title": "Scheduled Timestamp",
+ "description": "The timestamp for when the Pod was scheduled (Pending->Initializing state).",
+ "type": "string",
+ "format": "date-time"
+ },
+ "creation_to_scheduled_seconds": {
+ "title": "Pod Creation to Scheduled",
+ "description": "The time in seconds it took to schedule the Pod.",
+ "type": "number"
+ },
+ "initialized_timestamp": {
+ "title": "initialized Timestamp",
+ "description": "The timestamp for when the Pod first entered Running state (all init containers exited successfuly and images are pulled). In the event of a pod restart this time is not reset.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "creation_to_initialized_seconds": {
+ "title": "Pod Creation to Initialized",
+ "description": "The time in seconds from the pod creation to when it was initialized.",
+ "type": "number"
+ },
+ "scheduled_to_initialized_seconds": {
+ "title": "Pod Scheduled to Initialized",
+ "description": "The time in seconds from the pod was scheduled to when it was initialized (Initializing->Running state).",
+ "type": "number"
+ },
+ "ready_timestamp": {
+ "title": "Ready Timestamp",
+ "description": "The timestamp for when the Pod first became Ready (all containers had readinessProbe success). In the event of a pod restart this time is not reset.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "creation_to_ready_seconds": {
+ "title": "Pod Creation to Ready",
+ "description": "The time in seconds from the pod creation to becoming Ready.",
+ "type": "number"
+ },
+ "initialized_to_ready_seconds": {
+ "title": "Pod Initialized to Ready",
+ "description": "The time in seconds from the pod was initialized (Running state) to when it first bacame Ready.",
+ "type": "number"
+ }
+ },
+ "additionalProperties": false,
+ "required": ["creation_timestamp"]
+ },
+ "container": {
+ "title": "Container Metrics",
+ "description": "Included if kube_transition_metric_type is equal to \"container\".",
+ "type": "object",
+ "properties": {
+ "name": {
+ "title": "Container name",
+ "description": "The name of the container to which metrics pertain",
+ "type": "string"
+ },
+ "init_container": {
+ "title": "Init Container",
+ "description": "True if the container is an init container, otherwise false.",
+ "type": "boolean"
+ },
+ "previous_to_running_seconds": {
+ "title": "Previous Container Finished to Running",
+ "description": "The time in seconds from the previous init container becoming Ready (exited 0) to this container running. Only set for init containers, absent for the first init container.",
+ "type": "number"
+ },
+ "initialized_to_running_seconds": {
+ "title": "Pod Initialized to Running",
+ "description": "The time in seconds from the Pod becoming initialized (all init containers exited 0) to this container running. Only set for non-init containers.",
+ "type": "number"
+ },
+ "running_timestamp": {
+ "title": "Running Timestamp",
+ "description": "The timestamp for when the container first entered Running state (first fork(2)/execve(2) in container environment). In the event of a pod restart, this timestamp is NOT updated.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "started_timestamp": {
+ "title": "Started Timestamp",
+ "description": "The timestamp for when the container first started state (startupProbe success). In the event of a pod restart, this timestamp is NOT updated. Only set for non-init containers.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "running_to_started_seconds": {
+ "title": "Running to Started",
+ "description": "The time in seconds from the container becoming running to this container started. Only set for non-init containers.",
+ "type": "number"
+ },
+ "ready_timestamp": {
+ "title": "Started Timestamp",
+ "description": "The timestamp for when the container first ready state (readinessProbe success). In the event of a pod restart, this timestamp is NOT updated.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "running_to_ready_seconds": {
+ "title": "Running to Ready",
+ "description": "The time in seconds from the container becoming running to this container ready. In init containers, this is the time the container exited with a successful status.",
+ "type": "number"
+ },
+ "started_to_ready_seconds": {
+ "title": "Running to Ready",
+ "description": "The time in seconds from the container becoming started to this container ready. Only set for non-init containers.",
+ "type": "number"
+ }
+ },
+ "additionalProperties": false,
+ "required": ["name", "init_container"]
+ },
+ "image_pull": {
+ "title": "Image Pull Metrics",
+ "description": "Included if kube_transition_metric_type is equal to \"image_pull\". Note that these metrics are only emitted in the event that an image pull occurs, if imagePullPolicy is set to IfNotPresent this will only occur if the image is not already present on the node.",
+ "type": "object",
+ "properties": {
+ "container_name": {
+ "title": "Container name",
+ "description": "The name of the container which initiated the image pull",
+ "type": "string"
+ },
+ "already_present": {
+ "title": "Already Present",
+ "description": "true if the image was already present on the machine, otherwise false.",
+ "type": "boolean"
+ },
+ "started_timestamp": {
+ "title": "Started Timestamp",
+ "description": "The timestamp for when the image pull was first initiated. This is obtained from the Event emitted by the Kubelet and may not be 100% accurate. In the event of ImagePullFailed this time is not reset for subsequent attempts.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "finished_timestamp": {
+ "title": "Finished Timestamp",
+ "description": "The timestamp for when the image pull was finished. This is obtained from the Event emitted by the Kubelet and may not be 100% accurate.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "duration_seconds": {
+ "title": "Duration",
+ "description": "The duration in seconds to complete the image pull successfully. This is based purely off the started_timestamp and finished_timestamp, which themselves are based on Event timestamps which are rounded to seconds. The duration here may not match perfectly the duration seen in the kubelet image pull message, due to slight latency in reporting of image pull Events and truncation of timestamps to seconds.",
+ "type": "number"
+ }
+ },
+ "additionalProperties": false,
+ "required": ["container_name", "started_timestamp"]
+ }
+ },
+ "additionalProperties": false,
+ "allOf": [
+ { "required": ["kube_namespace", "pod_name", "type"] },
+ {
+ "oneOf": [
+ { "required": ["pod"] },
+ { "required": ["container"] },
+ { "required": ["image_pull"] }
+ ]
+ }
+ ]
+ },
+ "time": {
+ "title": "Metric Timestamp",
+ "description": "The time at which this metric was emitted.",
+ "type": "string",
+ "format": "date-time"
+ },
+ "message": {
+ "title": "Message",
+ "description": "An additional message emitted along with metrics.",
+ "type": "string"
+ }
+ },
+ "additionalProperties": false,
+ "required": ["kube_transition_metrics", "time"]
+}
diff --git a/internal/logging/schemavalidation.go b/internal/logging/schemavalidation.go
new file mode 100644
index 0000000..e2807a4
--- /dev/null
+++ b/internal/logging/schemavalidation.go
@@ -0,0 +1,70 @@
+package logging
+
+import (
+ "bytes"
+ "embed"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/fs"
+ "path/filepath"
+
+ "github.com/rs/zerolog/log"
+ jsonschema "github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+//go:embed schemas
+var schemaFiles embed.FS
+
+// NewValidationWriter compiles the JSONSchema a producers a Writer which validates the provided JSON documents against
+// the schema.
+// It discards the data after validation.
+func NewValidationWriter() io.Writer {
+ compiler := jsonschema.NewCompiler()
+ //nolint: wrapcheck
+ err := fs.WalkDir(schemaFiles, ".", func(path string, dirEntry fs.DirEntry, err error) error {
+ if err != nil {
+ log.Panic().Str("path", path).Err(err).Msg("Failed to walk schemas")
+ }
+
+ if !dirEntry.Type().IsRegular() || filepath.Ext(dirEntry.Name()) != ".json" {
+ return nil
+ }
+
+ reader, err := schemaFiles.Open(path)
+ if err != nil {
+ return err
+ }
+
+ return compiler.AddResource(filepath.Clean(path), reader)
+ })
+ if err != nil {
+ log.Panic().Err(err).Msg("Failed to add all jsonschema resources")
+ }
+
+ schema, err := compiler.Compile("schemas/kube_transition_metrics.schema.json")
+ if err != nil {
+ log.Panic().Err(err).Msg("Failed to compile jsonschema")
+ }
+
+ return &validationWriter{schema: schema}
+}
+
+type validationWriter struct {
+ schema *jsonschema.Schema
+}
+
+func (w *validationWriter) Write(data []byte) (int, error) {
+ decoder := json.NewDecoder(bytes.NewReader(data))
+ decoder.UseNumber()
+
+ var err error
+ var document interface{}
+ if err = decoder.Decode(&document); err != nil {
+ err = fmt.Errorf("failed to decode document for schema validation: %w", err)
+ } else if err = w.schema.Validate(document); err != nil {
+ err = fmt.Errorf("document is not validated by schema: %w", err)
+ }
+
+ return len(data), err
+}
diff --git a/internal/statistics/README.md b/internal/statistics/README.md
deleted file mode 100644
index ceae4d5..0000000
--- a/internal/statistics/README.md
+++ /dev/null
@@ -1,141 +0,0 @@
-# Statistics generation
-
-## Overview
-
-This module handles the collection, processing, and exportation of Pod
-life-cycle statistics in JSON format to `stdout`.
-These logs can be integrated with log processing pipelines like ELK or DataDog.
-Distinguish metric logs from debug and informative logs by the presence of a
-top-level `kube_transition_metric_type`.
-
-## Log Structure
-
-All non-metric logs include a top-level `level` key, which can be: `debug`,
-`info`, `warn`, `error`, or `panic`.
-Non-metric logs are sent to `stderr`, whereas life-cycle metrics are sent to
-`stdout`.
-
-Logs with the `kube_transition_metric_type` key contain metrics about the pod
-life-cycle.
-The key `kube_transition_metrics` contains a dictionary of metrics within these
-logs.
-
-There are three `kube_transition_metric_type`s:
-- `pod`: Metrics about pod life-cycle.
-- `container`: Metrics about container life-cycle.
-- `image_pull`: Metrics related to Docker/OCI image pulls.
-
-All metric logs have the `kube_namespace` and `pod_name` keys at the top level,
-which represent the Kubernetes namespace and Pod name associated with the
-metric.
-The `container` and `image_pull` metric logs also include the `container_name`
-key.
-
-Metrics are set once and will not be available until the information is
-recorded.
-
-The available metrics for each type are as follows:
-
-### `pod` metrics
-
-* `scheduled_latency`: a floating-point value representing the duration in
- seconds between the Pod creation and the `PodScheduled` Condition (when a pod
- is assigned to a node).
-* `initialized_latency`: a floating-point value representing the duration in
- seconds between the Pod creation and the `PodInitialized` Condition (when all
- initContainers have finished and all images are pulled).
- As a Pod can be forced to restart and initContainers are re-run, this metrics
- only represents the time to the first `PodInitialized` condition.
-* `ready_latency`: a floating-point value representing the duration in seconds
- between the Pod creation and the `PodReady` Condition (when all the containers
- have started and their readinessProbe have succeeded).
- As a Pod can become unhealthy after it's first Ready, this metric only
- represents the time to the first `PodReady` Condition.
-
-#### Example
-
-```json
-{
- "kube_transition_metrics": {
- "ready_latency": 60,
- "scheduled_latency": 1,
- "initialized_latency": 9
- },
- "level": "info",
- "kube_transition_metric_type": "pod",
- "time": "2023-09-14T14:53:50Z",
- "kube_namespace": "default",
- "pod_name": "example-pod-748d867d77-rjdd2"
-}
-```
-
-### `container` metrics
-
-* `init_container`: a boolean value representing if the container in question is
- an initContainer.
-* `started_latency`: a floating-point value representing the duration in
- seconds between the Pod creation and container "start" container status (when
- the startProbe passes if it is set).
-* `ready_latency`: a floating-point value representing the duration in
- seconds between the Pod creation and the "ready" container status (when the
- readinessProbe has passed).
-* `running_latency`: a floating-point value representing the duration in seconds
- between the Pod creation and the "Running" container status (when a
- container's entry point is first `execve(2)`ed).
-
-#### Example
-
-Note: as it's relatively rare for a container to have a separate readinessProbe
-and startProbe, it's not uncommon for some of these values to equal each-other.
-```json
-{
- "kube_transition_metrics": {
- "started_latency": 136.075222622,
- "init_container": false,
- "ready_latency": 136.075222622,
- "running_latency": 70.558999639
- },
- "container_name": "is-a-containers-container",
- "level": "info",
- "kube_transition_metric_type": "container",
- "time": "2023-09-14T15:06:39Z",
- "kube_namespace": "default",
- "pod_name": "other-example-76587b845b-xvbpn"
-}
-```
-
-### `image_pull` metrics
-
-Note: Images are pulled only if the image is absent on the node, or if the
-`imagePullPolicy` is set to "Always".
-An image won't be pulled more than once for the same pod, even if the same image
-is used across multiple containers.
-This metric is emitted solely when the Kubelet initiates an "ImagePulling"
-event.
-
-* `init_container`: A boolean indicating whether the container, for which the
- image was pulled, is an initContainer.
-* `image_pull_duration`: Represents the duration (in seconds) between the
- "ImagePulling" and "ImagePulled" events.
- Though the "ImagePulled" event message contains a more precise, human-readable
- duration of the image pull, it isn't parsed.
- Thus, this metric's accuracy might differ slightly since it relies on event
- timestamps.
-
-#### Example
-
-```json
-{
- "kube_transition_metrics": {
- "image_pull_duration": 16,
- "init_container": true
- },
- "container_name": "an-init-container",
- "level": "info",
- "kube_transition_metric_type": "image_pull",
- "time": "2023-09-14T15:13:22Z",
- "kube_namespace": "default",
- "pod_name": "my-pod-28245073-nzpvm"
- "message": "Successfully pulled image \"ghcr.io/backmarket-oss/kube-transition-metrics:latest\" in 15094.782505ms (15273.331032ms including waiting)"
-}
-```
diff --git a/internal/statistics/container_statistics.go b/internal/statistics/container_statistics.go
index b9851e0..839eed1 100644
--- a/internal/statistics/container_statistics.go
+++ b/internal/statistics/container_statistics.go
@@ -4,6 +4,7 @@ import (
"time"
"github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
corev1 "k8s.io/api/core/v1"
)
@@ -13,6 +14,9 @@ type containerStatistic struct {
pod *podStatistic
imagePull imagePullStatistic
+ // Previous init container, null if first init container or non-init container
+ previous *containerStatistic
+
// The timestamp for when the container first turned Running.
runningTimestamp time.Time
@@ -46,32 +50,68 @@ func (cs containerStatistic) logger() zerolog.Logger {
Logger()
}
+func (cs containerStatistic) appendInitFields(event *zerolog.Event) {
+ if !cs.runningTimestamp.IsZero() && cs.previous != nil && !cs.previous.readyTimestamp.IsZero() {
+ event.Dur("previous_to_running_seconds", cs.runningTimestamp.Sub(cs.previous.readyTimestamp))
+ }
+}
+
+func (cs containerStatistic) appendNonInitFields(event *zerolog.Event) {
+ if !cs.runningTimestamp.IsZero() && !cs.pod.scheduledTimestamp.IsZero() {
+ event.Dur("initialized_to_running_seconds", cs.runningTimestamp.Sub(cs.pod.scheduledTimestamp))
+ }
+}
+
func (cs containerStatistic) event() *zerolog.Event {
event := zerolog.Dict()
+ event.Str("name", cs.name)
event.Bool("init_container", cs.initContainer)
+ if cs.initContainer {
+ cs.appendInitFields(event)
+ } else {
+ cs.appendNonInitFields(event)
+ }
+
+ if !cs.runningTimestamp.IsZero() {
+ event.Time("running_timestamp", cs.runningTimestamp)
+ }
if !cs.startedTimestamp.IsZero() {
- event.Float64("started_latency",
- cs.startedTimestamp.Sub(cs.pod.creationTimestamp).Seconds())
+ event.Time("started_timestamp", cs.startedTimestamp)
+ if !cs.runningTimestamp.IsZero() {
+ event.Dur("running_to_started_seconds", cs.startedTimestamp.Sub(cs.runningTimestamp))
+ }
}
if !cs.readyTimestamp.IsZero() {
- event.Float64("ready_latency",
- cs.readyTimestamp.Sub(cs.pod.creationTimestamp).Seconds())
- }
- if !cs.runningTimestamp.IsZero() {
- event.Float64("running_latency",
- cs.runningTimestamp.Sub(cs.pod.creationTimestamp).Seconds())
+ event.Time("ready_timestamp", cs.readyTimestamp)
+
+ // As init containers do not supported startup, liveliness, or readiness probes the Started container status field is
+ // not set for init containers.
+ // Instead, readiness represents the time an init container has excited successfully,allowing the following containers
+ // to proceed.
+ // Given this, presenting both running_to_ready_seconds and started_to_ready_seconds is useful to cover the differing
+ // meanings for both container types.
+ // See: https://github.com/kubernetes/website/blob/b397a8f/content/en/docs/concepts/workloads/pods/init-containers.md
+ if !cs.runningTimestamp.IsZero() {
+ event.Dur("running_to_ready_seconds", cs.readyTimestamp.Sub(cs.runningTimestamp))
+ }
+ if !cs.startedTimestamp.IsZero() {
+ event.Dur("started_to_ready_seconds", cs.readyTimestamp.Sub(cs.runningTimestamp))
+ }
}
return event
}
func (cs containerStatistic) report() {
- logger := cs.logger()
-
- eventLogger := logger.Output(metricOutput).With().
- Str("kube_transition_metric_type", "container").
- Dict("kube_transition_metrics", cs.event()).
+ metrics := zerolog.Dict()
+ metrics.Str("type", "container")
+ metrics.Str("kube_namespace", cs.pod.namespace)
+ metrics.Str("pod_name", cs.pod.name)
+ metrics.Dict("container", cs.event())
+
+ eventLogger := log.Output(metricOutput).With().
+ Dict("kube_transition_metrics", metrics).
Logger()
eventLogger.Log().Msg("")
}
diff --git a/internal/statistics/image_pull_collector.go b/internal/statistics/image_pull_collector.go
index edefae3..721ca38 100644
--- a/internal/statistics/image_pull_collector.go
+++ b/internal/statistics/image_pull_collector.go
@@ -135,11 +135,11 @@ func (ev imagePullingEvent) handle(statistic *podStatistic) bool {
}
imagePullStatistic := &containerStatistic.imagePull
- if !imagePullStatistic.finishedAt.IsZero() {
+ if !imagePullStatistic.finishedTimestamp.IsZero() {
logger.Debug().Str("container_name", ev.containerName).
Msg("Skipping event for initialized pod")
- } else if imagePullStatistic.startedAt.IsZero() {
- imagePullStatistic.startedAt = ev.event.FirstTimestamp.Time
+ } else if imagePullStatistic.startedTimestamp.IsZero() {
+ imagePullStatistic.startedTimestamp = ev.event.FirstTimestamp.Time
}
return false
@@ -194,8 +194,12 @@ func (ev imagePulledEvent) handle(statistic *podStatistic) bool {
}
imagePullStatistic := &containerStatistic.imagePull
- if imagePullStatistic.finishedAt.IsZero() {
- imagePullStatistic.finishedAt = ev.event.LastTimestamp.Time
+ if imagePullStatistic.finishedTimestamp.IsZero() {
+ imagePullStatistic.finishedTimestamp = ev.event.LastTimestamp.Time
+ if imagePullStatistic.startedTimestamp.IsZero() {
+ imagePullStatistic.alreadyPresent = true
+ imagePullStatistic.startedTimestamp = imagePullStatistic.finishedTimestamp
+ }
}
imagePullStatistic.log(ev.event.Message)
diff --git a/internal/statistics/image_pull_statistics.go b/internal/statistics/image_pull_statistics.go
index 424122c..b1e51bc 100644
--- a/internal/statistics/image_pull_statistics.go
+++ b/internal/statistics/image_pull_statistics.go
@@ -4,30 +4,42 @@ import (
"time"
"github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
)
type imagePullStatistic struct {
container *containerStatistic
- startedAt time.Time
- finishedAt time.Time
+ alreadyPresent bool
+ startedTimestamp time.Time
+ finishedTimestamp time.Time
}
func (s imagePullStatistic) log(message string) {
- metrics := zerolog.Dict()
- metrics.Bool("init_container", s.container.initContainer)
- if !s.finishedAt.IsZero() && !s.startedAt.IsZero() {
- metrics.Float64(
- "image_pull_duration",
- s.finishedAt.Sub(s.startedAt).Seconds(),
- )
+ imagePullMetrics := zerolog.Dict()
+
+ imagePullMetrics.Str("container_name", s.container.name)
+ imagePullMetrics.Bool("already_present", s.alreadyPresent)
+ if !s.startedTimestamp.IsZero() {
+ imagePullMetrics.Time("started_timestamp", s.startedTimestamp)
+ }
+ if !s.finishedTimestamp.IsZero() {
+ imagePullMetrics.Time("finished_timestamp", s.finishedTimestamp)
+ if !s.startedTimestamp.IsZero() {
+ imagePullMetrics.Dur("duration_seconds", s.finishedTimestamp.Sub(s.startedTimestamp))
+ }
}
+ metrics := zerolog.Dict()
+ metrics.Str("type", "image_pull")
+ metrics.Dict("image_pull", imagePullMetrics)
+ metrics.Str("kube_namespace", s.container.pod.namespace)
+ metrics.Str("pod_name", s.container.pod.name)
+
logger :=
- s.container.logger().
+ log.
Output(metricOutput).
With().
- Str("kube_transition_metric_type", "image_pull").
Dict("kube_transition_metrics", metrics).
Logger()
logger.Log().Msg(message)
diff --git a/internal/statistics/metric_output.go b/internal/statistics/metric_output.go
index 1277caa..3e9051c 100644
--- a/internal/statistics/metric_output.go
+++ b/internal/statistics/metric_output.go
@@ -3,7 +3,13 @@ package statistics
import (
"io"
"os"
+
+ "github.com/BackMarket-oss/kube-transition-metrics/internal/logging"
+ "github.com/rs/zerolog"
)
//nolint:gochecknoglobals
-var metricOutput io.Writer = os.Stdout
+var metricOutput io.Writer = zerolog.MultiLevelWriter(
+ os.Stdout,
+ logging.NewValidationWriter(),
+)
diff --git a/internal/statistics/pod_statistics.go b/internal/statistics/pod_statistics.go
index 695148a..2388320 100644
--- a/internal/statistics/pod_statistics.go
+++ b/internal/statistics/pod_statistics.go
@@ -46,8 +46,11 @@ func (s *podStatistic) initialize(pod *corev1.Pod) {
s.initContainers = make(map[string]*containerStatistic)
s.containers = make(map[string]*containerStatistic)
+ var previous *containerStatistic
for _, container := range pod.Spec.InitContainers {
s.initContainers[container.Name] = newContainerStatistic(s, true, container)
+ s.initContainers[container.Name].previous = previous
+ previous = s.initContainers[container.Name]
}
for _, container := range pod.Spec.Containers {
s.containers[container.Name] = newContainerStatistic(s, false, container)
@@ -64,31 +67,38 @@ func (s podStatistic) logger() zerolog.Logger {
func (s podStatistic) event() *zerolog.Event {
event := zerolog.Dict()
+ event.Time("creation_timestamp", s.creationTimestamp)
if !s.scheduledTimestamp.IsZero() {
- event.Float64(
- "scheduled_latency",
- s.scheduledTimestamp.Sub(s.creationTimestamp).Seconds())
+ event.Time("scheduled_timestamp", s.scheduledTimestamp)
+ event.Dur("creation_to_scheduled_seconds", s.scheduledTimestamp.Sub(s.creationTimestamp))
}
if !s.initializedTimestamp.IsZero() {
- event.Float64(
- "initialized_latency",
- s.initializedTimestamp.Sub(s.creationTimestamp).Seconds())
+ event.Time("initialized_timestamp", s.initializedTimestamp)
+ event.Dur("creation_to_initialized_seconds", s.initializedTimestamp.Sub(s.creationTimestamp))
+ if !s.scheduledTimestamp.IsZero() {
+ event.Dur("scheduled_to_initialized_seconds", s.initializedTimestamp.Sub(s.scheduledTimestamp))
+ }
}
if !s.readyTimestamp.IsZero() {
- event.Float64(
- "ready_latency",
- s.readyTimestamp.Sub(s.creationTimestamp).Seconds())
+ event.Time("ready_timestamp", s.readyTimestamp)
+ event.Dur("creation_to_ready_seconds", s.readyTimestamp.Sub(s.creationTimestamp))
+ if !s.initializedTimestamp.IsZero() {
+ event.Dur("initialized_to_ready_seconds", s.readyTimestamp.Sub(s.initializedTimestamp))
+ }
}
return event
}
func (s podStatistic) report() {
- logger := s.logger()
-
- eventLogger := logger.Output(metricOutput).With().
- Str("kube_transition_metric_type", "pod").
- Dict("kube_transition_metrics", s.event()).Logger()
+ metrics := zerolog.Dict()
+ metrics.Str("type", "pod")
+ metrics.Str("kube_namespace", s.namespace)
+ metrics.Str("pod_name", s.name)
+ metrics.Dict("pod", s.event())
+
+ eventLogger := log.Output(metricOutput).With().
+ Dict("kube_transition_metrics", metrics).Logger()
eventLogger.Log().Msg("")
for _, containerStatistics := range s.initContainers {
@@ -116,10 +126,8 @@ func (s *podStatistic) update(pod *corev1.Pod) {
s.scheduledTimestamp = condition.LastTransitionTime.Time
}
case corev1.PodInitialized:
- // Pod Initialized occursafter all images pulled, no need to continue to
- // track
-
if s.initializedTimestamp.IsZero() {
+ // Pod Initialized occurs, after all images pulled, no need to continue to track
go s.imagePullCollector.cancel("pod_initialized")
s.initializedTimestamp = condition.LastTransitionTime.Time
}
diff --git a/internal/statistics/statistics_test.go b/internal/statistics/statistics_test.go
index f0f470b..b0fef5d 100644
--- a/internal/statistics/statistics_test.go
+++ b/internal/statistics/statistics_test.go
@@ -8,13 +8,16 @@ import (
"testing"
"time"
+ "github.com/BackMarket-oss/kube-transition-metrics/internal/logging"
"github.com/BackMarket-oss/kube-transition-metrics/internal/options"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
-func setupLoggerToBuffer() *bytes.Buffer {
+func redirectMetricsToBuffer() *bytes.Buffer {
+ logging.Configure()
+
buf := &bytes.Buffer{}
metricOutput = buf
@@ -56,18 +59,15 @@ func (mts MockTimeSource) Now() time.Time {
return mts.mockedTime
}
-func TestPodStatisticUpdate(t *testing.T) {
- // Redirect logger to buffer
- buf := setupLoggerToBuffer()
-
- // 1. Setup a sample pod
- format := "2006-01-02T15:04:05Z07:00"
- created, err := time.Parse(format, "2023-08-28T00:00:00Z")
- if err != nil {
- panic(err)
- }
+func stubImagePullCollector(ipc *imagePullCollector) {
+ go func() {
+ // Stub cancel so that when it is called it can unblock
+ <-ipc.cancelChan
+ }()
+}
- pod := &corev1.Pod{
+func newTestingPod(created time.Time) *corev1.Pod {
+ return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
CreationTimestamp: metav1.NewTime(created),
Name: "test-pod",
@@ -102,12 +102,14 @@ func TestPodStatisticUpdate(t *testing.T) {
},
},
}
+}
- stat := podStatistic{}
+func initializePodStatistic(pod *corev1.Pod) *podStatistic {
+ stat := &podStatistic{}
stat.initialize(pod)
// Stub time source to fix time to constant for testing
- stat.timeSource = MockTimeSource{created.Add(3 * time.Second)}
+ stat.timeSource = MockTimeSource{pod.CreationTimestamp.Time}
// Stub StatisticEventHandler
eh := &StatisticEventHandler{
@@ -115,13 +117,26 @@ func TestPodStatisticUpdate(t *testing.T) {
}
stat.imagePullCollector = newImagePullCollector(eh, "", pod.UID)
- // Update the pod statistic for the "new" state
- stat.update(pod)
+ return stat
+}
+func checkBasicPodStatisticFields(t *testing.T, stat *podStatistic) {
+ t.Helper()
assert.NotZero(t, stat.scheduledTimestamp, "scheduledTimestamp was not set")
assert.NotZero(
t, stat.initializedTimestamp, "initializedTimestamp was not set")
assert.NotEmpty(t, stat.containers, "containers map was not populated")
+}
+
+func TestImageCollectorCancel(t *testing.T) {
+ format := "2006-01-02T15:04:05Z07:00"
+ created, err := time.Parse(format, "2024-03-31T00:00:00Z")
+ if err != nil {
+ panic(err)
+ }
+ pod := newTestingPod(created)
+ stat := initializePodStatistic(pod)
+ stat.update(pod)
// Check that the imagePullCollector would have been canceled for the right
// reasons upon pod initialization.
@@ -137,7 +152,10 @@ func TestPodStatisticUpdate(t *testing.T) {
assert.Fail(
t, "ImagePullCollector cancel channel was not written to within 1 second")
}
+}
+func decodeMetrics(t *testing.T, buf *bytes.Buffer) []map[string]interface{} {
+ t.Helper()
decoder := json.NewDecoder(buf)
statisticLogs := make([]map[string]interface{}, 0)
for {
@@ -148,15 +166,41 @@ func TestPodStatisticUpdate(t *testing.T) {
t.Errorf("Invalid JSON output")
}
- if mapDocument, ok := document.(map[string]interface{}); ok {
- if _, ok := mapDocument["kube_transition_metric_type"]; ok {
- statisticLogs = append(statisticLogs, mapDocument)
- }
- } else {
- t.Errorf("Log document is not a map")
+ if !assert.IsType(t, make(map[string]interface{}), document, "Log document is not an object") {
+ continue
+ }
+ mapDocument, _ := document.(map[string]interface{})
+ if !assert.IsType(t, make(map[string]interface{}), mapDocument["kube_transition_metrics"],
+ "kube_transition_metric key of log document is not an object") {
+ continue
}
+ mapMetrics, _ := mapDocument["kube_transition_metrics"].(map[string]interface{})
+ statisticLogs = append(statisticLogs, mapMetrics)
}
+ return statisticLogs
+}
+
+func TestPodStatisticUpdate(t *testing.T) {
+ buf := redirectMetricsToBuffer()
+
+ format := "2006-01-02T15:04:05Z07:00"
+ created, err := time.Parse(format, "2023-08-28T00:00:00Z")
+ if err != nil {
+ panic(err)
+ }
+ pod := newTestingPod(created)
+ stat := initializePodStatistic(pod)
+
+ stubImagePullCollector(&stat.imagePullCollector)
+ stat.timeSource = MockTimeSource{pod.CreationTimestamp.Time.Add(3 * time.Second)}
+
+ // Update the pod statistic for the "new" state
+ stat.update(pod)
+
+ checkBasicPodStatisticFields(t, stat)
+ statisticLogs := decodeMetrics(t, buf)
+
if !assert.Len(
t, statisticLogs, 2, "Not the correct number of statistic logs") {
return
@@ -167,39 +211,34 @@ func TestPodStatisticUpdate(t *testing.T) {
assert.Equal(t, "test-pod", log["pod_name"])
}
- sharedAssertations(statisticLogs[0])
-
+ metrics := statisticLogs[0]
+ sharedAssertations(metrics)
assert.Equal(t,
- "pod", statisticLogs[0]["kube_transition_metric_type"],
+ "pod", metrics["type"],
"first log metric is not of type pod")
+ if assert.IsType(t, make(map[string]interface{}), metrics["pod"]) {
+ podMetrics, _ := metrics["pod"].(map[string]interface{})
+ assert.InDelta(t,
+ 2*time.Second.Seconds(), podMetrics["creation_to_initialized_seconds"], 1e-5,
+ "Initialized latency is not correct")
+ assert.InDelta(t,
+ time.Second.Seconds(), podMetrics["creation_to_scheduled_seconds"], 1e-5,
+ "Scheduled latency is not correct")
+ }
- assert.IsType(t,
- make(map[string]interface{}),
- statisticLogs[0]["kube_transition_metrics"],
- "key kube_transition_metrics is not a JSON object")
- metrics, _ :=
- statisticLogs[0]["kube_transition_metrics"].(map[string]interface{})
- assert.InDelta(t,
- 2*time.Second.Seconds(), metrics["initialized_latency"], 1e-5,
- "Initialized latency is not correct")
- assert.InDelta(t,
- time.Second.Seconds(), metrics["scheduled_latency"], 1e-5,
- "Scheduled latency is not correct")
-
- sharedAssertations(statisticLogs[1])
+ metrics = statisticLogs[1]
+ sharedAssertations(metrics)
assert.Equal(t,
- "container", statisticLogs[1]["kube_transition_metric_type"],
+ "container", metrics["type"],
"second log metric is not of type container")
- assert.IsType(t,
- make(map[string]interface{}), statisticLogs[1]["kube_transition_metrics"],
- "key kube_transition_metrics is not a JSON object")
- metrics, _ =
- statisticLogs[1]["kube_transition_metrics"].(map[string]interface{})
- assert.Equal(t,
- false, metrics["init_container"], "Container should not be an init container")
- assert.InDelta(t,
- 3*time.Second.Seconds(), metrics["ready_latency"], 1e-5,
- "Container ready latency is not correct")
+ if assert.IsType(t, make(map[string]interface{}), metrics["container"]) {
+ containerMetrics, _ := metrics["container"].(map[string]interface{})
+ assert.Equal(t,
+ false, containerMetrics["init_container"], "Container should not be an init container")
+ assert.InDelta(t,
+ 2*time.Second.Seconds(), containerMetrics["initialized_to_running_seconds"], 1e-5,
+ "Container runnning latency is not correct")
+ }
}
func TestContainerStatisticUpdate(t *testing.T) {