Merge pull request #628 from fengyehong/agg

Add kube pod resource limits aggregation
kubernetes-monitoring · Jun 22, 2021 · e0dc356 · e0dc356
2 parents aead52c + 15e00d7
commit e0dc356
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 9 deletions.
diff --git a/dashboards/resources/cluster.libsonnet b/dashboards/resources/cluster.libsonnet
@@ -149,7 +149,7 @@ local template = grafana.template;
         )
         .addPanel(
           g.panel('CPU Limits Commitment') +
-          g.statPanel('sum(kube_pod_container_resource_limits{%(clusterLabel)s="$cluster", resource="cpu"}) / sum(kube_node_status_allocatable{resource="cpu",%(clusterLabel)s="$cluster"})' % $._config)
+          g.statPanel('sum(namespace_cpu:kube_pod_container_resource_limits:sum{%(clusterLabel)s="$cluster"}) / sum(kube_node_status_allocatable{resource="cpu",%(clusterLabel)s="$cluster"})' % $._config)
         )
         .addPanel(
           g.panel('Memory Utilisation') +
@@ -161,7 +161,7 @@ local template = grafana.template;
         )
         .addPanel(
           g.panel('Memory Limits Commitment') +
-          g.statPanel('sum(kube_pod_container_resource_limits{%(clusterLabel)s="$cluster", resource="memory"}) / sum(kube_node_status_allocatable{resource="memory",%(clusterLabel)s="$cluster"})' % $._config)
+          g.statPanel('sum(namespace_memory:kube_pod_container_resource_limits:sum{%(clusterLabel)s="$cluster"}) / sum(kube_node_status_allocatable{resource="memory",%(clusterLabel)s="$cluster"})' % $._config)
         )
       )
       .addRow(
@@ -179,9 +179,9 @@ local template = grafana.template;
           g.tablePanel(podWorkloadColumns + [
             'sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
             'sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
-            'sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{%(clusterLabel)s="$cluster"}) by (namespace) / sum(kube_pod_container_resource_requests{%(clusterLabel)s="$cluster", resource="cpu"}) by (namespace)' % $._config,
-            'sum(kube_pod_container_resource_limits{%(clusterLabel)s="$cluster", resource="cpu"}) by (namespace)' % $._config,
-            'sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{%(clusterLabel)s="$cluster"}) by (namespace) / sum(kube_pod_container_resource_limits{%(clusterLabel)s="$cluster", resource="cpu"}) by (namespace)' % $._config,
+            'sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{%(clusterLabel)s="$cluster"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_requests:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
+            'sum(namespace_cpu:kube_pod_container_resource_limits:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
+            'sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{%(clusterLabel)s="$cluster"}) by (namespace) / sum(namespace_cpu:kube_pod_container_resource_limits:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
           ], tableStyles {
             'Value #C': { alias: 'CPU Usage' },
             'Value #D': { alias: 'CPU Requests' },
@@ -208,10 +208,10 @@ local template = grafana.template;
           g.tablePanel(podWorkloadColumns + [
             // Not using container_memory_usage_bytes here because that includes page cache
             'sum(container_memory_rss{%(clusterLabel)s="$cluster", container!=""}) by (namespace)' % $._config,
-            'sum(kube_pod_container_resource_requests{%(clusterLabel)s="$cluster", resource="memory"}) by (namespace)' % $._config,
-            'sum(container_memory_rss{%(clusterLabel)s="$cluster", container!=""}) by (namespace) / sum(kube_pod_container_resource_requests{%(clusterLabel)s="$cluster", resource="memory"}) by (namespace)' % $._config,
-            'sum(kube_pod_container_resource_limits{%(clusterLabel)s="$cluster", resource="memory"}) by (namespace)' % $._config,
-            'sum(container_memory_rss{%(clusterLabel)s="$cluster", container!=""}) by (namespace) / sum(kube_pod_container_resource_limits{%(clusterLabel)s="$cluster", resource="memory"}) by (namespace)' % $._config,
+            'sum(namespace_memory:kube_pod_container_resource_requests:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
+            'sum(container_memory_rss{%(clusterLabel)s="$cluster", container!=""}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_requests:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
+            'sum(namespace_memory:kube_pod_container_resource_limits:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
+            'sum(container_memory_rss{%(clusterLabel)s="$cluster", container!=""}) by (namespace) / sum(namespace_memory:kube_pod_container_resource_limits:sum{%(clusterLabel)s="$cluster"}) by (namespace)' % $._config,
           ], tableStyles {
             'Value #C': { alias: 'Memory Usage', unit: 'bytes' },
             'Value #D': { alias: 'Memory Requests', unit: 'bytes' },

diff --git a/rules/apps.libsonnet b/rules/apps.libsonnet
@@ -86,6 +86,34 @@
               )
             ||| % $._config,
           },
+          {
+            record: 'namespace_memory:kube_pod_container_resource_limits:sum',
+            expr: |||
+              sum by (namespace, cluster) (
+                  sum by (namespace, pod, cluster) (
+                      max by (namespace, pod, container, cluster) (
+                        kube_pod_container_resource_limits{resource="memory",%(kubeStateMetricsSelector)s}
+                      ) * on(namespace, pod, cluster) group_left() max by (namespace, pod) (
+                        kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                      )
+                  )
+              )
+            ||| % $._config,
+          },
+          {
+            record: 'namespace_cpu:kube_pod_container_resource_limits:sum',
+            expr: |||
+              sum by (namespace, cluster) (
+                  sum by (namespace, pod, cluster) (
+                      max by (namespace, pod, container, cluster) (
+                        kube_pod_container_resource_limits{resource="cpu",%(kubeStateMetricsSelector)s}
+                      ) * on(namespace, pod, cluster) group_left() max by (namespace, pod) (
+                        kube_pod_status_phase{phase=~"Pending|Running"} == 1
+                      )
+                  )
+              )
+            ||| % $._config,
+          },
           // workload aggregation for deployments
           {
             record: 'namespace_workload_pod:kube_pod_owner:relabel',