From 2adf66829213c119e8657e86bfaa5a0f402e4918 Mon Sep 17 00:00:00 2001 From: Feng Kun Date: Thu, 26 Mar 2020 16:05:50 +0800 Subject: [PATCH] fix: get metrics failed if values contain NaN feat: set retention time to 2h to reduce memory usage of prometheus Signed-off-by: Feng Kun --- pkg/monitor/storage/thanos/metric/metric.go | 5 ++++- pkg/platform/controller/addon/prometheus/controller.go | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/monitor/storage/thanos/metric/metric.go b/pkg/monitor/storage/thanos/metric/metric.go index b1666807f..dc200f403 100644 --- a/pkg/monitor/storage/thanos/metric/metric.go +++ b/pkg/monitor/storage/thanos/metric/metric.go @@ -21,6 +21,7 @@ package metric import ( "context" "fmt" + "math" "strings" "time" "tkestack.io/tke/api/monitor" @@ -222,7 +223,9 @@ func MergeResult(results model.Matrix, timestamp string, groupByWithoutTimestamp } } values[index].([]interface{})[0] = sp.Timestamp.Unix() * 1000 - values[index].([]interface{})[fieldIndex+1] = float64(sp.Value) + if !math.IsNaN(float64(sp.Value)) { + values[index].([]interface{})[fieldIndex+1] = float64(sp.Value) + } for j, tag := range tags { values[index].([]interface{})[len(fieldIndexes)+1+j] = tag } diff --git a/pkg/platform/controller/addon/prometheus/controller.go b/pkg/platform/controller/addon/prometheus/controller.go index 6cc31d532..0af04ee5c 100755 --- a/pkg/platform/controller/addon/prometheus/controller.go +++ b/pkg/platform/controller/addon/prometheus/controller.go @@ -1166,6 +1166,7 @@ func createPrometheusCRD(components images.Components, clusterName string, remot ScrapeInterval: "60s", RemoteRead: remoteReadSpecs, RemoteWrite: remoteWriteSpecs, + Retention: "2h", EvaluationInterval: "1m", AdditionalScrapeConfigs: &corev1.SecretKeySelector{ LocalObjectReference: corev1.LocalObjectReference{Name: prometheusSecret},