diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 585cc45df94..cdea01c0c41 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -349,6 +349,7 @@ type tsdbMetrics struct { tsdbHeadTruncateFail *prometheus.Desc tsdbHeadTruncateTotal *prometheus.Desc tsdbHeadGcDuration *prometheus.Desc + tsdbHeadSeries *prometheus.Desc tsdbActiveAppenders *prometheus.Desc tsdbSeriesNotFound *prometheus.Desc tsdbChunks *prometheus.Desc @@ -454,6 +455,10 @@ func newTSDBMetrics(r prometheus.Registerer) *tsdbMetrics { "cortex_ingester_tsdb_head_gc_duration_seconds", "Runtime of garbage collection in the TSDB head.", nil, nil), + tsdbHeadSeries: prometheus.NewDesc( + "cortex_ingester_tsdb_head_series", + "Total number of series in the TSDB head block.", + []string{"user"}, nil), tsdbActiveAppenders: prometheus.NewDesc( "cortex_ingester_tsdb_head_active_appenders", "Number of currently active TSDB appender transactions.", @@ -573,6 +578,7 @@ func (sm *tsdbMetrics) Describe(out chan<- *prometheus.Desc) { out <- sm.tsdbHeadTruncateFail out <- sm.tsdbHeadTruncateTotal out <- sm.tsdbHeadGcDuration + out <- sm.tsdbHeadSeries out <- sm.tsdbActiveAppenders out <- sm.tsdbSeriesNotFound out <- sm.tsdbChunks @@ -622,6 +628,7 @@ func (sm *tsdbMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfCounters(out, sm.tsdbHeadTruncateFail, "prometheus_tsdb_head_truncations_failed_total") data.SendSumOfCounters(out, sm.tsdbHeadTruncateTotal, "prometheus_tsdb_head_truncations_total") data.SendSumOfSummaries(out, sm.tsdbHeadGcDuration, "prometheus_tsdb_head_gc_duration_seconds") + data.SendSumOfGaugesPerUser(out, sm.tsdbHeadSeries, "prometheus_tsdb_head_series") data.SendSumOfGauges(out, sm.tsdbActiveAppenders, "prometheus_tsdb_head_active_appenders") data.SendSumOfCounters(out, sm.tsdbSeriesNotFound, "prometheus_tsdb_head_series_not_found_total") data.SendSumOfGauges(out, sm.tsdbChunks, "prometheus_tsdb_head_chunks") diff --git a/pkg/ingester/metrics_test.go b/pkg/ingester/metrics_test.go index c60e0ae6e24..d62b8c064e2 100644 --- a/pkg/ingester/metrics_test.go +++ b/pkg/ingester/metrics_test.go @@ -139,6 +139,13 @@ func TestTSDBMetrics(t *testing.T) { # TYPE cortex_ingester_tsdb_head_active_appenders gauge cortex_ingester_tsdb_head_active_appenders 1982620 + # HELP cortex_ingester_tsdb_head_series Total number of series in the TSDB head block. + # TYPE cortex_ingester_tsdb_head_series gauge + # 20 * (12345 and 85787 and 999 respectively) + cortex_ingester_tsdb_head_series{user="user1"} 246900 + cortex_ingester_tsdb_head_series{user="user2"} 1715740 + cortex_ingester_tsdb_head_series{user="user3"} 19980 + # HELP cortex_ingester_tsdb_head_series_not_found_total Total number of TSDB requests for series that were not found. # TYPE cortex_ingester_tsdb_head_series_not_found_total counter cortex_ingester_tsdb_head_series_not_found_total 2081751 @@ -351,6 +358,12 @@ func TestTSDBMetricsWithRemoval(t *testing.T) { # TYPE cortex_ingester_tsdb_head_active_appenders gauge cortex_ingester_tsdb_head_active_appenders 1962640 + # HELP cortex_ingester_tsdb_head_series Total number of series in the TSDB head block. + # TYPE cortex_ingester_tsdb_head_series gauge + # 20 * (12345 and 85787 and 999 respectively) + cortex_ingester_tsdb_head_series{user="user1"} 246900 + cortex_ingester_tsdb_head_series{user="user2"} 1715740 + # HELP cortex_ingester_tsdb_head_series_not_found_total Total number of TSDB requests for series that were not found. # TYPE cortex_ingester_tsdb_head_series_not_found_total counter cortex_ingester_tsdb_head_series_not_found_total 2081751 @@ -542,6 +555,12 @@ func populateTSDBMetrics(base float64) *prometheus.Registry { }) checkpointCreationTotal.Add(19 * base) + series := promauto.With(r).NewGauge(prometheus.GaugeOpts{ + Name: "prometheus_tsdb_head_series", + Help: "Total number of series in the head block.", + }) + series.Set(20 * base) + activeAppenders := promauto.With(r).NewGauge(prometheus.GaugeOpts{ Name: "prometheus_tsdb_head_active_appenders", Help: "Number of currently active appender transactions",