@@ -254,11 +254,12 @@ func (r tsdbCloseCheckResult) shouldClose() bool {
254
254
}
255
255
256
256
type userTSDB struct {
257
- db * tsdb.DB
258
- userID string
259
- activeSeries * ActiveSeries
260
- seriesInMetric * metricCounter
261
- limiter * Limiter
257
+ db * tsdb.DB
258
+ userID string
259
+ activeSeries * ActiveSeries
260
+ seriesInMetric * metricCounter
261
+ labelSetCounter * labelSetCounter
262
+ limiter * Limiter
262
263
263
264
instanceSeriesCount * atomic.Int64 // Shared across all userTSDB instances created by ingester.
264
265
instanceLimitsFn func () * InstanceLimits
@@ -399,6 +400,10 @@ func (u *userTSDB) PreCreation(metric labels.Labels) error {
399
400
return err
400
401
}
401
402
403
+ if err := u .labelSetCounter .canAddSeriesForLabelSet (context .TODO (), u , metric ); err != nil {
404
+ return err
405
+ }
406
+
402
407
return nil
403
408
}
404
409
@@ -412,6 +417,7 @@ func (u *userTSDB) PostCreation(metric labels.Labels) {
412
417
return
413
418
}
414
419
u .seriesInMetric .increaseSeriesForMetric (metricName )
420
+ u .labelSetCounter .increaseSeriesLabelSet (u , metric )
415
421
}
416
422
417
423
// PostDeletion implements SeriesLifecycleCallback interface.
@@ -425,6 +431,7 @@ func (u *userTSDB) PostDeletion(metrics map[chunks.HeadSeriesRef]labels.Labels)
425
431
continue
426
432
}
427
433
u .seriesInMetric .decreaseSeriesForMetric (metricName )
434
+ u .labelSetCounter .decreaseSeriesLabelSet (u , metric )
428
435
}
429
436
}
430
437
@@ -924,6 +931,7 @@ func (i *Ingester) updateActiveSeries() {
924
931
925
932
userDB .activeSeries .Purge (purgeTime )
926
933
i .metrics .activeSeriesPerUser .WithLabelValues (userID ).Set (float64 (userDB .activeSeries .Active ()))
934
+ userDB .labelSetCounter .UpdateMetric (userDB , i .metrics .activeSeriesPerLabelSet )
927
935
}
928
936
}
929
937
@@ -1100,38 +1108,43 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte
1100
1108
// of it, so that we can return it back to the distributor, which will return a
1101
1109
// 400 error to the client. The client (Prometheus) will not retry on 400, and
1102
1110
// we actually ingested all samples which haven't failed.
1103
- switch cause := errors .Cause (err ); cause {
1104
- case storage .ErrOutOfBounds :
1111
+ switch cause := errors .Cause (err ); {
1112
+ case errors . Is ( cause , storage .ErrOutOfBounds ) :
1105
1113
sampleOutOfBoundsCount ++
1106
1114
updateFirstPartial (func () error { return wrappedTSDBIngestErr (err , model .Time (s .TimestampMs ), ts .Labels ) })
1107
1115
continue
1108
1116
1109
- case storage .ErrOutOfOrderSample :
1117
+ case errors . Is ( cause , storage .ErrOutOfOrderSample ) :
1110
1118
sampleOutOfOrderCount ++
1111
1119
updateFirstPartial (func () error { return wrappedTSDBIngestErr (err , model .Time (s .TimestampMs ), ts .Labels ) })
1112
1120
continue
1113
1121
1114
- case storage .ErrDuplicateSampleForTimestamp :
1122
+ case errors . Is ( cause , storage .ErrDuplicateSampleForTimestamp ) :
1115
1123
newValueForTimestampCount ++
1116
1124
updateFirstPartial (func () error { return wrappedTSDBIngestErr (err , model .Time (s .TimestampMs ), ts .Labels ) })
1117
1125
continue
1118
1126
1119
- case storage .ErrTooOldSample :
1127
+ case errors . Is ( cause , storage .ErrTooOldSample ) :
1120
1128
sampleTooOldCount ++
1121
1129
updateFirstPartial (func () error { return wrappedTSDBIngestErr (err , model .Time (s .TimestampMs ), ts .Labels ) })
1122
1130
continue
1123
1131
1124
- case errMaxSeriesPerUserLimitExceeded :
1132
+ case errors . Is ( cause , errMaxSeriesPerUserLimitExceeded ) :
1125
1133
perUserSeriesLimitCount ++
1126
1134
updateFirstPartial (func () error { return makeLimitError (perUserSeriesLimit , i .limiter .FormatError (userID , cause )) })
1127
1135
continue
1128
1136
1129
- case errMaxSeriesPerMetricLimitExceeded :
1137
+ case errors . Is ( cause , errMaxSeriesPerMetricLimitExceeded ) :
1130
1138
perMetricSeriesLimitCount ++
1131
1139
updateFirstPartial (func () error {
1132
1140
return makeMetricLimitError (perMetricSeriesLimit , copiedLabels , i .limiter .FormatError (userID , cause ))
1133
1141
})
1134
1142
continue
1143
+ case errors .As (cause , & errMaxSeriesPerLabelSetLimitExceeded {}):
1144
+ updateFirstPartial (func () error {
1145
+ return makeMetricLimitError (perLabelsetSeriesLimit , copiedLabels , i .limiter .FormatError (userID , cause ))
1146
+ })
1147
+ continue
1135
1148
}
1136
1149
1137
1150
// The error looks an issue on our side, so we should rollback
@@ -2018,6 +2031,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) {
2018
2031
userID : userID ,
2019
2032
activeSeries : NewActiveSeries (),
2020
2033
seriesInMetric : newMetricCounter (i .limiter , i .cfg .getIgnoreSeriesLimitForMetricNamesMap ()),
2034
+ labelSetCounter : newLabelSetCounter (i .limiter ),
2021
2035
ingestedAPISamples : util_math .NewEWMARate (0.2 , i .cfg .RateUpdatePeriod ),
2022
2036
ingestedRuleSamples : util_math .NewEWMARate (0.2 , i .cfg .RateUpdatePeriod ),
2023
2037
0 commit comments