Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions docs/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,11 @@ We expose several kinds of exporters, including Prometheus, Google Stackdriver,
|-----------------------------------------------------------------------------------------| ----------- |-------------------------------------------------| ----------- |
| `tekton_pipelines_controller_pipelinerun_duration_seconds_[bucket, sum, count]` | Histogram/LastValue(Gauge) | `*pipeline`=&lt;pipeline_name&gt; <br> `*pipelinerun`=&lt;pipelinerun_name&gt; <br> `status`=&lt;status&gt; <br> `namespace`=&lt;pipelinerun-namespace&gt; | experimental |
| `tekton_pipelines_controller_pipelinerun_taskrun_duration_seconds_[bucket, sum, count]` | Histogram/LastValue(Gauge) | `*pipeline`=&lt;pipeline_name&gt; <br> `*pipelinerun`=&lt;pipelinerun_name&gt; <br> `status`=&lt;status&gt; <br> `*task`=&lt;task_name&gt; <br> `*taskrun`=&lt;taskrun_name&gt;<br> `namespace`=&lt;pipelineruns-taskruns-namespace&gt; <br> `*reason`=&lt;reason&gt; | experimental |
| `tekton_pipelines_controller_pipelinerun_count` | Counter | `status`=&lt;status&gt; <br> `*reason`=&lt;reason&gt; | deprecate |
| `tekton_pipelines_controller_pipelinerun_total` | Counter | `status`=&lt;status&gt; | experimental |
| `tekton_pipelines_controller_running_pipelineruns_count` | Gauge | | deprecate |
| `tekton_pipelines_controller_running_pipelineruns` | Gauge | | experimental |
| `tekton_pipelines_controller_taskrun_duration_seconds_[bucket, sum, count]` | Histogram/LastValue(Gauge) | `status`=&lt;status&gt; <br> `*task`=&lt;task_name&gt; <br> `*taskrun`=&lt;taskrun_name&gt;<br> `namespace`=&lt;pipelineruns-taskruns-namespace&gt; <br> `*reason`=&lt;reason&gt; | experimental |
| `tekton_pipelines_controller_taskrun_count` | Counter | `status`=&lt;status&gt; <br> `*reason`=&lt;reason&gt; | deprecate |
| `tekton_pipelines_controller_taskrun_total` | Counter | `status`=&lt;status&gt; | experimental |
| `tekton_pipelines_controller_running_taskruns_count` | Gauge | | deprecate |
| `tekton_pipelines_controller_running_taskruns` | Gauge | | experimental |
| `tekton_pipelines_controller_running_taskruns_throttled_by_quota_count` | Gauge | <br> `namespace`=&lt;pipelinerun-namespace&gt; | deprecate |
| `tekton_pipelines_controller_running_taskruns_throttled_by_node_count` | Gauge | <br> `namespace`=&lt;pipelinerun-namespace&gt; | deprecate |
| `tekton_pipelines_controller_running_taskruns_throttled_by_quota` | Gauge | <br> `namespace`=&lt;pipelinerun-namespace&gt; | experimental |
| `tekton_pipelines_controller_running_taskruns_throttled_by_node` | Gauge | <br> `namespace`=&lt;pipelinerun-namespace&gt; | experimental |
| `tekton_pipelines_controller_client_latency_[bucket, sum, count]` | Histogram | | experimental |
Expand Down
56 changes: 1 addition & 55 deletions pkg/pipelinerunmetrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,41 +61,21 @@ var (
stats.UnitDimensionless)
prDurationView *view.View

prCount = stats.Float64("pipelinerun_count",
"number of pipelineruns",
stats.UnitDimensionless)
prCountView *view.View

prTotal = stats.Float64("pipelinerun_total",
"Number of pipelineruns",
stats.UnitDimensionless)
prTotalView *view.View

runningPRsCount = stats.Float64("running_pipelineruns_count",
"Number of pipelineruns executing currently",
stats.UnitDimensionless)
runningPRsCountView *view.View

runningPRs = stats.Float64("running_pipelineruns",
"Number of pipelineruns executing currently",
stats.UnitDimensionless)
runningPRsView *view.View

runningPRsWaitingOnPipelineResolutionCount = stats.Float64("running_pipelineruns_waiting_on_pipeline_resolution_count",
"Number of pipelineruns executing currently that are waiting on resolution requests for their pipeline references.",
stats.UnitDimensionless)
runningPRsWaitingOnPipelineResolutionCountView *view.View

runningPRsWaitingOnPipelineResolution = stats.Float64("running_pipelineruns_waiting_on_pipeline_resolution",
"Number of pipelineruns executing currently that are waiting on resolution requests for their pipeline references.",
stats.UnitDimensionless)
runningPRsWaitingOnPipelineResolutionView *view.View

runningPRsWaitingOnTaskResolutionCount = stats.Float64("running_pipelineruns_waiting_on_task_resolution_count",
"Number of pipelineruns executing currently that are waiting on resolution requests for the task references of their taskrun children.",
stats.UnitDimensionless)
runningPRsWaitingOnTaskResolutionCountView *view.View

runningPRsWaitingOnTaskResolution = stats.Float64("running_pipelineruns_waiting_on_task_resolution",
"Number of pipelineruns executing currently that are waiting on resolution requests for the task references of their taskrun children.",
stats.UnitDimensionless)
Expand Down Expand Up @@ -201,9 +181,7 @@ func viewRegister(cfg *config.Metrics) error {
}
}

prCountViewTags := []tag.Key{statusTag}
if cfg.CountWithReason {
prCountViewTags = append(prCountViewTags, reasonTag)
prunTag = append(prunTag, reasonTag)
}

Expand All @@ -214,47 +192,26 @@ func viewRegister(cfg *config.Metrics) error {
TagKeys: append([]tag.Key{statusTag, namespaceTag}, prunTag...),
}

prCountView = &view.View{
Description: prCount.Description(),
Measure: prCount,
Aggregation: view.Count(),
TagKeys: prCountViewTags,
}
prTotalView = &view.View{
Description: prTotal.Description(),
Measure: prTotal,
Aggregation: view.Count(),
TagKeys: []tag.Key{statusTag},
}

runningPRsCountView = &view.View{
Description: runningPRsCount.Description(),
Measure: runningPRsCount,
Aggregation: view.LastValue(),
}
runningPRsView = &view.View{
Description: runningPRs.Description(),
Measure: runningPRs,
Aggregation: view.LastValue(),
TagKeys: runningPRTag,
}

runningPRsWaitingOnPipelineResolutionCountView = &view.View{
Description: runningPRsWaitingOnPipelineResolutionCount.Description(),
Measure: runningPRsWaitingOnPipelineResolutionCount,
Aggregation: view.LastValue(),
}
runningPRsWaitingOnPipelineResolutionView = &view.View{
Description: runningPRsWaitingOnPipelineResolution.Description(),
Measure: runningPRsWaitingOnPipelineResolution,
Aggregation: view.LastValue(),
}

runningPRsWaitingOnTaskResolutionCountView = &view.View{
Description: runningPRsWaitingOnTaskResolutionCount.Description(),
Measure: runningPRsWaitingOnTaskResolutionCount,
Aggregation: view.LastValue(),
}
runningPRsWaitingOnTaskResolutionView = &view.View{
Description: runningPRsWaitingOnTaskResolution.Description(),
Measure: runningPRsWaitingOnTaskResolution,
Expand All @@ -263,26 +220,18 @@ func viewRegister(cfg *config.Metrics) error {

return view.Register(
prDurationView,
prCountView,
prTotalView,
runningPRsCountView,
runningPRsView,
runningPRsWaitingOnPipelineResolutionCountView,
runningPRsWaitingOnPipelineResolutionView,
runningPRsWaitingOnTaskResolutionCountView,
runningPRsWaitingOnTaskResolutionView,
)
}

func viewUnregister() {
view.Unregister(prDurationView,
prCountView,
prTotalView,
runningPRsCountView,
runningPRsView,
runningPRsWaitingOnPipelineResolutionCountView,
runningPRsWaitingOnPipelineResolutionView,
runningPRsWaitingOnTaskResolutionCountView,
runningPRsWaitingOnTaskResolutionView)
}

Expand Down Expand Up @@ -412,7 +361,6 @@ func (r *Recorder) DurationAndCount(pr *v1.PipelineRun, beforeCondition *apis.Co
}

metrics.Record(ctx, prDuration.M(duration.Seconds()))
metrics.Record(ctx, prCount.M(1))
metrics.Record(ctx, prTotal.M(1))

return nil
Expand Down Expand Up @@ -491,11 +439,9 @@ func (r *Recorder) RunningPipelineRuns(lister listers.PipelineRunLister) error {
if err != nil {
return err
}
metrics.Record(ctx, runningPRsWaitingOnPipelineResolutionCount.M(float64(prsWaitResolvingPipelineRef)))
metrics.Record(ctx, runningPRsWaitingOnPipelineResolution.M(float64(prsWaitResolvingPipelineRef)))
metrics.Record(ctx, runningPRsWaitingOnTaskResolutionCount.M(float64(trsWaitResolvingTaskRef)))
metrics.Record(ctx, runningPRsWaitingOnTaskResolution.M(float64(trsWaitResolvingTaskRef)))
metrics.Record(ctx, runningPRsCount.M(float64(runningPipelineRuns)))
metrics.Record(ctx, runningPRs.M(float64(runningPipelineRuns)))

return nil
}
Expand Down
7 changes: 1 addition & 6 deletions pkg/pipelinerunmetrics/metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -532,11 +532,9 @@ func TestRecordPipelineRunDurationCount(t *testing.T) {
metricstest.CheckStatsNotReported(t, "pipelinerun_duration_seconds")
}
if test.expectedCountTags != nil {
metricstest.CheckCountData(t, "pipelinerun_count", test.expectedCountTags, test.expectedCount)
delete(test.expectedCountTags, "reason")
metricstest.CheckCountData(t, "pipelinerun_total", test.expectedCountTags, test.expectedCount)
} else {
metricstest.CheckStatsNotReported(t, "pipelinerun_count")
metricstest.CheckStatsNotReported(t, "pipelinerun_total")
}
})
Expand Down Expand Up @@ -582,7 +580,6 @@ func TestRecordRunningPipelineRunsCount(t *testing.T) {
if err := metrics.RunningPipelineRuns(informer.Lister()); err != nil {
t.Errorf("RunningPipelineRuns: %v", err)
}
metricstest.CheckLastValueData(t, "running_pipelineruns_count", map[string]string{}, 1)
metricstest.CheckLastValueData(t, "running_pipelineruns", map[string]string{}, 1)
}

Expand Down Expand Up @@ -787,15 +784,13 @@ func TestRecordRunningPipelineRunsResolutionWaitCounts(t *testing.T) {
if err := metrics.RunningPipelineRuns(informer.Lister()); err != nil {
t.Errorf("RunningTaskRuns: %v", err)
}
metricstest.CheckLastValueData(t, "running_pipelineruns_waiting_on_pipeline_resolution_count", map[string]string{}, tc.prWaitCount)
metricstest.CheckLastValueData(t, "running_pipelineruns_waiting_on_task_resolution_count", map[string]string{}, tc.trWaitCount)
metricstest.CheckLastValueData(t, "running_pipelineruns_waiting_on_pipeline_resolution", map[string]string{}, tc.prWaitCount)
metricstest.CheckLastValueData(t, "running_pipelineruns_waiting_on_task_resolution", map[string]string{}, tc.trWaitCount)
}
}

func unregisterMetrics() {
metricstest.Unregister("pipelinerun_duration_seconds", "pipelinerun_count", "pipelinerun_total", "running_pipelineruns_waiting_on_pipeline_resolution_count", "running_pipelineruns_waiting_on_pipeline_resolution", "running_pipelineruns_waiting_on_task_resolution_count", "running_pipelineruns_waiting_on_task_resolution", "running_pipelineruns_count", "running_pipelineruns")
metricstest.Unregister("pipelinerun_duration_seconds", "pipelinerun_total", "running_pipelineruns_waiting_on_pipeline_resolution", "running_pipelineruns_waiting_on_task_resolution", "running_pipelineruns")

// Allow the recorder singleton to be recreated.
once = sync.Once{}
Expand Down
59 changes: 0 additions & 59 deletions pkg/taskrunmetrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,8 @@ var (

trDurationView *view.View
prTRDurationView *view.View
trCountView *view.View
trTotalView *view.View
runningTRsCountView *view.View
runningTRsView *view.View
runningTRsThrottledByQuotaCountView *view.View
runningTRsThrottledByNodeCountView *view.View
runningTRsThrottledByQuotaView *view.View
runningTRsThrottledByNodeView *view.View
runningTRsWaitingOnTaskResolutionCountView *view.View
Expand All @@ -78,30 +74,14 @@ var (
"The pipelinerun's taskrun execution time in seconds",
stats.UnitDimensionless)

trCount = stats.Float64("taskrun_count",
"number of taskruns",
stats.UnitDimensionless)

trTotal = stats.Float64("taskrun_total",
"Number of taskruns",
stats.UnitDimensionless)

runningTRsCount = stats.Float64("running_taskruns_count",
"Number of taskruns executing currently",
stats.UnitDimensionless)

runningTRs = stats.Float64("running_taskruns",
"Number of taskruns executing currently",
stats.UnitDimensionless)

runningTRsThrottledByQuotaCount = stats.Float64("running_taskruns_throttled_by_quota_count",
"Number of taskruns executing currently, but whose underlying Pods or Containers are suspended by k8s because of defined ResourceQuotas. Such suspensions can occur as part of initial scheduling of the Pod, or scheduling of any of the subsequent Container(s) in the Pod after the first Container is started",
stats.UnitDimensionless)

runningTRsThrottledByNodeCount = stats.Float64("running_taskruns_throttled_by_node_count",
"Number of taskruns executing currently, but whose underlying Pods or Containers are suspended by k8s because of Node level constraints. Such suspensions can occur as part of initial scheduling of the Pod, or scheduling of any of the subsequent Container(s) in the Pod after the first Container is started",
stats.UnitDimensionless)

runningTRsWaitingOnTaskResolutionCount = stats.Float64("running_taskruns_waiting_on_task_resolution_count",
"Number of taskruns executing currently that are waiting on resolution requests for their task references.",
stats.UnitDimensionless)
Expand Down Expand Up @@ -217,9 +197,7 @@ func viewRegister(cfg *config.Metrics) error {
}
}

trCountViewTags := []tag.Key{statusTag}
if cfg.CountWithReason {
trCountViewTags = append(trCountViewTags, reasonTag)
trunTag = append(trunTag, reasonTag)
}

Expand All @@ -236,39 +214,18 @@ func viewRegister(cfg *config.Metrics) error {
TagKeys: append([]tag.Key{statusTag, namespaceTag}, append(trunTag, prunTag...)...),
}

trCountView = &view.View{
Description: trCount.Description(),
Measure: trCount,
Aggregation: view.Count(),
TagKeys: trCountViewTags,
}
trTotalView = &view.View{
Description: trTotal.Description(),
Measure: trTotal,
Aggregation: view.Count(),
TagKeys: []tag.Key{statusTag},
}
runningTRsCountView = &view.View{
Description: runningTRsCount.Description(),
Measure: runningTRsCount,
Aggregation: view.LastValue(),
}

runningTRsView = &view.View{
Description: runningTRs.Description(),
Measure: runningTRs,
Aggregation: view.LastValue(),
}
runningTRsThrottledByQuotaCountView = &view.View{
Description: runningTRsThrottledByQuotaCount.Description(),
Measure: runningTRsThrottledByQuotaCount,
Aggregation: view.LastValue(),
}
runningTRsThrottledByNodeCountView = &view.View{
Description: runningTRsThrottledByNodeCount.Description(),
Measure: runningTRsThrottledByNodeCount,
Aggregation: view.LastValue(),
}
runningTRsWaitingOnTaskResolutionCountView = &view.View{
Description: runningTRsWaitingOnTaskResolutionCount.Description(),
Measure: runningTRsWaitingOnTaskResolutionCount,
Expand Down Expand Up @@ -300,12 +257,8 @@ func viewRegister(cfg *config.Metrics) error {
return view.Register(
trDurationView,
prTRDurationView,
trCountView,
trTotalView,
runningTRsCountView,
runningTRsView,
runningTRsThrottledByQuotaCountView,
runningTRsThrottledByNodeCountView,
runningTRsWaitingOnTaskResolutionCountView,
runningTRsThrottledByQuotaView,
runningTRsThrottledByNodeView,
Expand All @@ -317,12 +270,8 @@ func viewUnregister() {
view.Unregister(
trDurationView,
prTRDurationView,
trCountView,
trTotalView,
runningTRsCountView,
runningTRsView,
runningTRsThrottledByQuotaCountView,
runningTRsThrottledByNodeCountView,
runningTRsWaitingOnTaskResolutionCountView,
runningTRsThrottledByQuotaView,
runningTRsThrottledByNodeView,
Expand Down Expand Up @@ -467,7 +416,6 @@ func (r *Recorder) DurationAndCount(ctx context.Context, tr *v1.TaskRun, beforeC
}

metrics.Record(ctx, durationStat.M(duration.Seconds()))
metrics.Record(ctx, trCount.M(1))
metrics.Record(ctx, trTotal.M(1))

return nil
Expand All @@ -492,9 +440,7 @@ func (r *Recorder) RunningTaskRuns(ctx context.Context, lister listers.TaskRunLi

var runningTrs int
trsThrottledByQuota := map[string]int{}
trsThrottledByQuotaCount := 0
trsThrottledByNode := map[string]int{}
trsThrottledByNodeCount := 0
var trsWaitResolvingTaskRef int
for _, pr := range trs {
// initialize metrics with namespace tag to zero if unset; will then update as needed below
Expand All @@ -516,12 +462,10 @@ func (r *Recorder) RunningTaskRuns(ctx context.Context, lister listers.TaskRunLi
if succeedCondition != nil && succeedCondition.Status == corev1.ConditionUnknown {
switch succeedCondition.Reason {
case pod.ReasonExceededResourceQuota:
trsThrottledByQuotaCount++
cnt := trsThrottledByQuota[pr.Namespace]
cnt++
trsThrottledByQuota[pr.Namespace] = cnt
case pod.ReasonExceededNodeResources:
trsThrottledByNodeCount++
cnt := trsThrottledByNode[pr.Namespace]
cnt++
trsThrottledByNode[pr.Namespace] = cnt
Expand All @@ -535,11 +479,8 @@ func (r *Recorder) RunningTaskRuns(ctx context.Context, lister listers.TaskRunLi
if err != nil {
return err
}
metrics.Record(ctx, runningTRsCount.M(float64(runningTrs)))
metrics.Record(ctx, runningTRs.M(float64(runningTrs)))
metrics.Record(ctx, runningTRsWaitingOnTaskResolutionCount.M(float64(trsWaitResolvingTaskRef)))
metrics.Record(ctx, runningTRsThrottledByQuotaCount.M(float64(trsThrottledByQuotaCount)))
metrics.Record(ctx, runningTRsThrottledByNodeCount.M(float64(trsThrottledByNodeCount)))

for ns, cnt := range trsThrottledByQuota {
var mutators []tag.Mutator
Expand Down
Loading
Loading