add total suffix in counter metrics

Signed-off-by: yeya24 <[email protected]>

Author: yeya24

docs/monitoring/README.md

@@ -56,32 +56,34 @@ pytorch_operator_is_leader
 
 ### Report PyTorch Job metrics:
 
+*Note*: If you are using release v1 pytorch-operator, these PyTorch metrics don't have suffix `total`. So you have to use metric name like `pytorch_operator_jobs_created` to get your metrics. See [PR](https://github.com/kubeflow/tf-operator/pull/1055) to get more information.
+
 **Job Creation**
 ```
-pytorch_operator_jobs_created
+pytorch_operator_jobs_created_total
 ```
 
 **Job Creation**
 ```
-sum (rate (pytorch_operator_jobs_created[60m]))
+sum (rate (pytorch_operator_jobs_created_total[60m]))
 ```
 
 **Job Deletion**
 ```
-pytorch_operator_jobs_deleted
+pytorch_operator_jobs_deleted_total
 ```
 
 **Successful Job Completions**
 ```
-pytorch_operator_jobs_successful
+pytorch_operator_jobs_successful_total
 ```
 
 **Failed Jobs**
 ```
-pytorch_operator_jobs_failed
+pytorch_operator_jobs_failed_total
 ```
 
 **Restarted Jobs**
 ```
-pytorch_operator_jobs_restarted
+pytorch_operator_jobs_restarted_total
 ```

pkg/controller.v1/pytorch/controller.go

@@ -72,7 +72,7 @@ var (
 	}
 
 	pytorchJobsDeletedCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "pytorch_operator_jobs_deleted",
+		Name: "pytorch_operator_jobs_deleted_total",
 		Help: "Counts number of PyTorch jobs deleted",
 	})
 )

pkg/controller.v1/pytorch/job.go

@@ -25,7 +25,7 @@ const (
 
 var (
 	pytorchJobsCreatedCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "pytorch_operator_jobs_created",
+		Name: "pytorch_operator_jobs_created_total",
 		Help: "Counts number of PyTorch jobs created",
 	})
 )
@@ -49,7 +49,7 @@ func (pc *PyTorchController) addPyTorchJob(obj interface{}) {
 
 			status := common.JobStatus{
 				Conditions: []common.JobCondition{
-					common.JobCondition{
+					{
 						Type:               common.JobFailed,
 						Status:             v1.ConditionTrue,
 						LastUpdateTime:     metav1.Now(),

pkg/controller.v1/pytorch/status.go

@@ -46,15 +46,15 @@ const (
 
 var (
 	pytorchJobsSuccessCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "pytorch_operator_jobs_successful",
+		Name: "pytorch_operator_jobs_successful_total",
 		Help: "Counts number of PyTorch jobs successful",
 	})
 	pytorchJobsFailureCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "pytorch_operator_jobs_failed",
+		Name: "pytorch_operator_jobs_failed_total",
 		Help: "Counts number of PyTorch jobs failed",
 	})
 	pytorchJobsRestartCount = promauto.NewCounter(prometheus.CounterOpts{
-		Name: "pytorch_operator_jobs_restarted",
+		Name: "pytorch_operator_jobs_restarted_total",
 		Help: "Counts number of PyTorch jobs restarted",
 	})
 )