check namespace config for node throttle metric

Now that @vdemeester 's fix so that the background metric polling goroutines for pipelinerunmetrics and taskrunmetrics can access config.observability, we can properly address issue around verifiying non-default config around adding the namespace label for the node and quota throttle metrics that @divyansh42 @khrm and myself discovered last week.

Signed-off-by: gabemontero <gmontero@redhat.com>

rh-pre-commit.version: 2.3.1
rh-pre-commit.check-secrets: ENABLED

Author: gabemontero

pkg/taskrunmetrics/metrics.go

@@ -528,7 +528,11 @@ func (r *Recorder) RunningTaskRuns(ctx context.Context, lister listers.TaskRunLi
 		metrics.Record(ctx, runningTRsThrottledByQuota.M(float64(cnt)))
 	}
 	for ns, cnt := range trsThrottledByNode {
-		ctx, err := tag.New(ctx, []tag.Mutator{tag.Insert(namespaceTag, ns)}...)
+		var mutators []tag.Mutator
+		if addNamespaceLabelToQuotaThrottleMetric {
+			mutators = []tag.Mutator{tag.Insert(namespaceTag, ns)}
+		}
+		ctx, err := tag.New(ctx, mutators...)
 		if err != nil {
 			return err
 		}

pkg/taskrunmetrics/metrics_test.go

@@ -583,6 +583,18 @@ func TestRecordRunningTaskRunsThrottledCounts(t *testing.T) {
 			reason:    pod.ReasonExceededNodeResources,
 			nodeCount: 3,
 		},
+		{
+			status:     corev1.ConditionUnknown,
+			reason:     pod.ReasonExceededResourceQuota,
+			quotaCount: 3,
+			addNS:      true,
+		},
+		{
+			status:    corev1.ConditionUnknown,
+			reason:    pod.ReasonExceededNodeResources,
+			nodeCount: 3,
+			addNS:     true,
+		},
 		{
 			status:    corev1.ConditionUnknown,
 			reason:    v1.TaskRunReasonResolvingTaskRef,