fix: fix metrics race condition (#1194)

## Which problem is this PR solving?

- Fixes race condition described in #1156, which was already fixed on
the 2.x branch.

## Short description of the changes

- Copy fix from that branch.

Fixes #1156.

metrics/otel_metrics.go

@@ -195,9 +195,15 @@ func (o *OTelMetrics) Register(name string, metricType string) {
 		o.counters[name] = ctr
 	case "gauge":
 		var f metric.Float64Callback = func(_ context.Context, result metric.Float64Observer) error {
-			o.lock.Lock()
-			defer o.lock.Unlock()
-			result.Observe(o.values[name])
+			// this callback is invoked from outside this function call, so we
+			// need to Rlock when we read the values map. We don't know how long
+			// Observe() takes, so we make a copy of the value and unlock before
+			// calling Observe.
+			o.lock.RLock()
+			v := o.values[name]
+			o.lock.RUnlock()
+
+			result.Observe(v)
 			return nil
 		}
 		g, err := o.meter.Float64ObservableGauge(name,