Add step specific metrics

internal/consensus/metrics.go

@@ -173,6 +173,10 @@ type Metrics struct {
 
 	// ApplyBlockLatency measures how long it takes to execute ApplyBlock in finalize commit step
 	ApplyBlockLatency metrics.Histogram `metrics_buckettype:"exprange" metrics_bucketsizes:"0.01, 10, 10"`
+
+	StepLatency                 metrics.Gauge `metrics_labels:"step"`
+	lastRecordedStepLatencyNano int64
+	StepCount                   metrics.Gauge `metrics_labels:"step"`
 }
 
 // RecordConsMetrics uses for recording the block related metrics during fast-sync.
@@ -249,6 +253,29 @@ func (m *Metrics) MarkStep(s cstypes.RoundStepType) {
 		stepTime := time.Since(m.stepStart).Seconds()
 		stepName := strings.TrimPrefix(s.String(), "RoundStep")
 		m.StepDuration.With("step", stepName).Observe(stepTime)
+		m.StepCount.With("step", s.String()).Add(1)
 	}
 	m.stepStart = time.Now()
 }
+
+func (m *Metrics) MarkStepLatency(s cstypes.RoundStepType) {
+	now := time.Now().UnixNano()
+	m.StepLatency.With("step", s.String()).Add(float64(now - m.lastRecordedStepLatencyNano))
+	m.lastRecordedStepLatencyNano = now
+}
+
+func (m *Metrics) ClearStepMetrics() {
+	for _, st := range []cstypes.RoundStepType{
+		cstypes.RoundStepNewHeight,
+		cstypes.RoundStepNewRound,
+		cstypes.RoundStepPropose,
+		cstypes.RoundStepPrevote,
+		cstypes.RoundStepPrevoteWait,
+		cstypes.RoundStepPrecommit,
+		cstypes.RoundStepPrecommitWait,
+		cstypes.RoundStepCommit,
+	} {
+		m.StepCount.With("step", st.String()).Set(0)
+		m.StepLatency.With("step", st.String()).Set(0)
+	}
+}

internal/consensus/state.go

@@ -659,6 +659,7 @@ func (cs *State) SetProposalAndBlock(
 
 func (cs *State) updateHeight(height int64) {
 	cs.metrics.Height.Set(float64(height))
+	cs.metrics.ClearStepMetrics()
 	cs.roundState.SetHeight(height)
 }
 
@@ -1043,6 +1044,8 @@ func (cs *State) handleMsg(ctx context.Context, mi msgInfo, fsyncUponCompletion
 		err   error
 	)
 
+	cs.metrics.MarkStepLatency(cs.roundState.Step())
+
 	msg, peerID := mi.Msg, mi.PeerID
 
 	switch msg := msg.(type) {
@@ -1179,6 +1182,7 @@ func (cs *State) handleTimeout(
 	// the timeout will now cause a state transition
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
+	cs.metrics.MarkStepLatency(rs.Step)
 
 	switch ti.Step {
 	case cstypes.RoundStepNewHeight: