From e91bd9e07f3196faba0edb644bd87a4a84ec5907 Mon Sep 17 00:00:00 2001 From: Terje Torkelsen Date: Wed, 9 Mar 2022 22:38:42 +0100 Subject: [PATCH 1/4] Support setting timestamp on metric --- config/opts.go | 1 + metrics/collector.go | 68 ++++++++++++++++++++++++++++++++++++++++++++ metrics/insights.go | 33 +++++++++++++-------- metrics/metrics.go | 23 +++++++++++++-- metrics/prober.go | 10 +++++-- 5 files changed, 118 insertions(+), 17 deletions(-) create mode 100644 metrics/collector.go diff --git a/config/opts.go b/config/opts.go index e224d40..f3f0453 100644 --- a/config/opts.go +++ b/config/opts.go @@ -26,6 +26,7 @@ type ( Metrics struct { ResourceIdLowercase bool `long:"metrics.resourceid.lowercase" env:"METRIC_RESOURCEID_LOWERCASE" description:"Publish lowercase Azure Resoruce ID in metrics"` + SetTimestamp bool `long:"metrics.set-timestamp" env:"METRIC_SET_TIMESTAMP" description:"Set timestamp on scraped metrics"` Template string `long:"metrics.template" env:"METRIC_TEMPLATE" description:"Template for metric name" default:"{name}"` Help string `long:"metrics.help" env:"METRIC_HELP" description:"Metric help (with template support)" default:"Azure monitor insight metric"` } diff --git a/metrics/collector.go b/metrics/collector.go new file mode 100644 index 0000000..1ff459f --- /dev/null +++ b/metrics/collector.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +type metricListCollectorDetails struct { + gauge prometheus.Gauge + desc *prometheus.Desc + ts time.Time +} + +type metricListCollector struct { + details []*metricListCollectorDetails +} + +func NewMetricListCollector(list *MetricList) *metricListCollector { + collector := &metricListCollector{ + details: []*metricListCollectorDetails{}, + } + + if list == nil { + return collector + } + + // create prometheus metrics and set rows + for _, metricName := range list.GetMetricNames() { + gaugeVec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: metricName, + Help: list.GetMetricHelp(metricName), + }, + list.GetMetricLabelNames(metricName), + ) + + latest := list.GetLatestMetric(metricName) + + gauge := gaugeVec.With(latest.Labels) + gauge.Set(latest.Value) + + desc := prometheus.NewDesc(metricName, list.GetMetricHelp(metricName), []string{}, latest.Labels) + + details := &metricListCollectorDetails{ + gauge, + desc, + latest.Timestamp, + } + + collector.details = append(collector.details, details) + } + + return collector +} + +func (c *metricListCollector) Describe(ch chan<- *prometheus.Desc) { + for _, detail := range c.details { + ch <- detail.desc + } +} + +func (c *metricListCollector) Collect(ch chan<- prometheus.Metric) { + for _, detail := range c.details { + s := prometheus.NewMetricWithTimestamp(detail.ts, detail.gauge) + ch <- s + } +} diff --git a/metrics/insights.go b/metrics/insights.go index 782a6b0..9c3241a 100644 --- a/metrics/insights.go +++ b/metrics/insights.go @@ -1,12 +1,14 @@ package metrics import ( + "regexp" + "strings" + "time" + "github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2018-03-01/insights" "github.com/Azure/go-autorest/autorest/azure" "github.com/Azure/go-autorest/autorest/to" "github.com/prometheus/client_golang/prometheus" - "regexp" - "strings" ) var ( @@ -26,10 +28,11 @@ type ( } PrometheusMetricResult struct { - Name string - Labels prometheus.Labels - Value float64 - Help string + Name string + Labels prometheus.Labels + Value float64 + Timestamp time.Time + Help string } ) @@ -70,7 +73,7 @@ func (p *MetricProber) FetchMetricsFromTarget(client *insights.MetricsClient, ta return ret, err } -func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value float64) (metric PrometheusMetricResult) { +func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value float64, timestamp time.Time) (metric PrometheusMetricResult) { // copy map to ensure we don't keep references metricLabels := prometheus.Labels{} for labelName, labelValue := range labels { @@ -78,9 +81,10 @@ func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value } metric = PrometheusMetricResult{ - Name: r.settings.MetricTemplate, - Labels: metricLabels, - Value: value, + Name: r.settings.MetricTemplate, + Labels: metricLabels, + Value: value, + Timestamp: timestamp, } // fallback if template is empty (should not be) @@ -140,8 +144,8 @@ func (r *AzureInsightMetricsResult) buildMetric(labels prometheus.Labels, value func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- PrometheusMetricResult) { if r.Result.Value != nil { // DEBUGGING - //data, _ := json.Marshal(r.Result) - //fmt.Println(string(data)) + // data, _ := json.Marshal(r.Result) + // fmt.Println(string(data)) for _, metric := range *r.Result.Value { if metric.Timeseries != nil { @@ -203,6 +207,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Total, + timeseriesData.TimeStamp.Time, ) } @@ -211,6 +216,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Minimum, + timeseriesData.TimeStamp.Time, ) } @@ -219,6 +225,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Maximum, + timeseriesData.TimeStamp.Time, ) } @@ -227,6 +234,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Average, + timeseriesData.TimeStamp.Time, ) } @@ -235,6 +243,7 @@ func (r *AzureInsightMetricsResult) SendMetricToChannel(channel chan<- Prometheu channel <- r.buildMetric( metricLabels, *timeseriesData.Count, + timeseriesData.TimeStamp.Time, ) } } diff --git a/metrics/metrics.go b/metrics/metrics.go index 1a32359..8eb95fd 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,6 +1,10 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) const ( MetricHelpDefault = "Azure monitor insight metric" @@ -13,8 +17,9 @@ type ( } MetricRow struct { - Labels prometheus.Labels - Value float64 + Labels prometheus.Labels + Value float64 + Timestamp time.Time } ) @@ -55,6 +60,18 @@ func (l *MetricList) GetMetricList(name string) []MetricRow { return l.List[name] } +func (l *MetricList) GetLatestMetric(name string) *MetricRow { + var latest *MetricRow + + for _, row := range l.List[name] { + if latest == nil || row.Timestamp.After(latest.Timestamp) { + latest = &row + } + } + + return latest +} + func (l *MetricList) GetMetricLabelNames(name string) []string { var list []string uniqueLabelMap := map[string]string{} diff --git a/metrics/prober.go b/metrics/prober.go index 8610f7c..426d067 100644 --- a/metrics/prober.go +++ b/metrics/prober.go @@ -211,8 +211,9 @@ func (p *MetricProber) collectMetricsFromTargets() { for result := range metricsChannel { metric := MetricRow{ - Labels: result.Labels, - Value: result.Value, + Labels: result.Labels, + Value: result.Value, + Timestamp: result.Timestamp, } p.metricList.Add(result.Name, metric) p.metricList.SetMetricHelp(result.Name, result.Help) @@ -220,6 +221,11 @@ func (p *MetricProber) collectMetricsFromTargets() { } func (p *MetricProber) publishMetricList() { + if p.Conf.Metrics.SetTimestamp { + p.prometheus.registry.MustRegister(NewMetricListCollector(p.metricList)) + return + } + if p.metricList == nil { return } From d9627d1f541a9574f7b1fb43d33ea4870ff2a785 Mon Sep 17 00:00:00 2001 From: Terje Torkelsen Date: Wed, 9 Mar 2022 22:48:45 +0100 Subject: [PATCH 2/4] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index eef3657..cb2e4cf 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ Application Options: --azure.servicediscovery.cache= Duration for caching Azure ServiceDiscovery of workspaces to reduce API calls (time.Duration) (default: 30m) [$AZURE_SERVICEDISCOVERY_CACHE] --metrics.resourceid.lowercase Publish lowercase Azure Resoruce ID in metrics [$METRIC_RESOURCEID_LOWERCASE] + --metrics.set-timestamp Set timestamp on scraped metrics [$METRIC_SET_TIMESTAMP] --metrics.template= Template for metric name (default: {name}) [$METRIC_TEMPLATE] --metrics.help= Metric help (with template support) (default: Azure monitor insight metric) [$METRIC_HELP] From 02906e4bd70ae35b397a7ff066182a5585d05699 Mon Sep 17 00:00:00 2001 From: Terje Torkelsen Date: Thu, 10 Mar 2022 08:19:16 +0100 Subject: [PATCH 3/4] Add all metrics --- metrics/collector.go | 20 ++++++++++---------- metrics/metrics.go | 12 ------------ 2 files changed, 10 insertions(+), 22 deletions(-) diff --git a/metrics/collector.go b/metrics/collector.go index 1ff459f..d3282bc 100644 --- a/metrics/collector.go +++ b/metrics/collector.go @@ -35,20 +35,20 @@ func NewMetricListCollector(list *MetricList) *metricListCollector { list.GetMetricLabelNames(metricName), ) - latest := list.GetLatestMetric(metricName) + for _, metric := range list.GetMetricList(metricName) { + gauge := gaugeVec.With(metric.Labels) + gauge.Set(metric.Value) - gauge := gaugeVec.With(latest.Labels) - gauge.Set(latest.Value) + desc := prometheus.NewDesc(metricName, list.GetMetricHelp(metricName), []string{}, metric.Labels) - desc := prometheus.NewDesc(metricName, list.GetMetricHelp(metricName), []string{}, latest.Labels) + details := &metricListCollectorDetails{ + gauge, + desc, + metric.Timestamp, + } - details := &metricListCollectorDetails{ - gauge, - desc, - latest.Timestamp, + collector.details = append(collector.details, details) } - - collector.details = append(collector.details, details) } return collector diff --git a/metrics/metrics.go b/metrics/metrics.go index 8eb95fd..7b4f221 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -60,18 +60,6 @@ func (l *MetricList) GetMetricList(name string) []MetricRow { return l.List[name] } -func (l *MetricList) GetLatestMetric(name string) *MetricRow { - var latest *MetricRow - - for _, row := range l.List[name] { - if latest == nil || row.Timestamp.After(latest.Timestamp) { - latest = &row - } - } - - return latest -} - func (l *MetricList) GetMetricLabelNames(name string) []string { var list []string uniqueLabelMap := map[string]string{} From 63d9f3e4defe3f91d94d99f534deb0c81673620e Mon Sep 17 00:00:00 2001 From: Terje Torkelsen Date: Thu, 10 Mar 2022 10:55:36 +0100 Subject: [PATCH 4/4] Fix problem with duplicate metric labels --- go.mod | 5 ++++- metrics/collector.go | 2 +- metrics/metrics.go | 25 +++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 01762d2..959a3c5 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,10 @@ require ( google.golang.org/protobuf v1.27.1 // indirect ) -require github.com/webdevops/go-prometheus-common v0.0.0-20220214222004-cea8f38b44b7 +require ( + github.com/google/go-cmp v0.5.5 + github.com/webdevops/go-prometheus-common v0.0.0-20220214222004-cea8f38b44b7 +) require ( github.com/Azure/go-autorest v14.2.0+incompatible // indirect diff --git a/metrics/collector.go b/metrics/collector.go index d3282bc..236674b 100644 --- a/metrics/collector.go +++ b/metrics/collector.go @@ -35,7 +35,7 @@ func NewMetricListCollector(list *MetricList) *metricListCollector { list.GetMetricLabelNames(metricName), ) - for _, metric := range list.GetMetricList(metricName) { + for _, metric := range list.GetUniqueMetricList(metricName) { gauge := gaugeVec.With(metric.Labels) gauge.Set(metric.Value) diff --git a/metrics/metrics.go b/metrics/metrics.go index 7b4f221..c5b4c7d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -3,6 +3,7 @@ package metrics import ( "time" + "github.com/google/go-cmp/cmp" "github.com/prometheus/client_golang/prometheus" ) @@ -60,6 +61,30 @@ func (l *MetricList) GetMetricList(name string) []MetricRow { return l.List[name] } +func (l *MetricList) GetUniqueMetricList(name string) []MetricRow { + rows := []MetricRow{} + + for _, row := range l.List[name] { + existed := false + + for idx, existedRow := range rows { + if cmp.Equal(row.Labels, existedRow.Labels) { + existed = true + + if row.Timestamp.After(existedRow.Timestamp) { + rows[idx] = row + } + } + } + + if !existed { + rows = append(rows, row) + } + } + + return rows +} + func (l *MetricList) GetMetricLabelNames(name string) []string { var list []string uniqueLabelMap := map[string]string{}