Merge pull request #163 from HileQAQ/prometheus

Add support to Prometheus

Author: liulanzheng

README.md

@@ -72,6 +72,8 @@ Accelerated Container Image is a __non-core__ sub-project of containerd.
 
 * See how to use overlaybd writable layer at [WRITABLE](docs/WRITABLE.md).
 
+* See how to use Prometheus to monitor metrics like latency/error count of snapshotter GRPC APIs at [PROMETHEUS](docs/PROMETHEUS.md).
+
 * Welcome to contribute! [CONTRIBUTING](docs/CONTRIBUTING.md)
 
 ## Overview

cmd/overlaybd-snapshotter/main.go

@@ -26,6 +26,7 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/containerd/accelerated-container-image/pkg/metrics"
 	overlaybd "github.com/containerd/accelerated-container-image/pkg/snapshot"
 
 	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
@@ -53,7 +54,6 @@ func parseConfig(fpath string) error {
 
 // TODO: use github.com/urfave/cli
 func main() {
-
 	pconfig = overlaybd.DefaultBootConfig()
 	if err := parseConfig(defaultConfigPath); err != nil {
 		logrus.Error(err)
@@ -62,7 +62,12 @@ func main() {
 	if pconfig.LogReportCaller {
 		logrus.SetReportCaller(true)
 	}
-	logrus.Infof("%+v", *pconfig)
+
+	metrics.Config = pconfig.ExporterConfig
+	if pconfig.ExporterConfig.Enable {
+		go metrics.Init()
+		logrus.Infof("set Prometheus metrics exporter in http://localhost:%d%s", metrics.Config.Port, metrics.Config.UriPrefix)
+	}
 
 	if err := setLogLevel(pconfig.LogLevel); err != nil {
 		logrus.Errorf("failed to set log level: %v", err)
@@ -117,6 +122,10 @@ func main() {
 	signal.Notify(signals, unix.SIGTERM, unix.SIGINT, unix.SIGPIPE)
 
 	<-handleSignals(context.TODO(), signals, srv)
+
+	if pconfig.ExporterConfig.Enable {
+		metrics.IsAlive.Set(0)
+	}
 }
 
 func handleSignals(ctx context.Context, signals chan os.Signal, server *grpc.Server) chan struct{} {

docs/PROMETHEUS.md

@@ -0,0 +1,67 @@
+# Collect Overlaybd metrics with Prometheus
+
+Prometheus is an open-source systems monitoring and alerting toolkit. You can configure overlaybd as a Prometheus target. This topic shows you how to configure overlaybd to use Prometheus.
+
+## Configure Overlaybd
+
+To configure overlaybd as a Prometheus target, you need to specify `uriPrefix` and `port` of the exporter, they can be set in `/etc/overlaybd-snapshotter/config.json` (overlaybd-snapshotter config path), this is an example.
+
+```json
+{
+    "root": "/var/lib/containerd/io.containerd.snapshotter.v1.overlaybd",
+    "address": "/run/overlaybd-snapshotter/overlaybd.sock",
+    "verbose": "info",
+    "rwMode": "overlayfs",
+    "logReportCaller": false,
+    "autoRemoveDev": false,
+    "exporterConfig": {
+        "enable": true,
+        "uriPrefix": "/metrics",
+        "port": 9863
+    }
+}
+```
+
+After configuration, you can run overlaybd-snapshotter and curl `http://localhost:9863/metrics` to get Prometheus metrics.
+
+## Configure Prometheus
+
+To use Prometheus to monitor overlaybd, you should modify the `scrape_configs`, and add a `job_name` for overlaybd in `.../prometheus/prometheus.yml`.
+
+```toml
+# my global config
+global:
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. The default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "prometheus"
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+      - targets: ["localhost:9090"]
+  - job_name: "overlaybd"
+    metrics_path: "/metrics"
+    static_configs:
+      - targets: ["localhost:9863"]
+```
+
+In this configuration, you set a job named `overlaybd`, and set `metrics_path` equal to `uriPrefix` in the overlaybd configuration. Next, you can start a Prometheus service using this configuration to monitor overlaybd.

docs/QUICKSTART.md

@@ -62,7 +62,12 @@ The config file is `/etc/overlaybd-snapshotter/config.json`. Please create the f
     "verbose": "info",
     "rwMode": "overlayfs",
     "logReportCaller": false,
-    "autoRemoveDev": false
+    "autoRemoveDev": false,
+    "exporterConfig": {
+        "enable": false,
+        "uriPrefix": "/metrics",
+        "port": 9863
+    }
 }
 ```
 | Field | Description |
@@ -73,6 +78,9 @@ The config file is `/etc/overlaybd-snapshotter/config.json`. Please create the f
 | `rwMode` | rootfs mode about wether to use native writable layer. See [Native Support for Writable](docs/WRITABLE.md) for detail. |
 | `logReportCaller` | enable/disable the calling method |
 | `autoRemoveDev` | enable/disable auto clean-up overlaybd device after container removed |
+| `exporterConfig.enable` | whether or not create a server to show Prometheus metrics |
+| `exporterConfig.uriPrefix` | URI prefix for export metrics, default `/metrics` |
+| `exporterConfig.port` | port for http server to show metrics, default `9863` |
 
 
 #### Start service

go.mod

@@ -7,22 +7,26 @@ require (
 	github.com/containerd/continuity v0.2.2
 	github.com/containerd/go-cni v1.1.5
 	github.com/go-sql-driver/mysql v1.6.0
+	github.com/moby/locker v1.0.1
 	github.com/moby/sys/mountinfo v0.5.0
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799
 	github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.14.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.1.3
 	github.com/urfave/cli v1.22.2
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
 	google.golang.org/grpc v1.43.0
 )
 
 require (
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/Microsoft/hcsshim v0.9.2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cilium/ebpf v0.7.0 // indirect
 	github.com/containerd/cgroups v1.0.3 // indirect
 	github.com/containerd/console v1.0.3 // indirect
@@ -41,25 +45,28 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
+	github.com/google/go-cmp v0.5.8 // indirect
 	github.com/google/uuid v1.2.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/moby/locker v1.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/moby/sys/signal v0.6.0 // indirect
 	github.com/moby/sys/symlink v0.2.0 // indirect
 	github.com/opencontainers/runc v1.1.1 // indirect
 	github.com/opencontainers/selinux v1.10.1 // indirect
 	github.com/pelletier/go-toml v1.9.3 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
 	github.com/russross/blackfriday/v2 v2.0.1 // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	golang.org/x/net v0.0.0-20211216030914-fe4d6282115f // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 	gotest.tools/v3 v3.0.3 // indirect
 )