feat: #207 metrics ingestion endpoint v0.2

Author: bohdan-shulha

README.md

@@ -16,6 +16,16 @@ Ptah.sh takes the pain out of deployment by easing common tasks used in many pro
 -   Load balancing of an incoming traffic and SSL auto-provisioning via Caddy Server.
 -   And many more features.
 
+## Components
+
+Ptah.sh is a collection of several interdependent services:
+
+-   [Ptah.sh Server](https://github.com/ptah-sh/ptah-server) - the core of the platform, responsible for managing the infrastructure, scaling, and load balancing.
+-   [Ptah.sh Agent](https://github.com/ptah-sh/ptah-agent) - the component installed on the target machine, responsible for running the containers and services.
+-   [Ptah.sh Caddy](https://github.com/ptah-sh/ptah-caddy) - the component installed on the target machine, responsible for running the Caddy Server and providing metrics to the Ptah.sh Server.
+-   [Ptah.sh GitHub Action](https://github.com/ptah-sh/deploy-action) - the component responsible for deploying the application to the target machine.
+-   [Ptah.sh Website](https://github.com/ptah-sh/ptah-sh.github.io) - the website of the Ptah.sh platform available at [ptah.sh](https://ptah.sh), containing the documentation, 1-Click Apps templates and the public-facing information.
+
 ## Ptah.sh Sponsors
 
 We would like to extend our thanks to the following sponsors for funding Ptah.sh development. If you are interested in becoming a sponsor, please send an e-mail to Bohdan Shulha via [contact@ptah.sh](mailto:contact@ptah.sh).

api-nodes/Http/Controllers/MetricsController.php

@@ -4,11 +4,11 @@
 
 use App\Models\DeploymentData\Process;
 use App\Models\Node;
+use App\Services\Metrics;
 use App\Util\Promexport;
 use Illuminate\Http\Request;
 use Illuminate\Http\Response;
 use Illuminate\Log\Logger;
-use Illuminate\Support\Facades\Http;
 
 class MetricsController
 {
@@ -97,10 +97,6 @@ public function __invoke(Request $request, Logger $log, Node $node)
                 continue;
             }
 
-            $query = '';
-            $query .= '?extra_label=swarm_id='.$node->swarm->id;
-            $query .= '&extra_label=node_id='.$node->id;
-
             $ingestMetrics = [];
 
             $lines = explode("\n", $metricsDoc);
@@ -192,9 +188,9 @@ public function __invoke(Request $request, Logger $log, Node $node)
                             $ingestMetrics[] = $line;
 
                             break;
-                        case 'ptah_node_disk_usage_free':
-                        case 'ptah_node_disk_usage_total':
-                        case 'ptah_node_disk_usage_used':
+                        case 'ptah_node_disk_free_bytes':
+                        case 'ptah_node_disk_total_bytes':
+                        case 'ptah_node_disk_used_bytes':
                             if (empty($labels['path'])) {
                                 break;
                             }
@@ -211,13 +207,16 @@ public function __invoke(Request $request, Logger $log, Node $node)
                         case 'ptah_node_cpu_system':
                         case 'ptah_node_cpu_total':
                         case 'ptah_node_cpu_user':
-                        case 'ptah_node_load_avg_1':
-                        case 'ptah_node_load_avg_5':
-                        case 'ptah_node_load_avg_15':
-                        case 'ptah_node_memory_free':
-                        case 'ptah_node_memory_total':
-                        case 'ptah_node_memory_used':
-                        case 'ptah_node_uptime':
+                        case 'ptah_node_load_avg_1m':
+                        case 'ptah_node_load_avg_5m':
+                        case 'ptah_node_load_avg_15m':
+                        case 'ptah_node_memory_total_bytes':
+                        case 'ptah_node_memory_used_bytes':
+                        case 'ptah_node_memory_free_bytes':
+                        case 'ptah_node_swap_total_bytes':
+                        case 'ptah_node_swap_used_bytes':
+                        case 'ptah_node_swap_free_bytes':
+                        case 'ptah_node_uptime_seconds':
                             $ingestMetrics[] = $line;
 
                             break;
@@ -229,17 +228,7 @@ public function __invoke(Request $request, Logger $log, Node $node)
                 }
             }
 
-            $log->info('Metrics:', [
-                'metrics' => $ingestMetrics,
-            ]);
-
-            // TODO: use value from config (/env vars)
-            $response = Http::withBody(implode("\n", $ingestMetrics), 'text/plain')->post("http://127.0.0.1:8080/api/v1/import/prometheus$query");
-
-            $log->info('VictoriaMetrics response:', [
-                'status' => $response->status(),
-                'body' => $response->body(),
-            ]);
+            $response = Metrics::importPrometheusMetrics($node->swarm->id, $node->id, $ingestMetrics);
         }
 
         $response = new Response('{}', 204);

app/Services/Metrics.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Http;
+
+// Multitenancy for single node: https://docs.victoriametrics.com/single-server-victoriametrics/#prometheus-querying-api-enhancements
+class Metrics
+{
+    public static function importPrometheusMetrics($swarmId, $nodeId, $metrics)
+    {
+        $query = '';
+        $query .= '?extra_label=swarm_id='.$swarmId;
+        $query .= '&extra_label=node_id='.$nodeId;
+
+        return Http::withBody(implode("\n", $metrics), 'text/plain')->post(config('database.victoriametrics.url').'/api/v1/import/prometheus'.$query);
+    }
+
+    public static function getMetrics($query, $nodeIds)
+    {
+        return Http::asForm()->post(config('database.victoriametrics.url').'/api/v1/query', [
+            'query' => $query,
+            'step' => '30s',
+            'extra_filters' => '{node_id=~"'.implode('|', $nodeIds).'"}',
+            'latency_offset' => '5s',
+        ])->json();
+    }
+
+    public static function getMetricsRange($query, $nodeIds)
+    {
+        return Http::asForm()->post(config('database.victoriametrics.url').'/api/v1/query_range', [
+            'query' => $query,
+            'start' => now()->subMinutes(5)->subSeconds(30)->getTimestampMs() / 1000,
+            'end' => now()->subSeconds(30)->getTimestampMs() / 1000,
+            'step' => '5s',
+            'extra_filters' => '{node_id=~"'.implode('|', $nodeIds).'"}',
+            'latency_offset' => '5s',
+        ])->json();
+    }
+}

config/database.php

@@ -170,4 +170,7 @@
 
     ],
 
+    'victoriametrics' => [
+        'url' => rtrim(env('VICTORIAMETRICS_URL', 'http://localhost:8428'), '/'),
+    ],
 ];