Feat (CI): Dump GH CI Stats to GCP Metrics (#10338)

closes: #XXXX
refs: #XXXX

## Description

This PR adds a workflow job and a node script which captures Github CI stats on completion of CI workflows and dump them to GCP metrics.
Based on those metrics, we'll have dashboard on grafana.
This is a pre-req of migration from Datadog to GCP/Grafana.

**Successful CI link:**
https://github.com/Muneeb147/agoric-sdk/actions/runs/11550905067/job/32146827437

**Screenshot:**
<img width="1634" alt="Screenshot 2024-10-28 at 4 39 08 PM" src="https://github.com/user-attachments/assets/eee4ca18-0f8b-4376-a9e8-9d2c9e91710d">

**Demo Clip of Metrics**:

https://github.com/user-attachments/assets/8a18e485-f775-4e4d-a3fa-ea14db41811c




### Security Considerations


### Scaling Considerations


### Documentation Considerations


### Testing Considerations


### Upgrade Considerations

.github/actions/dump-ci-stats-to-gcp-metrics.cjs

@@ -0,0 +1,169 @@
+const Monitoring = require('@google-cloud/monitoring');
+
+const gcpCredentials = JSON.parse(process.env.GCP_CREDENTIALS);
+const monitoring = new Monitoring.MetricServiceClient({
+  projectId: gcpCredentials.project_id,
+  credentials: {
+    client_email: gcpCredentials.client_email,
+    private_key: gcpCredentials.private_key,
+  },
+});
+
+async function sendMetricsToGCP(metricType, metricValue, labels) {
+  const projectId = gcpCredentials.project_id;
+
+  const request = {
+    name: monitoring.projectPath(projectId),
+    timeSeries: [
+      {
+        metric: {
+          type: `custom.googleapis.com/github/${metricType}`,
+          labels: labels,
+        },
+        resource: {
+          type: 'global',
+          labels: {
+            project_id: projectId,
+          },
+        },
+        points: [
+          {
+            interval: {
+              endTime: {
+                seconds: Math.floor(Date.now() / 1000),
+              },
+            },
+            value: {
+              doubleValue: metricValue,
+            },
+          },
+        ],
+      },
+    ],
+  };
+  try {
+    await monitoring.createTimeSeries(request);
+    console.log(`Metric ${metricType} sent successfully.`);
+  } catch (error) {
+    console.error('Error sending metric:', error);
+  }
+}
+
+// Function to fetch workflow and job details via GitHub API
+async function fetchWorkflowDetails() {
+  const runId = process.argv[2];
+  const repo = process.env.GITHUB_REPOSITORY;
+  const apiUrl = `https://api.github.com/repos/${repo}/actions/runs/${runId}`;
+
+  try {
+    const response = await fetch(apiUrl, {
+      headers: {
+        Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
+        Accept: 'application/vnd.github.v3+json',
+      },
+    });
+
+    if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
+    const data = await response.json();
+
+    return {
+      workflowId: data.id,
+      workflowName: data.name,
+      status: data.status, // "completed", "in_progress", etc.
+      conclusion: data.conclusion, // "success", "failure"
+      startTime: data.created_at,
+      endTime: data.updated_at,
+      trigger: data.event, // "push", "pull_request", etc.
+      jobs: await fetchJobDetails(repo, data.id), // Fetch individual job details
+    };
+  } catch (error) {
+    console.error('Error fetching workflow details:', error);
+    process.exit(1);
+  }
+}
+
+async function fetchJobDetails(repo, runId) {
+  const apiUrl = `https://api.github.com/repos/${repo}/actions/runs/${runId}/jobs`;
+
+  try {
+    const response = await fetch(apiUrl, {
+      headers: {
+        Authorization: `Bearer ${process.env.GITHUB_TOKEN}`,
+        Accept: 'application/vnd.github.v3+json',
+      },
+    });
+
+    if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
+    const data = await response.json();
+    return data.jobs;
+  } catch (error) {
+    console.error('Error fetching job details:', error);
+    return [];
+  }
+}
+
+// Main function to send metrics
+(async () => {
+  try {
+    const workflowStats = await fetchWorkflowDetails();
+
+    const workflowLabels = {
+      workflow_name: workflowStats.workflowName,
+      workflow_id: workflowStats.workflowId,
+      trigger: workflowStats.trigger,
+    };
+
+    const workflowDuration =
+      (new Date(workflowStats.endTime) - new Date(workflowStats.startTime)) /
+      1000;
+    await sendMetricsToGCP(
+      'ci_workflow_duration',
+      workflowDuration,
+      workflowLabels,
+    );
+
+    for (const job of workflowStats.jobs) {
+      const jobLabels = {
+        workflow_name: workflowStats.workflowName,
+        job_name: job.name,
+        runner_name: job.runner_name,
+        conclusion: job.conclusion,
+      };
+
+      const jobExecutionTime =
+        (new Date(job.completed_at) - new Date(job.started_at)) / 1000;
+      await sendMetricsToGCP(
+        'ci_job_execution_time',
+        jobExecutionTime,
+        jobLabels,
+      );
+
+      // Send job status (1 for success, 0 for failure)
+      const jobStatus = job.conclusion === 'success' ? 1 : 0;
+      await sendMetricsToGCP('ci_job_status', jobStatus, jobLabels);
+
+      // Capture step-level metrics for step details per job
+      for (const step of job.steps) {
+        const stepExecutionTime =
+          (new Date(step.completed_at) - new Date(step.started_at)) / 1000;
+        const stepLabels = {
+          workflow_name: workflowStats.workflowName,
+          job_name: job.name,
+          step_name: step.name,
+          runner_name: job.runner_name,
+        };
+
+        await sendMetricsToGCP(
+          'ci_step_execution_time',
+          stepExecutionTime,
+          stepLabels,
+        );
+      }
+    }
+  } catch (error) {
+    console.error('Error in main function:', error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+})();

.github/workflows/dump-ci-stats.yml

@@ -0,0 +1,39 @@
+name: Dump GH CI Stats
+
+on:
+  workflow_run: # This allows the workflow to be reused
+    workflows:
+      [
+        'Integration Tests',
+        'Test Golang',
+        'golangci-lint',
+        'Build release Docker Images',
+        'Test all Packages',
+        'Test Documentation',
+        'Manage integration check',
+        'after-merge.yml',
+      ]
+    types:
+      - completed
+
+jobs:
+  dump_ci_stats:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v2
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18'
+
+      - name: Install GCP Monitoring/Metrics Client
+        run: yarn add @google-cloud/monitoring --ignore-workspace-root-check
+
+      - name: Run Final Job and Send Logs to GCP
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GCP_CREDENTIALS: ${{ secrets.GCP_CREDENTIALS }}
+        run: |
+          node .github/actions/dump-ci-stats-to-gcp-metrics.cjs ${{ github.event.workflow_run.id }}