monitor: gracefully stop runners when image gets too old (#6)

Author: delan

monitor/monitor.toml.example

@@ -1,6 +1,9 @@
 # Prepend this to any internal URL in our own responses. Must end with trailing slash.
 external_base_url = "http://[::1]:8000/"
 
+# Maximum age of base images before they need to be rebuilt, in seconds.
+base_image_max_age = 86400
+
 [profiles.servo-windows10]
 configuration_name = "windows10"
 base_vm_name = "servo-windows10"

monitor/src/main.rs

@@ -413,6 +413,30 @@ fn monitor_thread() -> eyre::Result<()> {
             runner.log_info();
         }
 
+        // Determine whether any profiles need their images rebuilt.
+        for (key, profile) in profiles.iter() {
+            let needs_rebuild = profile.image_needs_rebuild();
+            if needs_rebuild.unwrap_or(true) {
+                let runner_count = profile.runners(&runners).count();
+                if needs_rebuild.is_none() {
+                    info!(
+                        key,
+                        runner_count, "profile image may or may not need rebuild"
+                    );
+                } else if runner_count > 0 {
+                    info!(
+                        key,
+                        runner_count, "profile image needs rebuild; waiting for runners"
+                    );
+                } else {
+                    info!(
+                        key,
+                        runner_count, "profile image needs rebuild; TODO start image rebuild"
+                    );
+                }
+            }
+        }
+
         let mut unregister_and_destroy = |id, runner: &Runner| {
             if runner.registration().is_some() {
                 if let Err(error) = runners.unregister_runner(id) {

monitor/src/profile.rs

@@ -5,14 +5,14 @@ use std::{
 
 use jane_eyre::eyre::{self, Context};
 use serde::{Deserialize, Serialize};
-use tracing::{debug, info};
+use tracing::{debug, info, warn};
 
 use crate::{
     data::get_profile_data_path,
     libvirt::update_screenshot,
     runner::{Runner, Runners, Status},
     zfs::snapshot_creation_time_unix,
-    DOTENV,
+    DOTENV, TOML,
 };
 
 #[derive(Clone, Debug, Deserialize)]
@@ -106,7 +106,7 @@ impl Profile {
     }
 
     pub fn target_runner_count(&self) -> usize {
-        if DOTENV.dont_create_runners {
+        if DOTENV.dont_create_runners || self.image_needs_rebuild().unwrap_or(true) {
             0
         } else {
             self.target_count
@@ -172,6 +172,28 @@ impl Profile {
         }
     }
 
+    /// Returns whether the image definitely needs to be rebuilt or not, or None
+    /// if we don’t know.
+    pub fn image_needs_rebuild(&self) -> Option<bool> {
+        if self.target_count == 0 {
+            // Profiles with zero target_count may have been set to zero because
+            // there is insufficient hugepages space to run them
+            return Some(false);
+        }
+
+        // If we fail to get the image age, err on the side of caution
+        let image_age = match self.image_age() {
+            Ok(result) => result,
+            Err(error) => {
+                warn!(?error, "Failed to get image age");
+                return None;
+            }
+        };
+
+        // If the profile has no image age, we may need to build its image for the first time
+        Some(image_age.map_or(true, |age| age > TOML.base_image_max_age()))
+    }
+
     pub fn image_age(&self) -> eyre::Result<Option<Duration>> {
         let now = SystemTime::now()
             .duration_since(UNIX_EPOCH)

monitor/src/settings.rs

@@ -39,6 +39,7 @@ pub struct Dotenv {
 #[derive(Deserialize)]
 pub struct Toml {
     pub external_base_url: String,
+    base_image_max_age: u64,
     profiles: BTreeMap<String, Profile>,
 }
 
@@ -90,6 +91,10 @@ impl Toml {
         Ok(result)
     }
 
+    pub fn base_image_max_age(&self) -> Duration {
+        Duration::from_secs(self.base_image_max_age)
+    }
+
     pub fn initial_profiles(&self) -> BTreeMap<String, Profile> {
         self.profiles.clone()
     }