Prevent myhoard starting more than one backup stream

myhoard/backup_stream.py

@@ -252,6 +252,10 @@ def running(self):
         yield
         self.stop()
 
+    def is_in_terminal_state(self) -> bool:
+        """Returns if the stream has finished operating, successfully or not"""
+        return bool(self.state.get("broken_info") or self.state.get("closed_info") or self.state.get("completed_info"))
+
     def activate(self) -> None:
         with self.lock:
             if self.mode != self.Mode.promoted:

myhoard/controller.py

@@ -802,11 +802,12 @@ def _check_binlog_apply_status(self) -> None:
             self.state_manager.update_state(promote_details=promote_details)
 
     def _create_new_backup_stream_if_requested_and_max_streams_not_exceeded(self):
-        # Only ever have two open backup streams. Uploading binlogs to more streams than that is
-        # unlikely to improve the system behavior. We'll create new backup stream once the latter
-        # one catches up with the first, the first is marked as closed, and removed from our list.
-        if len(self.backup_streams) >= 2:
-            return
+        # Only ever have one "active" backup stream.
+        # Having multiple backup streams adds complexity, increases node resource usage, and
+        # more easily triggers various latent bugs.
+        for existing_backup_stream in self.backup_streams:
+            if not existing_backup_stream.is_in_terminal_state():
+                return
         with self.lock:
             if self.state["backup_request"]:
                 request: BackupRequest = self.state["backup_request"]