Merge pull request #149 from aiven/kmichel-lenient-rebuild

Rebuild tables without primary key or with invalid dates

#149

Author: packi

myhoard/restore_coordinator.py

@@ -450,6 +450,15 @@ def rebuild_tables(self) -> None:
         self._ensure_mysql_server_is_started(with_binlog=False, with_gtids=False)
         # Rebuilding very large tables can be slow
         with self._mysql_cursor(timeout=3600.0) as cursor:
+            cursor.execute("SELECT @@SESSION.sql_mode")
+            unwanted_sql_modes = {"STRICT_ALL_TABLES", " STRICT_TRANS_TABLES", "NO_ZERO_DATE", "NO_ZERO_IN_DATE"}
+            sql_mode = cursor.fetchone()["@@SESSION.sql_mode"]
+            lenient_sql_mode = ",".join([mode for mode in sql_mode.split(",") if mode not in unwanted_sql_modes])
+            if lenient_sql_mode != sql_mode:
+                self.log.info("Switching SQL mode from %r to %r during tables rebuild", sql_mode, lenient_sql_mode)
+                cursor.execute("SET SESSION sql_mode = %s", (lenient_sql_mode,))
+            self.log.info("Disabling requirement for primary keys during tables rebuild")
+            cursor.execute("SET SESSION sql_require_primary_key = false")
             cursor.execute(
                 "SELECT TABLE_SCHEMA,TABLE_NAME,TABLE_ROWS,AVG_ROW_LENGTH"
                 " FROM INFORMATION_SCHEMA.TABLES WHERE ENGINE='InnoDB'"

test/test_restore_coordinator.py

@@ -49,6 +49,10 @@ def _restore_coordinator_sequence(
         cursor.execute("CREATE TABLE t0 (id INTEGER PRIMARY KEY, data TEXT)")
         cursor.execute("CREATE TABLE t1 (id INTEGER PRIMARY KEY, data TEXT)")
         cursor.execute("CREATE TABLE t2 (id INTEGER PRIMARY KEY, data TEXT)")
+        cursor.execute("SET SESSION sql_require_primary_key = false")
+        cursor.execute("SET SESSION sql_mode = ''")
+        cursor.execute("CREATE TABLE bad0 (id INTEGER, data TEXT)")
+        cursor.execute("CREATE TABLE bad1 (id INTEGER, data DATETIME default '0000-00-00 00:00:00')")
         cursor.execute("COMMIT")
 
     private_key_pem, public_key_pem = generate_rsa_key_pair()
@@ -321,6 +325,8 @@ def stream_is_closed(stream):
         assert rc.state_manager.update_log.count({"last_rebuilt_table": "`db1`.`t0`"}) == 1
         assert rc.state_manager.update_log.count({"last_rebuilt_table": "`db1`.`t1`"}) == 2
         assert rc.state_manager.update_log.count({"last_rebuilt_table": "`db1`.`t2`"}) == 1
+        assert rc.state_manager.update_log.count({"last_rebuilt_table": "`db1`.`bad0`"}) == 1
+        assert rc.state_manager.update_log.count({"last_rebuilt_table": "`db1`.`bad1`"}) == 1
     else:
         assert rc.state["restore_errors"] == 0
     assert rc.state["remote_read_errors"] == 0