From a701c089fa2a345243985a765506a52b50e50963 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jun 2023 10:50:32 +0100 Subject: [PATCH 1/8] Fix schema delta error in 1.85 (#15738) There appears to be a race where you can end up with entries in `event_push_summary` with both a `NULL` and `main` thread ID. Fixes #15736 Introduced in #15597 --- changelog.d/15738.bugfix | 1 + .../main/delta/77/05thread_notifications_backfill.sql | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/15738.bugfix diff --git a/changelog.d/15738.bugfix b/changelog.d/15738.bugfix new file mode 100644 index 000000000000..7129ab078281 --- /dev/null +++ b/changelog.d/15738.bugfix @@ -0,0 +1 @@ +Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. diff --git a/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql b/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql index ce6f9ff93748..b09aa817aeb7 100644 --- a/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql +++ b/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql @@ -21,6 +21,14 @@ DELETE FROM background_updates WHERE update_name = 'event_push_backfill_thread_i -- Overwrite any null thread_id values. UPDATE event_push_actions_staging SET thread_id = 'main' WHERE thread_id IS NULL; UPDATE event_push_actions SET thread_id = 'main' WHERE thread_id IS NULL; + +-- Empirically we can end up with entries in the push summary table with both a +-- `NULL` and `main` thread ID, which causes the update below to fail. We fudge +-- this by deleting any `NULL` rows that have a corresponding `main`. +DELETE FROM event_push_summary AS a WHERE thread_id IS NULL AND EXISTS ( + SELECT 1 FROM event_push_summary AS b + WHERE b.thread_id = 'main' AND a.user_id = b.user_id AND a.room_id = b.room_id +); UPDATE event_push_summary SET thread_id = 'main' WHERE thread_id IS NULL; -- Drop the background updates to calculate the indexes used to find null thread_ids. From 7acf7f2f8df9726c961b392f21ee7a92d062fb39 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jun 2023 10:51:17 +0100 Subject: [PATCH 2/8] 1.85.1 --- CHANGES.md | 9 +++++++++ changelog.d/15738.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/15738.bugfix diff --git a/CHANGES.md b/CHANGES.md index ea13b554baa2..81bf3cc11091 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.85.1 (2023-06-07) +=========================== + +Bugfixes +-------- + +- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738)) + + Synapse 1.85.0 (2023-06-06) =========================== diff --git a/changelog.d/15738.bugfix b/changelog.d/15738.bugfix deleted file mode 100644 index 7129ab078281..000000000000 --- a/changelog.d/15738.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. diff --git a/debian/changelog b/debian/changelog index 2278a832837e..6d6f10ddf184 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.85.1) stable; urgency=medium + + * New Synapse release 1.85.1. + + -- Synapse Packaging team Wed, 07 Jun 2023 10:51:12 +0100 + matrix-synapse-py3 (1.85.0) stable; urgency=medium * New Synapse release 1.85.0. diff --git a/pyproject.toml b/pyproject.toml index 745b58d7b524..5b6123dff6de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml" [tool.poetry] name = "matrix-synapse" -version = "1.85.0" +version = "1.85.1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From f7c6553ebce51a46f1c78aa0a3fc6cc1effb346d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jun 2023 13:02:42 +0100 Subject: [PATCH 3/8] Fix schema delta error in 1.85 (#15739) Some users seem to have multiple rows per user / room with a null thread ID, which we need to handle. --- changelog.d/15739.bugfix | 1 + .../delta/77/05thread_notifications_backfill.sql | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 changelog.d/15739.bugfix diff --git a/changelog.d/15739.bugfix b/changelog.d/15739.bugfix new file mode 100644 index 000000000000..7129ab078281 --- /dev/null +++ b/changelog.d/15739.bugfix @@ -0,0 +1 @@ +Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. diff --git a/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql b/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql index b09aa817aeb7..a5da7a17a08f 100644 --- a/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql +++ b/synapse/storage/schema/main/delta/77/05thread_notifications_backfill.sql @@ -23,13 +23,25 @@ UPDATE event_push_actions_staging SET thread_id = 'main' WHERE thread_id IS NULL UPDATE event_push_actions SET thread_id = 'main' WHERE thread_id IS NULL; -- Empirically we can end up with entries in the push summary table with both a --- `NULL` and `main` thread ID, which causes the update below to fail. We fudge +-- `NULL` and `main` thread ID, which causes the insert below to fail. We fudge -- this by deleting any `NULL` rows that have a corresponding `main`. DELETE FROM event_push_summary AS a WHERE thread_id IS NULL AND EXISTS ( SELECT 1 FROM event_push_summary AS b WHERE b.thread_id = 'main' AND a.user_id = b.user_id AND a.room_id = b.room_id ); -UPDATE event_push_summary SET thread_id = 'main' WHERE thread_id IS NULL; +-- Copy the NULL threads to have a 'main' thread ID. +-- +-- Note: Some people seem to have duplicate rows with a `NULL` thread ID, in +-- which case we just fudge it with using MAX of the values. The counts *may* be +-- wrong for such rooms, but a) its an edge case, and b) they'll be fixed when +-- the user reads the room. +INSERT INTO event_push_summary (user_id, room_id, notif_count, stream_ordering, unread_count, last_receipt_stream_ordering, thread_id) + SELECT user_id, room_id, MAX(notif_count), MAX(stream_ordering), MAX(unread_count), MAX(last_receipt_stream_ordering), 'main' + FROM event_push_summary + WHERE thread_id IS NULL + GROUP BY user_id, room_id, thread_id; + +DELETE FROM event_push_summary AS a WHERE thread_id IS NULL; -- Drop the background updates to calculate the indexes used to find null thread_ids. DELETE FROM background_updates WHERE update_name = 'event_push_actions_thread_id_null'; From 28423977be8637bab096ed32085f06e715abe51b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jun 2023 13:04:20 +0100 Subject: [PATCH 4/8] Update changelog --- CHANGES.md | 2 +- changelog.d/15739.bugfix | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 changelog.d/15739.bugfix diff --git a/CHANGES.md b/CHANGES.md index 81bf3cc11091..a0f9235cac83 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Synapse 1.85.1 (2023-06-07) Bugfixes -------- -- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738)) +- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738), [\#15739](https://github.com/matrix-org/synapse/issues/15739)) Synapse 1.85.0 (2023-06-06) diff --git a/changelog.d/15739.bugfix b/changelog.d/15739.bugfix deleted file mode 100644 index 7129ab078281..000000000000 --- a/changelog.d/15739.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. From 6cd6a2ae59e718b0695774e7348097af2c27d973 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Jun 2023 13:07:40 +0100 Subject: [PATCH 5/8] Update changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a0f9235cac83..5babc22f2a54 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.85.1 (2023-06-07) =========================== +Note: this release only fixes a bug that stopped some deployments from upgrading to v1.85.0. There is no need to upgrade to v1.85.1 if successfully running v1.85.0. + Bugfixes -------- From 733342ad3ef271a2c5bd4ba442a15fa3be3dab30 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Jun 2023 13:03:48 +0100 Subject: [PATCH 6/8] Fix using TLS for replication (#15746) Fixes #15744. --- changelog.d/15746.bugfix | 1 + synapse/http/replicationagent.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/15746.bugfix diff --git a/changelog.d/15746.bugfix b/changelog.d/15746.bugfix new file mode 100644 index 000000000000..8d3e22f2e5ce --- /dev/null +++ b/changelog.d/15746.bugfix @@ -0,0 +1 @@ +Fix regression where using TLS for replication did not work. Introduced in v1.85.0. diff --git a/synapse/http/replicationagent.py b/synapse/http/replicationagent.py index 800f21873d66..d6ba6f0e577d 100644 --- a/synapse/http/replicationagent.py +++ b/synapse/http/replicationagent.py @@ -76,7 +76,7 @@ def endpointForURI(self, uri: URI) -> IStreamClientEndpoint: endpoint = wrapClientTLS( # The 'port' argument below isn't actually used by the function self.context_factory.creatorForNetloc( - self.instance_map[worker_name].host, + self.instance_map[worker_name].host.encode("utf-8"), self.instance_map[worker_name].port, ), endpoint, From a4921b23703776c9399433906b57c90fadb55bb6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Jun 2023 13:04:26 +0100 Subject: [PATCH 7/8] 1.85.2 --- CHANGES.md | 9 +++++++++ changelog.d/15746.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/15746.bugfix diff --git a/CHANGES.md b/CHANGES.md index 5babc22f2a54..f3eb0182f698 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.85.2 (2023-06-08) +=========================== + +Bugfixes +-------- + +- Fix regression where using TLS for replication did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746)) + + Synapse 1.85.1 (2023-06-07) =========================== diff --git a/changelog.d/15746.bugfix b/changelog.d/15746.bugfix deleted file mode 100644 index 8d3e22f2e5ce..000000000000 --- a/changelog.d/15746.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix regression where using TLS for replication did not work. Introduced in v1.85.0. diff --git a/debian/changelog b/debian/changelog index 6d6f10ddf184..a7503ea60a7a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.85.2) stable; urgency=medium + + * New Synapse release 1.85.2. + + -- Synapse Packaging team Thu, 08 Jun 2023 13:04:18 +0100 + matrix-synapse-py3 (1.85.1) stable; urgency=medium * New Synapse release 1.85.1. diff --git a/pyproject.toml b/pyproject.toml index 5b6123dff6de..02c9255f6ebb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml" [tool.poetry] name = "matrix-synapse" -version = "1.85.1" +version = "1.85.2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From ac3a70a7dd4070bf3953b8913f7c316d701db588 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Jun 2023 13:15:56 +0100 Subject: [PATCH 8/8] Fix up changelog --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index f3eb0182f698..893ceccaeaeb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Synapse 1.85.2 (2023-06-08) Bugfixes -------- -- Fix regression where using TLS for replication did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746)) +- Fix regression where using TLS for HTTP replication between workers did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746)) Synapse 1.85.1 (2023-06-07)