From 7a9725c5cec626cd6cafe1f0c75b22b52dc2c0c5 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 30 Jul 2025 15:41:31 -0600 Subject: [PATCH 001/181] WIP on clean shutdown of SynapseHomeServer class --- synapse/api/auth/base.py | 3 +- synapse/api/auth_blocking.py | 5 +- synapse/api/filtering.py | 5 +- synapse/app/_base.py | 3 +- synapse/app/phone_stats_home.py | 20 ++++--- synapse/crypto/keyring.py | 5 +- synapse/federation/federation_base.py | 4 +- synapse/federation/federation_client.py | 4 +- synapse/federation/federation_server.py | 2 +- synapse/federation/send_queue.py | 6 +- synapse/federation/sender/__init__.py | 19 +++---- synapse/federation/transport/client.py | 5 +- .../federation/transport/server/__init__.py | 9 +-- synapse/federation/transport/server/_base.py | 7 ++- synapse/handlers/account.py | 5 +- synapse/handlers/account_validity.py | 3 +- synapse/handlers/admin.py | 3 +- synapse/handlers/appservice.py | 5 +- synapse/handlers/auth.py | 5 +- synapse/handlers/cas.py | 3 +- synapse/handlers/deactivate_account.py | 3 +- synapse/handlers/device.py | 3 +- synapse/handlers/devicemessage.py | 7 ++- synapse/handlers/directory.py | 4 +- synapse/handlers/e2e_keys.py | 13 +++-- synapse/handlers/event_auth.py | 5 +- synapse/handlers/events.py | 3 +- synapse/handlers/federation.py | 9 ++- synapse/handlers/federation_event.py | 12 ++-- synapse/handlers/identity.py | 3 +- synapse/handlers/initial_sync.py | 3 +- synapse/handlers/jwt.py | 3 +- synapse/handlers/message.py | 3 +- synapse/handlers/pagination.py | 3 +- synapse/handlers/presence.py | 39 +++++++------ synapse/handlers/profile.py | 7 +-- synapse/handlers/receipts.py | 3 +- synapse/handlers/register.py | 3 +- synapse/handlers/reports.py | 3 +- synapse/handlers/room.py | 7 ++- synapse/handlers/room_list.py | 3 +- synapse/handlers/room_member.py | 5 +- synapse/handlers/room_policy.py | 3 +- synapse/handlers/search.py | 3 +- synapse/handlers/send_email.py | 3 +- synapse/handlers/sliding_sync/__init__.py | 1 - synapse/handlers/sliding_sync/room_lists.py | 11 ++-- synapse/handlers/sso.py | 5 +- synapse/handlers/stats.py | 6 +- synapse/handlers/typing.py | 34 +++++------- synapse/handlers/ui_auth/checkers.py | 5 +- synapse/handlers/user_directory.py | 16 +++--- synapse/http/client.py | 3 +- synapse/http/matrixfederationclient.py | 3 +- synapse/http/server.py | 6 +- synapse/media/media_repository.py | 5 +- synapse/media/media_storage.py | 5 +- synapse/media/storage_provider.py | 3 +- synapse/media/thumbnailer.py | 3 +- synapse/module_api/__init__.py | 5 +- synapse/notifier.py | 3 +- synapse/push/__init__.py | 3 +- synapse/push/bulk_push_rule_evaluator.py | 3 +- synapse/push/mailer.py | 3 +- synapse/push/pusher.py | 3 +- synapse/push/pusherpool.py | 3 +- synapse/replication/http/_base.py | 6 +- synapse/replication/tcp/client.py | 6 +- synapse/rest/admin/devices.py | 19 ++++--- synapse/rest/admin/experimental_features.py | 7 ++- synapse/rest/admin/media.py | 11 ++-- synapse/rest/admin/rooms.py | 15 ++--- synapse/rest/admin/server_notice_servlet.py | 5 +- synapse/rest/admin/users.py | 55 ++++++++++--------- synapse/rest/client/account.py | 21 +++---- synapse/rest/client/account_data.py | 9 +-- synapse/rest/client/account_validity.py | 5 +- synapse/rest/client/auth.py | 3 +- synapse/rest/client/capabilities.py | 3 +- synapse/rest/client/devices.py | 13 +++-- synapse/rest/client/filter.py | 5 +- synapse/rest/client/keys.py | 3 +- synapse/rest/client/login.py | 3 +- synapse/rest/client/media.py | 9 +-- synapse/rest/client/presence.py | 3 +- synapse/rest/client/profile.py | 5 +- synapse/rest/client/pusher.py | 5 +- synapse/rest/client/register.py | 13 +++-- synapse/rest/client/reporting.py | 7 ++- synapse/rest/client/room.py | 9 +-- .../rest/client/room_upgrade_rest_servlet.py | 3 +- synapse/rest/client/sendtodevice.py | 3 +- synapse/rest/client/sync.py | 7 ++- synapse/rest/client/thread_subscriptions.py | 1 - synapse/rest/client/transactions.py | 3 +- synapse/rest/client/user_directory.py | 3 +- synapse/rest/client/voip.py | 3 +- synapse/rest/consent/consent_resource.py | 3 +- synapse/rest/media/download_resource.py | 5 +- synapse/rest/media/thumbnail_resource.py | 5 +- synapse/server.py | 26 +++++++++ .../server_notices/server_notices_manager.py | 7 ++- synapse/state/__init__.py | 3 +- synapse/storage/_base.py | 2 +- synapse/storage/background_updates.py | 3 +- synapse/storage/controllers/persist_events.py | 12 ++-- synapse/storage/controllers/state.py | 9 +-- synapse/storage/database.py | 3 +- synapse/storage/databases/main/__init__.py | 6 +- .../databases/main/event_federation.py | 3 +- .../databases/main/event_push_actions.py | 16 +++++- synapse/storage/databases/main/events.py | 16 +++--- synapse/storage/databases/main/metrics.py | 5 ++ .../databases/main/monthly_active_users.py | 3 +- synapse/storage/databases/main/presence.py | 3 +- synapse/storage/databases/main/roommember.py | 18 +++++- synapse/util/task_scheduler.py | 3 +- tests/handlers/test_appservice.py | 3 +- tests/handlers/test_room_policy.py | 3 +- tests/handlers/test_room_summary.py | 5 +- .../databases/main/test_events_worker.py | 3 +- 121 files changed, 498 insertions(+), 333 deletions(-) diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index f97a71caf7c..51cddd8fb03 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -19,6 +19,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, Optional, Tuple from netaddr import IPAddress @@ -52,7 +53,7 @@ class BaseAuth: """Common base class for all auth implementations.""" def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index 303c9ba03e3..5734416a6f4 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Optional from synapse.api.constants import LimitBlockingTypes, UserTypes @@ -46,7 +47,7 @@ def __init__(self, hs: "HomeServer"): self._mau_limits_reserved_threepids = ( hs.config.server.mau_limits_reserved_threepids ) - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips async def check_auth_blocking( @@ -84,7 +85,7 @@ async def check_auth_blocking( if requester: if requester.authenticated_entity.startswith("@"): user_id = requester.authenticated_entity - elif self._is_mine_server_name(requester.authenticated_entity): + elif self.hs._is_mine_server_name(requester.authenticated_entity): # We never block the server from doing actions on behalf of # users. return diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 34dd12368a0..1dcce813838 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -21,6 +21,7 @@ # # import json +import weakref from typing import ( TYPE_CHECKING, Awaitable, @@ -152,7 +153,7 @@ def matrix_user_id_validator(user_id: object) -> bool: class Filtering: def __init__(self, hs: "HomeServer"): - self._hs = hs + self._hs = weakref.proxy(hs) self.store = hs.get_datastores().main self.DEFAULT_FILTER_COLLECTION = FilterCollection(hs, {}) @@ -320,7 +321,7 @@ def blocks_all_room_timeline(self) -> bool: class Filter: def __init__(self, hs: "HomeServer", filter_json: JsonMapping): - self._hs = hs + self._hs = weakref.proxy(hs) self._store = hs.get_datastores().main self.filter_json = filter_json diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 16aab93cd6c..89eb5d0e23a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -40,6 +40,7 @@ Tuple, cast, ) +import weakref from cryptography.utils import CryptographyDeprecationWarning from typing_extensions import ParamSpec @@ -551,7 +552,7 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: signal.signal(signal.SIGHUP, run_sighup) - register_sighup(refresh_certificate, hs) + register_sighup(refresh_certificate, weakref.proxy(hs)) register_sighup(reload_cache_config, hs.config) # Apply the cache config. diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 7e8c7cf37ea..c6935f5089c 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -198,9 +198,11 @@ def performance_stats_init() -> None: # Rather than update on per session basis, batch up the requests. # If you increase the loop period, the accuracy of user_daily_visits # table will decrease - clock.looping_call( - hs.get_datastores().main.generate_user_daily_visits, - 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, + hs.register_looping_call( + clock.looping_call( + hs.get_datastores().main.generate_user_daily_visits, + 5 * MILLISECONDS_PER_SECOND, + ) ) # monthly active user limiting functionality @@ -237,11 +239,13 @@ async def generate_monthly_active_users() -> None: if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") - clock.looping_call( - phone_stats_home, - PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, - hs, - stats, + hs.register_looping_call( + clock.looping_call( + phone_stats_home, + PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, + hs, + stats, + ) ) # We need to defer this init for the cases that we daemonize diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 643d2d4e66d..41f327f4b05 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -21,6 +21,7 @@ import abc import logging +import weakref from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple import attr @@ -175,7 +176,7 @@ def __init__( process_batch_callback=self._inner_fetch_key_requests, ) - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) # build a FetchKeyResult for each of our own keys, to shortcircuit the # fetcher. @@ -279,7 +280,7 @@ async def process_request(self, verify_request: VerifyJsonRequest) -> None: # If we are the originating server, short-circuit the key-fetch for any keys # we already have - if self._is_mine_server_name(verify_request.server_name): + if self.hs._is_mine_server_name(verify_request.server_name): for key_id in verify_request.key_ids: if key_id in self._local_verify_keys: found_keys[key_id] = self._local_verify_keys[key_id] diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 8d1e156dab7..a95379f4fd9 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -20,6 +20,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional, Sequence from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership @@ -56,9 +57,8 @@ def __init__(self, message: str, event_id: str): class FederationBase: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) - self._is_mine_server_name = hs.is_mine_server_name self.keyring = hs.get_keyring() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self.store = hs.get_datastores().main diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 35c5ac63119..78f912e4d4e 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -926,7 +926,7 @@ async def _try_destination_list( for destination in destinations: # We don't want to ask our own server for information we don't have - if self._is_mine_server_name(destination): + if self.hs._is_mine_server_name(destination): continue try: @@ -1606,7 +1606,7 @@ async def forward_third_party_invite( self, destinations: Iterable[str], room_id: str, event_dict: JsonDict ) -> None: for destination in destinations: - if self._is_mine_server_name(destination): + if self.hs._is_mine_server_name(destination): continue try: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3e6b8b84936..6d70f277cbd 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -980,7 +980,7 @@ async def _on_send_membership_event( authorising_server = get_domain_from_id( event.content[EventContentFields.AUTHORISING_USER] ) - if not self._is_mine_server_name(authorising_server): + if not self.hs._is_mine_server_name(authorising_server): raise SynapseError( 400, f"Cannot authorise membership event for {authorising_server}. We can only authorise requests from our own homeserver", diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index e309836a526..d004f63b024 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -37,6 +37,7 @@ """ import logging +import weakref from typing import ( TYPE_CHECKING, Dict, @@ -74,8 +75,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.notifier = hs.get_notifier() - self.is_mine_id = hs.is_mine_id - self.is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) # We may have multiple federation sender instances, so we need to track # their positions separately. @@ -208,7 +208,7 @@ def build_and_send_edu( key: Optional[Hashable] = None, ) -> None: """As per FederationSender""" - if self.is_mine_server_name(destination): + if self.hs.is_mine_server_name(destination): logger.info("Not sending EDU to ourselves") return diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 8010cc62f3a..8b9dced7493 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -131,6 +131,7 @@ import abc import logging +import weakref from collections import OrderedDict from typing import ( TYPE_CHECKING, @@ -369,7 +370,7 @@ async def _handle(self) -> None: class FederationSender(AbstractFederationSender): def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.server_name = hs.hostname self.store = hs.get_datastores().main @@ -378,8 +379,6 @@ def __init__(self, hs: "HomeServer"): self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() - self.is_mine_id = hs.is_mine_id - self.is_mine_server_name = hs.is_mine_server_name self._presence_router: Optional["PresenceRouter"] = None self._transaction_manager = TransactionManager(hs) @@ -512,7 +511,7 @@ async def _process_event_queue_loop(self) -> None: async def handle_event(event: EventBase) -> None: # Only send events for this server. send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of() - is_mine = self.is_mine_id(event.sender) + is_mine = self.hs.is_mine_id(event.sender) if not is_mine and send_on_behalf_of is None: logger.debug("Not sending remote-origin event %s", event) return @@ -824,7 +823,7 @@ async def send_read_receipt(self, receipt: ReadReceipt) -> None: domains: StrCollection = [ d for d in domains_set - if not self.is_mine_server_name(d) + if not self.hs.is_mine_server_name(d) and self._federation_shard_config.should_handle(self._instance_name, d) ] @@ -901,7 +900,7 @@ async def send_presence_to_destinations( # Ensure we only send out presence states for local users. for state in states: - assert self.is_mine_id(state.user_id) + assert self.hs.is_mine_id(state.user_id) destinations = await filter_destinations_by_retry_limiter( [ @@ -915,7 +914,7 @@ async def send_presence_to_destinations( ) for destination in destinations: - if self.is_mine_server_name(destination): + if self.hs.is_mine_server_name(destination): continue queue = self._get_per_destination_queue(destination) @@ -940,7 +939,7 @@ def build_and_send_edu( content: content of EDU key: clobbering key for this edu """ - if self.is_mine_server_name(destination): + if self.hs.is_mine_server_name(destination): logger.info("Not sending EDU to ourselves") return @@ -988,7 +987,7 @@ async def send_device_messages( if self._federation_shard_config.should_handle( self._instance_name, destination ) - and not self.is_mine_server_name(destination) + and not self.hs.is_mine_server_name(destination) ], clock=self.clock, store=self.store, @@ -1015,7 +1014,7 @@ def wake_destination(self, destination: str) -> None: might have come back. """ - if self.is_mine_server_name(destination): + if self.hs.is_mine_server_name(destination): logger.warning("Not waking up ourselves") return diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 62bf96ce913..8c73a8cb397 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -22,6 +22,7 @@ import logging import urllib +import weakref from typing import ( TYPE_CHECKING, Any, @@ -68,7 +69,7 @@ class TransportLayerClient: def __init__(self, hs: "HomeServer"): self.client = hs.get_federation_http_client() - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) async def get_room_state_ids( self, destination: str, room_id: str, event_id: str @@ -270,7 +271,7 @@ async def send_transaction( transaction.transaction_id, ) - if self._is_mine_server_name(transaction.destination): + if self.hs._is_mine_server_name(transaction.destination): raise RuntimeError("Transport layer cannot send to itself!") # FIXME: This is only used by the tests. The actual json sent is diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index 174d02ab6bb..a43367b9518 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -20,6 +20,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Tuple, Type from synapse.api.errors import FederationDeniedError, SynapseError @@ -63,7 +64,7 @@ def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None) servlet_groups: List of servlet groups to register. Defaults to ``DEFAULT_SERVLET_GROUPS``. """ - self.hs = hs + self.hs = weakref.proxy(hs) self.clock = hs.get_clock() self.servlet_groups = servlet_groups @@ -72,11 +73,11 @@ def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None) self.authenticator = Authenticator(hs) self.ratelimiter = hs.get_federation_ratelimiter() - self.register_servlets() + self.register_servlets(hs) - def register_servlets(self) -> None: + def register_servlets(self, hs: "HomeServer") -> None: register_servlets( - self.hs, + hs=hs, resource=self, ratelimiter=self.ratelimiter, authenticator=self.authenticator, diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index cba309635b7..667103318b5 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -23,6 +23,7 @@ import logging import re import time +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast @@ -64,7 +65,7 @@ def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self.keyring = hs.get_keyring() self.server_name = hs.hostname - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self.federation_domain_whitelist = ( hs.config.federation.federation_domain_whitelist @@ -108,7 +109,7 @@ async def authenticate_request( json_request["signatures"].setdefault(origin, {})[key] = sig # if the origin_server sent a destination along it needs to match our own server_name - if destination is not None and not self._is_mine_server_name( + if destination is not None and not self.hs._is_mine_server_name( destination ): raise AuthenticationError( @@ -277,7 +278,7 @@ def __init__( ratelimiter: FederationRateLimiter, server_name: str, ): - self.hs = hs + self.hs = weakref.proxy(hs) self.authenticator = authenticator self.ratelimiter = ratelimiter self.server_name = server_name diff --git a/synapse/handlers/account.py b/synapse/handlers/account.py index 37cc3d3ff56..72eda9d8675 100644 --- a/synapse/handlers/account.py +++ b/synapse/handlers/account.py @@ -19,6 +19,7 @@ # # +import weakref from typing import TYPE_CHECKING, Dict, List, Tuple from synapse.api.errors import Codes, SynapseError @@ -31,7 +32,7 @@ class AccountHandler: def __init__(self, hs: "HomeServer"): self._main_store = hs.get_datastores().main - self._is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self._federation_client = hs.get_federation_client() self._use_account_validity_in_account_status = ( hs.config.server.use_account_validity_in_account_status @@ -75,7 +76,7 @@ async def get_account_statuses( Codes.INVALID_PARAM, ) - if self._is_mine(user_id): + if self.hs._is_mine(user_id): status = await self._get_local_account_status(user_id) statuses[user_id.to_string()] = status else: diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 7004d95a0f3..055ff2b60aa 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -21,6 +21,7 @@ import email.mime.multipart import email.utils import logging +import weakref from typing import TYPE_CHECKING, List, Optional, Tuple from synapse.api.errors import AuthError, StoreError, SynapseError @@ -37,7 +38,7 @@ class AccountValidityHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.config = hs.config self.store = hs.get_datastores().main self.send_email_handler = hs.get_send_email_handler() diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index e90d675b59c..794fbbf865d 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -21,6 +21,7 @@ import abc import logging +import weakref from typing import ( TYPE_CHECKING, Any, @@ -73,7 +74,7 @@ def __init__(self, hs: "HomeServer"): self._redact_all_events, REDACT_ALL_EVENTS_ACTION_NAME ) - self.hs = hs + self.hs = weakref.proxy(hs) async def get_redact_task(self, redact_id: str) -> Optional[ScheduledTask]: """Get the current status of an active redaction process diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 93224d0c1bd..4b5a94f1232 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,6 +19,7 @@ # # import logging +import weakref from typing import ( TYPE_CHECKING, Collection, @@ -75,7 +76,7 @@ class ApplicationServicesHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.store = hs.get_datastores().main - self.is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) self.appservice_api = hs.get_application_service_api() self.scheduler = hs.get_application_service_scheduler() self.started_scheduler = False @@ -836,7 +837,7 @@ def _get_services_for_3pn(self, protocol: str) -> List[ApplicationService]: return [s for s in services if s.is_interested_in_protocol(protocol)] async def _is_unknown_user(self, user_id: str) -> bool: - if not self.is_mine_id(user_id): + if not self.hs.is_mine_id(user_id): # we don't know if they are unknown or not since it isn't one of our # users. We can't poke ASes. return False diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 257453674cf..9d6ff12a194 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -24,6 +24,7 @@ import time import unicodedata import urllib.parse +import weakref from binascii import crc32 from http import HTTPStatus from typing import ( @@ -213,7 +214,9 @@ def __init__(self, hs: "HomeServer"): self.password_auth_provider = hs.get_password_auth_provider() - self.hs = hs # FIXME better possibility to access registrationHandler later? + self.hs = weakref.proxy( + hs + ) # FIXME better possibility to access registrationHandler later? self.macaroon_gen = hs.get_macaroon_generator() self._password_enabled_for_login = hs.config.auth.password_enabled_for_login self._password_enabled_for_reauth = hs.config.auth.password_enabled_for_reauth diff --git a/synapse/handlers/cas.py b/synapse/handlers/cas.py index fbe79c2e4c3..c8c1ff68cc3 100644 --- a/synapse/handlers/cas.py +++ b/synapse/handlers/cas.py @@ -20,6 +20,7 @@ # import logging import urllib.parse +import weakref from typing import TYPE_CHECKING, Dict, List, Optional from xml.etree import ElementTree as ET @@ -66,7 +67,7 @@ class CasHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self._hostname = hs.hostname self._store = hs.get_datastores().main self._auth_handler = hs.get_auth_handler() diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 305363892fa..4868b70fcbf 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -20,6 +20,7 @@ # import itertools import logging +import weakref from typing import TYPE_CHECKING, Optional from synapse.api.constants import Membership @@ -38,7 +39,7 @@ class DeactivateAccountHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main - self.hs = hs + self.hs = weakref.proxy(hs) self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 80d49fc18df..7e416c6daa5 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -21,6 +21,7 @@ # import logging import random +import weakref from threading import Lock from typing import ( TYPE_CHECKING, @@ -125,7 +126,7 @@ class DeviceHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func - self.hs = hs + self.hs = weakref.proxy(hs) self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index b43cbd9c154..a79f557fe3f 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -20,6 +20,7 @@ # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Any, Dict, Optional @@ -52,7 +53,7 @@ def __init__(self, hs: "HomeServer"): """ self.store = hs.get_datastores().main self.notifier = hs.get_notifier() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self.device_handler = hs.get_device_handler() if hs.config.experimental.msc3814_enabled: self.event_sources = hs.get_event_sources() @@ -108,7 +109,7 @@ async def on_direct_to_device_edu(self, origin: str, content: JsonDict) -> None: message_id = content["message_id"] for user_id, by_device in content["messages"].items(): # we use UserID.from_string to catch invalid user ids - if not self.is_mine(UserID.from_string(user_id)): + if not self.hs.is_mine(UserID.from_string(user_id)): logger.warning("To-device message to non-local user %s", user_id) raise SynapseError(400, "Not a user here") @@ -262,7 +263,7 @@ async def send_device_message( continue # we use UserID.from_string to catch invalid user ids - if self.is_mine(UserID.from_string(user_id)): + if self.hs.is_mine(UserID.from_string(user_id)): messages_by_device = { device_id: { "content": message_content, diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 11284ccd0bc..f43c4ab24f4 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -21,6 +21,7 @@ import logging import string +import weakref from typing import TYPE_CHECKING, Iterable, List, Literal, Optional, Sequence from synapse.api.constants import MAX_ALIAS_LENGTH, EventTypes @@ -47,8 +48,9 @@ class DirectoryHandler: def __init__(self, hs: "HomeServer"): + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() - self.hs = hs + self.hostname = hs.hostname self.state = hs.get_state_handler() self.appservice_handler = hs.get_application_service_handler() self.event_creation_handler = hs.get_event_creation_handler() diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index b9abad21881..c70a2dcddc4 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -20,6 +20,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Tuple import attr @@ -68,7 +69,7 @@ def __init__(self, hs: "HomeServer"): self.federation = hs.get_federation_client() self.device_handler = hs.get_device_handler() self._appservice_handler = hs.get_application_service_handler() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self.clock = hs.get_clock() self._worker_lock_handler = hs.get_worker_locks_handler() self._task_scheduler = hs.get_task_scheduler() @@ -201,7 +202,7 @@ async def filter_device_key_query( for user_id, device_ids in device_keys_query.items(): # we use UserID.from_string to catch invalid user ids - if self.is_mine(UserID.from_string(user_id)): + if self.hs.is_mine(UserID.from_string(user_id)): local_query[user_id] = device_ids else: remote_queries[user_id] = device_ids @@ -549,7 +550,7 @@ async def query_local_devices( result_dict: Dict[str, Dict[str, dict]] = {} for user_id, device_ids in query.items(): # we use UserID.from_string to catch invalid user ids - if not self.is_mine(UserID.from_string(user_id)): + if not self.hs.is_mine(UserID.from_string(user_id)): logger.warning("Request for keys for non-local user %s", user_id) log_kv( { @@ -620,7 +621,7 @@ async def on_federation_query_client_keys( "device_keys", {} ) if any( - not self.is_mine(UserID.from_string(user_id)) + not self.hs.is_mine(UserID.from_string(user_id)) for user_id in device_keys_query ): raise SynapseError(400, "User is not hosted on this homeserver") @@ -761,7 +762,7 @@ async def claim_one_time_keys( for user_id, one_time_keys in query.items(): # we use UserID.from_string to catch invalid user ids - if self.is_mine(UserID.from_string(user_id)): + if self.hs.is_mine(UserID.from_string(user_id)): for device_id, algorithms in one_time_keys.items(): for algorithm, count in algorithms.items(): local_query.append((user_id, device_id, algorithm, count)) @@ -1431,7 +1432,7 @@ async def _get_e2e_cross_signing_verify_key( # cross-sign a remote user, but does not share any rooms with them yet. # Thus, we would not have their key list yet. We instead fetch the key, # store it and notify clients of new, associated device IDs. - if self.is_mine(user) or key_type not in ["master", "self_signing"]: + if self.hs.is_mine(user) or key_type not in ["master", "self_signing"]: # Note that master and self_signing keys are the only cross-signing keys we # can request over federation raise NotFoundError("No %s key found for %s" % (key_type, user_id)) diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index c4dbf22408b..49c5e930264 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -19,6 +19,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, List, Mapping, Optional, Union from synapse import event_auth @@ -54,7 +55,7 @@ def __init__(self, hs: "HomeServer"): self._store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state self._server_name = hs.hostname - self._is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) async def check_auth_rules_from_context( self, @@ -255,7 +256,7 @@ async def check_restricted_join_rules( if not await self.is_user_in_rooms(allowed_rooms, user_id): # If this is a remote request, the user might be in an allowed room # that we do not know about. - if not self._is_mine_id(user_id): + if not self.hs._is_mine_id(user_id): for room_id in allowed_rooms: if not await self._store.is_host_joined(room_id, self._server_name): raise SynapseError( diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 3f46032a43a..6a92054bb19 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -21,6 +21,7 @@ import logging import random +import weakref from typing import TYPE_CHECKING, Iterable, List, Optional from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState @@ -44,7 +45,7 @@ class EventStreamHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.clock = hs.get_clock() - self.hs = hs + self.hs = weakref.proxy(hs) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index c709ed2c63f..210a7e45b31 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -25,6 +25,7 @@ import enum import itertools import logging +import weakref from enum import Enum from http import HTTPStatus from typing import ( @@ -133,7 +134,7 @@ class FederationHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -143,8 +144,6 @@ def __init__(self, hs: "HomeServer"): self.state_handler = hs.get_state_handler() self.server_name = hs.hostname self.keyring = hs.get_keyring() - self.is_mine_id = hs.is_mine_id - self.is_mine_server_name = hs.is_mine_server_name self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self.event_creation_handler = hs.get_event_creation_handler() self.event_builder_factory = hs.get_event_builder_factory() @@ -460,7 +459,7 @@ async def try_backfill(domains: StrCollection) -> bool: for dom in domains: # We don't want to ask our own server for information we don't have - if self.is_mine_server_name(dom): + if self.hs.is_mine_server_name(dom): continue try: @@ -1075,7 +1074,7 @@ async def on_invite_request( 400, "The invite event was not from the server sending it" ) - if not self.is_mine_id(event.state_key): + if not self.hs.is_mine_id(event.state_key): raise SynapseError(400, "The invite event must be for this server") # block any attempts to invite the server notices mxid diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 5cec2b01e55..d370174e3ea 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -22,6 +22,7 @@ import collections import itertools import logging +import weakref from http import HTTPStatus from typing import ( TYPE_CHECKING, @@ -159,8 +160,6 @@ def __init__(self, hs: "HomeServer"): self._message_handler = hs.get_message_handler() self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() self._state_resolution_handler = hs.get_state_resolution_handler() - # avoid a circular dependency by deferring execution here - self._get_room_member_handler = hs.get_room_member_handler self._federation_client = hs.get_federation_client() self._third_party_event_rules = ( @@ -168,8 +167,7 @@ def __init__(self, hs: "HomeServer"): ) self._notifier = hs.get_notifier() - self._is_mine_id = hs.is_mine_id - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self._server_name = hs.hostname self._instance_name = hs.get_instance_name() @@ -680,7 +678,7 @@ async def backfill( server from invalid events (there is probably no point in trying to re-fetch invalid events from every other HS in the room.) """ - if self._is_mine_server_name(dest): + if self.hs._is_mine_server_name(dest): raise SynapseError(400, "Can't backfill from self.") events = await self._federation_client.backfill( @@ -1939,7 +1937,7 @@ async def _maybe_kick_guest_users(self, event: EventBase) -> None: event.room_id ) current_state_list = list(current_state.values()) - await self._get_room_member_handler().kick_guest_users(current_state_list) + await self.hs._get_room_member_handler().kick_guest_users(current_state_list) async def _check_for_soft_fail( self, @@ -2313,7 +2311,7 @@ async def _notify_persisted_event( # users if event.internal_metadata.is_outlier(): if event.membership != Membership.INVITE: - if not self._is_mine_id(target_user_id): + if not self.hs._is_mine_id(target_user_id): return target_user = UserID.from_string(target_user_id) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index d96b585308b..d193c435f83 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -24,6 +24,7 @@ import logging import urllib.parse +import weakref from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional, Tuple import attr @@ -67,7 +68,7 @@ def __init__(self, hs: "HomeServer"): ip_allowlist=hs.config.server.federation_ip_range_allowlist, ) self.federation_http_client = hs.get_federation_http_client() - self.hs = hs + self.hs = weakref.proxy(hs) self._web_client_location = hs.config.email.invite_client_location diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 75d64d2d50b..9d24a8ef3f0 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, List, Optional, Tuple from synapse.api.constants import ( @@ -64,7 +65,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() self.state_handler = hs.get_state_handler() - self.hs = hs + self.hs = weakref.proxy(hs) self.state = hs.get_state_handler() self.clock = hs.get_clock() self.validator = EventValidator() diff --git a/synapse/handlers/jwt.py b/synapse/handlers/jwt.py index 400f3a59aa1..d6e7ee50020 100644 --- a/synapse/handlers/jwt.py +++ b/synapse/handlers/jwt.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import weakref from typing import TYPE_CHECKING, Optional, Tuple from authlib.jose import JsonWebToken, JWTClaims @@ -32,7 +33,7 @@ class JwtHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.jwt_secret = hs.config.jwt.jwt_secret self.jwt_subject_claim = hs.config.jwt.jwt_subject_claim diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index aa295fb6c8a..6e282c22782 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -21,6 +21,7 @@ # import logging import random +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple @@ -475,7 +476,7 @@ async def _expire_event(self, event_id: str) -> None: class EventCreationHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.validator = EventValidator() self.event_builder_factory = hs.get_event_builder_factory() self.server_name = hs.hostname # nb must be called this for @measure_func diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 4070b74b7af..0258a3aa535 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -19,6 +19,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, List, Optional, Set, Tuple, cast from twisted.python.failure import Failure @@ -78,7 +79,7 @@ class PaginationHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index c652e333a6d..2629dc73785 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -76,6 +76,7 @@ import contextlib import itertools import logging +import weakref from bisect import bisect from contextlib import contextmanager from types import TracebackType @@ -192,13 +193,12 @@ class BasePresenceHandler(abc.ABC): writer""" def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.clock = hs.get_clock() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.presence_router = hs.get_presence_router() self.state = hs.get_state_handler() - self.is_mine_id = hs.is_mine_id self._presence_enabled = hs.config.server.presence_enabled self._track_presence = hs.config.server.track_presence @@ -400,7 +400,7 @@ async def maybe_send_presence_to_interested_destinations( if not self._federation: return - states = [s for s in states if self.is_mine_id(s.user_id)] + states = [s for s in states if self.hs.is_mine_id(s.user_id)] if not states: return @@ -665,7 +665,7 @@ async def process_replication_rows( for new_state in states: old_state = self.user_to_current_state.get(new_state.user_id) self.user_to_current_state[new_state.user_id] = new_state - is_mine = self.is_mine_id(new_state.user_id) + is_mine = self.hs.is_mine_id(new_state.user_id) if not old_state or should_notify(old_state, new_state, is_mine): state_to_notify.append(new_state) @@ -793,7 +793,7 @@ def __init__(self, hs: "HomeServer"): obj=state.user_id, then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, ) - if self.is_mine_id(state.user_id): + if self.hs.is_mine_id(state.user_id): self.wheel_timer.insert( now=now, obj=state.user_id, @@ -971,7 +971,7 @@ async def _update_states( new_state, should_notify, should_ping = handle_update( prev_state, new_state, - is_mine=self.is_mine_id(user_id), + is_mine=self.hs.is_mine_id(user_id), wheel_timer=self.wheel_timer, now=now, # When overriding disabled presence, don't kick off all the @@ -1077,7 +1077,7 @@ async def _handle_timeouts(self) -> None: changes = handle_timeouts( states, - is_mine_fn=self.is_mine_id, + is_mine_fn=self.hs.is_mine_id, syncing_user_devices=syncing_user_devices, user_to_devices=self._user_to_device_to_current_state, now=now, @@ -1603,7 +1603,7 @@ async def _handle_state_delta(self, room_id: str, deltas: List[StateDelta]) -> N prev_local_users = [] prev_remote_hosts = set() for user_id in prev_users: - if self.is_mine_id(user_id): + if self.hs.is_mine_id(user_id): prev_local_users.append(user_id) else: prev_remote_hosts.add(get_domain_from_id(user_id)) @@ -1615,7 +1615,7 @@ async def _handle_state_delta(self, room_id: str, deltas: List[StateDelta]) -> N newly_joined_local_users = [] newly_joined_remote_hosts = set() for user_id in newly_joined_users: - if self.is_mine_id(user_id): + if self.hs.is_mine_id(user_id): newly_joined_local_users.append(user_id) else: host = get_domain_from_id(user_id) @@ -1765,8 +1765,7 @@ def __init__(self, hs: "HomeServer"): # # AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler self.server_name = hs.hostname - self.get_presence_handler = hs.get_presence_handler - self.get_presence_router = hs.get_presence_router + self.hs = weakref.proxy(hs) self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -1838,7 +1837,9 @@ async def get_new_events( # Figure out which other users this user should explicitly receive # updates for additional_users_interested_in = ( - await self.get_presence_router().get_interested_users(user.to_string()) + await self.hs.get_presence_router().get_interested_users( + user.to_string() + ) ) # We have a set of users that we're interested in the presence of. We want to @@ -1912,8 +1913,10 @@ async def get_new_events( interested_and_updated_users.update(additional_users_interested_in) # Retrieve the current presence state for each user - users_to_state = await self.get_presence_handler().current_state_for_users( - interested_and_updated_users + users_to_state = ( + await self.hs.get_presence_handler().current_state_for_users( + interested_and_updated_users + ) ) presence_updates = list(users_to_state.values()) @@ -1953,8 +1956,10 @@ async def _filter_all_presence_updates_for_user( if updated_users is not None: # Get the actual presence update for each change - users_to_state = await self.get_presence_handler().current_state_for_users( - updated_users + users_to_state = ( + await self.hs.get_presence_handler().current_state_for_users( + updated_users + ) ) presence_updates = list(users_to_state.values()) @@ -1973,7 +1978,7 @@ async def _filter_all_presence_updates_for_user( # for a single user # Filter through the presence router - users_to_state_set = await self.get_presence_router().get_users_for_states( + users_to_state_set = await self.hs.get_presence_router().get_users_for_states( presence_updates ) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index dbff28e7fb5..8fa03e278ea 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -20,6 +20,7 @@ # import logging import random +import weakref from typing import TYPE_CHECKING, List, Optional, Union from synapse.api.constants import ProfileFields @@ -58,7 +59,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self.clock = hs.get_clock() - self.hs = hs + self.hs = weakref.proxy(hs) self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( @@ -73,8 +74,6 @@ def __init__(self, hs: "HomeServer"): hs.config.server.allowed_avatar_mimetypes ) - self._is_mine_server_name = hs.is_mine_server_name - self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict: @@ -366,7 +365,7 @@ async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: else: server_name = host - if self._is_mine_server_name(server_name): + if self.hs._is_mine_server_name(server_name): media_info: Optional[ Union[LocalMedia, RemoteMedia] ] = await self.store.get_local_media(media_id) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index c776654d12c..4f4301d61eb 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -19,6 +19,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Tuple from synapse.api.constants import EduTypes, ReceiptTypes @@ -49,7 +50,7 @@ def __init__(self, hs: "HomeServer"): self.event_handler = hs.get_event_handler() self._storage_controllers = hs.get_storage_controllers() - self.hs = hs + self.hs = weakref.proxy(hs) # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 6322d980d42..0bc4cdbf52f 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -23,6 +23,7 @@ """Contains functions for registering clients.""" import logging +import weakref from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, TypedDict from prometheus_client import Counter @@ -100,7 +101,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.auth_blocking = hs.get_auth_blocking() self._auth_handler = hs.get_auth_handler() diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py index a7b8a4bed74..fe68927c0e8 100644 --- a/synapse/handlers/reports.py +++ b/synapse/handlers/reports.py @@ -14,6 +14,7 @@ # # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING @@ -31,7 +32,7 @@ class ReportsHandler: def __init__(self, hs: "HomeServer"): - self._hs = hs + self._hs = weakref.proxy(hs) self._store = hs.get_datastores().main self._clock = hs.get_clock() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d8c4d0c20e7..58d7d60ee40 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -26,6 +26,7 @@ import math import random import string +import weakref from collections import OrderedDict from http import HTTPStatus from typing import ( @@ -125,7 +126,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.auth_blocking = hs.get_auth_blocking() self.clock = hs.get_clock() - self.hs = hs + self.hs = weakref.proxy(hs) self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self.event_creation_handler = hs.get_event_creation_handler() self.room_member_handler = hs.get_room_member_handler() @@ -1441,7 +1442,7 @@ async def _generate_and_create_room_id( class RoomContextHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -1806,7 +1807,7 @@ class RoomShutdownHandler: DEFAULT_ROOM_NAME = "Content Violation Notification" def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.room_member_handler = hs.get_room_member_handler() self._room_creation_handler = hs.get_room_creation_handler() self._replication = hs.get_replication_data_handler() diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 9d4307fb078..6b36765ee2a 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Any, List, Optional, Tuple import attr @@ -64,7 +65,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.hs = hs + self.hs = weakref.proxy(hs) self.enable_room_list_search = hs.config.roomdirectory.enable_room_list_search self.response_cache: ResponseCache[ Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]] diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b6800a9f638..97a9fd73e33 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -22,6 +22,7 @@ import abc import logging import random +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple @@ -89,7 +90,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): # ought to be separated out a lot better. def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.auth = hs.get_auth() @@ -2163,7 +2164,7 @@ class RoomForgetterHandler(StateDeltasHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) - self._hs = hs + self._hs = weakref.proxy(hs) self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._clock = hs.get_clock() diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index 170c477d6f4..8695fbc7975 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -15,6 +15,7 @@ # import logging +import weakref from typing import TYPE_CHECKING from synapse.events import EventBase @@ -29,7 +30,7 @@ class RoomPolicyHandler: def __init__(self, hs: "HomeServer"): - self._hs = hs + self._hs = weakref.proxy(hs) self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._event_auth_handler = hs.get_event_auth_handler() diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 1a71135d5fa..93a8b4fd4fb 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -21,6 +21,7 @@ import itertools import logging +import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple import attr @@ -60,7 +61,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.state_handler = hs.get_state_handler() self.clock = hs.get_clock() - self.hs = hs + self.hs = weakref.proxy(hs) self._event_serializer = hs.get_event_client_serializer() self._relations_handler = hs.get_relations_handler() self._storage_controllers = hs.get_storage_controllers() diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 92fed980e60..8027fd19235 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -21,6 +21,7 @@ import email.utils import logging +import weakref from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from io import BytesIO @@ -158,7 +159,7 @@ async def _sendmail( class SendEmailHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self._reactor = hs.get_reactor() diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index cb56eb53fc7..513b3614bee 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -102,7 +102,6 @@ def __init__(self, hs: "HomeServer"): self.event_sources = hs.get_event_sources() self.relations_handler = hs.get_relations_handler() self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync - self.is_mine_id = hs.is_mine_id self.connection_store = SlidingSyncConnectionStore(self.store) self.extensions = SlidingSyncExtensionHandler(hs) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index e196199f8ad..b495e99c0ea 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -15,6 +15,7 @@ import enum import logging +import weakref from itertools import chain from typing import ( TYPE_CHECKING, @@ -187,7 +188,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.storage_controllers = hs.get_storage_controllers() self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync - self.is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) async def compute_interested_rooms( self, @@ -469,7 +470,7 @@ async def _compute_interested_rooms_new_tables( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.is_mine_id): + if room_sync_config.must_await_full_state(self.hs.is_mine_id): filtered_sync_room_map = { room_id: room for room_id, room in filtered_sync_room_map.items() @@ -592,7 +593,7 @@ async def _compute_interested_rooms_new_tables( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.is_mine_id): + if room_sync_config.must_await_full_state(self.hs.is_mine_id): if room_id in partial_state_rooms: continue @@ -683,7 +684,7 @@ async def _compute_interested_rooms_fallback( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.is_mine_id): + if room_sync_config.must_await_full_state(self.hs.is_mine_id): filtered_sync_room_map = { room_id: room for room_id, room in filtered_sync_room_map.items() @@ -778,7 +779,7 @@ async def _compute_interested_rooms_fallback( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.is_mine_id): + if room_sync_config.must_await_full_state(self.hs.is_mine_id): if room_id in partial_state_rooms: continue diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 48f7ba094e0..e555ff603cd 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -22,6 +22,7 @@ import hashlib import io import logging +import weakref from typing import ( TYPE_CHECKING, Any, @@ -203,7 +204,7 @@ def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self._store = hs.get_datastores().main self._server_name = hs.hostname - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self._registration_handler = hs.get_registration_handler() self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() @@ -816,7 +817,7 @@ def is_allowed_mime_type(content_type: str) -> bool: avatar_url_parts = profile[ProfileFields.AVATAR_URL].split("/") server_name = avatar_url_parts[-2] media_id = avatar_url_parts[-1] - if self._is_mine_server_name(server_name): + if self.hs._is_mine_server_name(server_name): media = await self._media_repo.store.get_local_media(media_id) if media is not None and upload_name == media.upload_name: logger.info("skipping saving the user avatar") diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index aa33260809b..7a547e983ae 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -20,6 +20,7 @@ # # import logging +import weakref from collections import Counter from typing import ( TYPE_CHECKING, @@ -53,13 +54,12 @@ class StatsHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() self.clock = hs.get_clock() self.notifier = hs.get_notifier() - self.is_mine_id = hs.is_mine_id self.stats_enabled = hs.config.stats.stats_enabled @@ -265,7 +265,7 @@ async def _handle_deltas( raise ValueError("%r is not a valid membership" % (membership,)) user_id = delta.state_key - if self.is_mine_id(user_id): + if self.hs.is_mine_id(user_id): # this accounts for transitions like leave → ban and so on. has_changed_joinedness = (prev_membership == Membership.JOIN) != ( membership == Membership.JOIN diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 3c49655598e..e84c1017d17 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -20,6 +20,7 @@ # import logging import random +import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple import attr @@ -82,8 +83,7 @@ def __init__(self, hs: "HomeServer"): self._storage_controllers = hs.get_storage_controllers() self.server_name = hs.config.server.server_name self.clock = hs.get_clock() - self.is_mine_id = hs.is_mine_id - self.is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self.federation = None if hs.should_send_federation(): @@ -139,7 +139,7 @@ def _handle_timeout_for_member(self, now: int, member: RoomMember) -> None: # Check if we need to resend a keep alive over federation for this # user. - if self.federation and self.is_mine_id(member.user_id): + if self.federation and self.hs.is_mine_id(member.user_id): last_fed_poke = self._member_last_federation_poke.get(member, None) if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now: run_as_background_process( @@ -176,7 +176,7 @@ async def _push_remote(self, member: RoomMember, typing: bool) -> None: store=self.store, ) for domain in hosts: - if not self.is_mine_server_name(domain): + if not self.hs.is_mine_server_name(domain): logger.debug("sending typing update to %s", domain) self.federation.build_and_send_edu( destination=domain, @@ -233,11 +233,11 @@ async def _send_changes_in_typing_to_remotes( return for user_id in now_typing - prev_typing: - if self.is_mine_id(user_id): + if self.hs.is_mine_id(user_id): await self._push_remote(RoomMember(room_id, user_id), True) for user_id in prev_typing - now_typing: - if self.is_mine_id(user_id): + if self.hs.is_mine_id(user_id): await self._push_remote(RoomMember(room_id, user_id), False) def get_current_token(self) -> int: @@ -268,7 +268,7 @@ def __init__(self, hs: "HomeServer"): self.notifier = hs.get_notifier() self.event_auth_handler = hs.get_event_auth_handler() - self.hs = hs + self.hs = weakref.proxy(hs) hs.get_federation_registry().register_edu_handler( EduTypes.TYPING, self._recv_edu @@ -304,7 +304,7 @@ async def started_typing( ) -> None: target_user_id = target_user.to_string() - if not self.is_mine_id(target_user_id): + if not self.hs.is_mine_id(target_user_id): raise SynapseError(400, "User is not hosted on this homeserver") if target_user != requester.user: @@ -339,7 +339,7 @@ async def stopped_typing( ) -> None: target_user_id = target_user.to_string() - if not self.is_mine_id(target_user_id): + if not self.hs.is_mine_id(target_user_id): raise SynapseError(400, "User is not hosted on this homeserver") if target_user != requester.user: @@ -360,7 +360,7 @@ async def stopped_typing( def user_left_room(self, user: UserID, room_id: str) -> None: user_id = user.to_string() - if self.is_mine_id(user_id): + if self.hs.is_mine_id(user_id): member = RoomMember(room_id=room_id, user_id=user_id) self._stopped_typing(member) @@ -509,14 +509,10 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self._main_store = hs.get_datastores().main self.clock = hs.get_clock() - # We can't call get_typing_handler here because there's a cycle: - # - # Typing -> Notifier -> TypingNotificationEventSource -> Typing - # - self.get_typing_handler = hs.get_typing_handler + self.hs = weakref.proxy(hs) def _make_event_for(self, room_id: str) -> JsonMapping: - typing = self.get_typing_handler()._room_typing[room_id] + typing = self.hs.get_typing_handler()._room_typing[room_id] return { "type": EduTypes.TYPING, "room_id": room_id, @@ -542,7 +538,7 @@ async def get_new_events_as( with Measure( self.clock, name="typing.get_new_events_as", server_name=self.server_name ): - handler = self.get_typing_handler() + handler = self.hs.get_typing_handler() events = [] @@ -581,7 +577,7 @@ async def get_new_events( self.clock, name="typing.get_new_events", server_name=self.server_name ): from_key = int(from_key) - handler = self.get_typing_handler() + handler = self.hs.get_typing_handler() events = [] for room_id in room_ids: @@ -597,4 +593,4 @@ async def get_new_events( return events, handler._latest_room_serial def get_current_key(self) -> int: - return self.get_typing_handler()._latest_room_serial + return self.hs.get_typing_handler()._latest_room_serial diff --git a/synapse/handlers/ui_auth/checkers.py b/synapse/handlers/ui_auth/checkers.py index 32dca8c43ba..e3f876b2108 100644 --- a/synapse/handlers/ui_auth/checkers.py +++ b/synapse/handlers/ui_auth/checkers.py @@ -20,6 +20,7 @@ # import logging +import weakref from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, ClassVar, Sequence, Type @@ -159,7 +160,7 @@ async def check_auth(self, authdict: dict, clientip: str) -> Any: class _BaseThreepidAuthChecker: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main async def _check_threepid(self, medium: str, authdict: dict) -> dict: @@ -260,7 +261,7 @@ class RegistrationTokenAuthChecker(UserInteractiveAuthChecker): def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.hs = hs + self.hs = weakref.proxy(hs) self._enabled = bool( hs.config.registration.registration_requires_token ) or bool(hs.config.registration.enable_registration_token_3pid_bypass) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 5f9e96706a5..13870829fac 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -20,6 +20,7 @@ # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Set, Tuple @@ -105,7 +106,6 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.notifier = hs.get_notifier() - self.is_mine_id = hs.is_mine_id self.update_user_directory = hs.config.worker.should_update_user_directory self.search_all_users = hs.config.userdirectory.user_directory_search_all_users self.exclude_remote_users = ( @@ -113,7 +113,7 @@ def __init__(self, hs: "HomeServer"): ) self.show_locked_users = hs.config.userdirectory.show_locked_users self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker - self._hs = hs + self._hs = weakref.proxy(hs) # The current position in the current_state_delta stream self.pos: Optional[int] = None @@ -360,7 +360,7 @@ async def _handle_room_publicity_change( # being added multiple times. The batching upserts shouldn't make this # too bad, though. for user_id in users_in_room: - if not self.is_mine_id( + if not self._hs.is_mine_id( user_id ) or await self.store.should_include_local_user_in_dir(user_id): await self._track_user_joined_room(room_id, user_id) @@ -393,7 +393,7 @@ async def _handle_room_membership_event( ) # Both cases ignore excluded local users, so start by discarding them. - is_remote = not self.is_mine_id(state_key) + is_remote = not self._hs.is_mine_id(state_key) if not is_remote and not await self.store.should_include_local_user_in_dir( state_key ): @@ -456,7 +456,7 @@ async def _track_user_joined_room(self, room_id: str, joining_user_id: str) -> N and ( # We can't apply any special rules to remote users so # they're always included - not self.is_mine_id(other) + not self._hs.is_mine_id(other) # Check the special rules whether the local user should be # included in the user directory or await self.store.should_include_local_user_in_dir(other) @@ -466,7 +466,7 @@ async def _track_user_joined_room(self, room_id: str, joining_user_id: str) -> N # First, if the joining user is our local user then we need an # update for every other user in the room. - if self.is_mine_id(joining_user_id): + if self._hs.is_mine_id(joining_user_id): for other_user_id in other_users_in_room: updates_to_users_who_share_rooms.add( (joining_user_id, other_user_id) @@ -475,7 +475,7 @@ async def _track_user_joined_room(self, room_id: str, joining_user_id: str) -> N # Next, we need an update for every other local user in the room # that they now share a room with the joining user. for other_user_id in other_users_in_room: - if self.is_mine_id(other_user_id): + if self._hs.is_mine_id(other_user_id): updates_to_users_who_share_rooms.add( (other_user_id, joining_user_id) ) @@ -504,7 +504,7 @@ async def _handle_remove_user(self, room_id: str, user_id: str) -> None: # Additionally, if they're a remote user and we're no longer joined # to any rooms they're in, remove them from the user directory. - if not self.is_mine_id(user_id): + if not self._hs.is_mine_id(user_id): rooms_user_is_in = await self.store.get_user_dir_rooms_user_is_in(user_id) if len(rooms_user_is_in) == 0: diff --git a/synapse/http/client.py b/synapse/http/client.py index dcaaafe45db..c28d020c6d3 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -20,6 +20,7 @@ # import logging import urllib.parse +import weakref from http import HTTPStatus from io import BytesIO from typing import ( @@ -345,7 +346,7 @@ def __init__( hs: "HomeServer", treq_args: Optional[Dict[str, Any]] = None, ): - self.hs = hs + self.hs = weakref.proxy(hs) self.reactor = hs.get_reactor() self._extra_treq_args = treq_args or {} diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 31bde39c71a..7c89e1c7e67 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -24,6 +24,7 @@ import random import sys import urllib.parse +import weakref from http import HTTPStatus from io import BytesIO, StringIO from typing import ( @@ -401,7 +402,7 @@ def __init__( hs: "HomeServer", tls_client_options_factory: Optional[FederationPolicyForHTTPS], ): - self.hs = hs + self.hs = weakref.proxy(hs) self.signing_key = hs.signing_key self.server_name = hs.hostname diff --git a/synapse/http/server.py b/synapse/http/server.py index 395d82fd168..92e3a97575c 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -25,6 +25,7 @@ import types import urllib import urllib.parse +import weakref from http import HTTPStatus from http.client import FOUND from inspect import isawaitable @@ -472,7 +473,7 @@ def __init__( super().__init__(canonical_json, extract_context, clock=self.clock) # Map of path regex -> method -> callback. self._routes: Dict[Pattern[str], Dict[bytes, _PathEntry]] = {} - self.hs = hs + self.hs = weakref.proxy(hs) def register_paths( self, @@ -505,6 +506,9 @@ def register_paths( callback, servlet_classname ) + def unregister_paths(self) -> None: + self._routes.clear() + def _get_handler_for_request( self, request: "SynapseRequest" ) -> Tuple[ServletCallback, str, Dict[str, str]]: diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index d7259176e7f..5899ff68b62 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -23,6 +23,7 @@ import logging import os import shutil +import weakref from io import BytesIO from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple @@ -89,7 +90,7 @@ class MediaRepository: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.client = hs.get_federation_client() self.clock = hs.get_clock() @@ -143,7 +144,7 @@ def __init__(self, hs: "HomeServer"): storage_providers.append(provider) self.media_storage: MediaStorage = MediaStorage( - self.hs, self.primary_base_path, self.filepaths, storage_providers + hs, self.primary_base_path, self.filepaths, storage_providers ) self.clock.looping_call( diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index afd33c02a12..2ff6b5f637a 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -24,6 +24,7 @@ import logging import os import shutil +import weakref from contextlib import closing from io import BytesIO from types import TracebackType @@ -170,7 +171,7 @@ def __init__( filepaths: MediaFilePaths, storage_providers: Sequence["StorageProvider"], ): - self.hs = hs + self.hs = weakref.proxy(hs) self.reactor = hs.get_reactor() self.local_media_directory = local_media_directory self.filepaths = filepaths @@ -415,7 +416,7 @@ class FileResponder(Responder): """ def __init__(self, hs: "HomeServer", open_file: BinaryIO): - self.hs = hs + self.hs = weakref.proxy(hs) self.open_file = open_file def write_to_consumer(self, consumer: IConsumer) -> Deferred: diff --git a/synapse/media/storage_provider.py b/synapse/media/storage_provider.py index 300952025a3..277c7752505 100644 --- a/synapse/media/storage_provider.py +++ b/synapse/media/storage_provider.py @@ -23,6 +23,7 @@ import logging import os import shutil +import weakref from typing import TYPE_CHECKING, Callable, Optional from synapse.config._base import Config @@ -144,7 +145,7 @@ class FileStorageProviderBackend(StorageProvider): """ def __init__(self, hs: "HomeServer", config: str): - self.hs = hs + self.hs = weakref.proxy(hs) self.reactor = hs.get_reactor() self.cache_directory = hs.config.media.media_store_path self.base_directory = config diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index 5d9afda3229..8a7c86897bd 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -20,6 +20,7 @@ # # import logging +import weakref from io import BytesIO from types import TracebackType from typing import TYPE_CHECKING, List, Optional, Tuple, Type @@ -264,7 +265,7 @@ def __init__( media_repo: "MediaRepository", media_storage: MediaStorage, ): - self.hs = hs + self.hs = weakref.proxy(hs) self.reactor = hs.get_reactor() self.media_repo = media_repo self.media_storage = media_storage diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index f039cd54c3e..67b2f810f75 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -20,6 +20,7 @@ # import email.utils import logging +import weakref from typing import ( TYPE_CHECKING, Any, @@ -258,7 +259,7 @@ class ModuleApi: """ def __init__(self, hs: "HomeServer", auth_handler: AuthHandler) -> None: - self._hs = hs + self._hs = weakref.proxy(hs) # TODO: Fix this type hint once the types for the data stores have been ironed # out. @@ -1943,7 +1944,7 @@ class AccountDataManager: """ def __init__(self, hs: "HomeServer") -> None: - self._hs = hs + self._hs = weakref.proxy(hs) self._store = hs.get_datastores().main self._handler = hs.get_account_data_handler() diff --git a/synapse/notifier.py b/synapse/notifier.py index 6190432b877..f12863371c3 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import ( TYPE_CHECKING, Awaitable, @@ -223,7 +224,7 @@ def __init__(self, hs: "HomeServer"): self.user_to_user_stream: Dict[str, _NotifierUserStream] = {} self.room_to_user_streams: Dict[str, Set[_NotifierUserStream]] = {} - self.hs = hs + self.hs = weakref.proxy(hs) self._storage_controllers = hs.get_storage_controllers() self.event_sources = hs.get_event_sources() self.store = hs.get_datastores().main diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 7bc99bd7857..8aeaf19e4a8 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -94,6 +94,7 @@ """ import abc +import weakref from typing import TYPE_CHECKING, Any, Dict, Optional import attr @@ -157,7 +158,7 @@ class ThrottleParams: class Pusher(metaclass=abc.ABCMeta): def __init__(self, hs: "HomeServer", pusher_config: PusherConfig): - self.hs = hs + self.hs = weakref.proxy(hs) self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index fed99319300..9d643a36228 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import ( TYPE_CHECKING, Any, @@ -127,7 +128,7 @@ class BulkPushRuleEvaluator: """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.server_name = hs.hostname self.store = hs.get_datastores().main self.server_name = hs.hostname # nb must be called this for @measure_func diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index fadba480dd8..71004b2feca 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -21,6 +21,7 @@ import logging import urllib.parse +import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, TypeVar import bleach @@ -122,7 +123,7 @@ def __init__( template_html: jinja2.Template, template_text: jinja2.Template, ): - self.hs = hs + self.hs = weakref.proxy(hs) self.template_html = template_html self.template_text = template_text diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index 9a5dd7a9d4b..445ca56eaa0 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Callable, Dict, Optional from synapse.push import Pusher, PusherConfig @@ -35,7 +36,7 @@ class PusherFactory: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.config = hs.config self.pusher_types: Dict[str, Callable[[HomeServer, PusherConfig], Pusher]] = { diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 0a7541b4c70..cb4388af241 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Dict, Iterable, Optional from prometheus_client import Gauge @@ -64,7 +65,7 @@ class PusherPool: """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.pusher_factory = PusherFactory(hs) self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 31204a83849..9cdd474097b 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -24,6 +24,7 @@ import urllib.parse from inspect import signature from typing import TYPE_CHECKING, Any, Awaitable, Callable, ClassVar, Dict, List, Tuple +import weakref from prometheus_client import Counter, Gauge @@ -205,6 +206,7 @@ def make_client(cls, hs: "HomeServer") -> Callable: parameter to specify which instance to hit (the instance must be in the `instance_map` config). """ + _hs = weakref.proxy(hs) clock = hs.get_clock() client = hs.get_replication_client() local_instance_name = hs.get_instance_name() @@ -224,8 +226,8 @@ async def send_request( *, instance_name: str = MAIN_PROCESS_INSTANCE_NAME, **kwargs: Any ) -> Any: # We have to pull these out here to avoid circular dependencies... - streams = hs.get_replication_command_handler().get_streams_to_replicate() - replication = hs.get_replication_data_handler() + streams = _hs.get_replication_command_handler().get_streams_to_replicate() + replication = _hs.get_replication_data_handler() with outgoing_gauge.track_inprogress(): if instance_name == local_instance_name: diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index b99f11f7c6d..6d53c6d1936 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -21,6 +21,7 @@ """A replication client for use by synapse workers.""" import logging +import weakref from typing import TYPE_CHECKING, Dict, Iterable, Optional, Set, Tuple from sortedcontainers import SortedList @@ -414,8 +415,7 @@ def __init__(self, hs: "HomeServer"): assert hs.should_send_federation() self.store = hs.get_datastores().main - self._is_mine_id = hs.is_mine_id - self._hs = hs + self._hs = weakref.proxy(hs) # We need to make a temporary value to ensure that mypy picks up the # right type. We know we should have a federation sender instance since @@ -470,7 +470,7 @@ async def _on_new_receipts( """ for receipt in rows: # we only want to send on receipts for our own users - if not self._is_mine_id(receipt.user_id): + if not self._hs._is_mine_id(receipt.user_id): continue # Private read receipts never get sent over federation. if receipt.receipt_type == ReceiptTypes.READ_PRIVATE: diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index c488bce58e1..b5c80ce9f80 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -19,6 +19,7 @@ # # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -52,7 +53,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_GET( self, request: SynapseRequest, user_id: str, device_id: str @@ -60,7 +61,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -80,7 +81,7 @@ async def on_DELETE( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -96,7 +97,7 @@ async def on_PUT( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -124,7 +125,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_worker_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_GET( self, request: SynapseRequest, user_id: str @@ -132,7 +133,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -162,7 +163,7 @@ async def on_POST( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Can only create devices for local users" ) @@ -197,7 +198,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_POST( self, request: SynapseRequest, user_id: str @@ -205,7 +206,7 @@ async def on_POST( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index afb71f4a0fc..bb993c216d7 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -20,6 +20,7 @@ # +import weakref from enum import Enum from http import HTTPStatus from typing import TYPE_CHECKING, Dict, Tuple @@ -68,7 +69,7 @@ def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() self.store = hs.get_datastores().main - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_GET( self, @@ -81,7 +82,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "User must be local to check what experimental features are enabled.", @@ -108,7 +109,7 @@ async def on_PUT( body = parse_json_object_from_request(request) target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "User must be local to enable experimental features.", diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 195f22a4c25..f767c492b4a 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -20,6 +20,7 @@ # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Optional, Tuple @@ -266,7 +267,7 @@ class DeleteMediaByID(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self.media_repository = hs.get_media_repository() async def on_DELETE( @@ -274,7 +275,7 @@ async def on_DELETE( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self._is_mine_server_name(server_name): + if not self.hs._is_mine_server_name(server_name): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only delete local media") if await self.store.get_local_media(media_id) is None: @@ -359,7 +360,7 @@ class UserMediaRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/media$") def __init__(self, hs: "HomeServer"): - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main self.media_repository = hs.get_media_repository() @@ -372,7 +373,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) - if not self.is_mine(UserID.from_string(user_id)): + if not self.hs.is_mine(UserID.from_string(user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") user = await self.store.get_user_by_id(user_id) @@ -416,7 +417,7 @@ async def on_DELETE( await assert_requester_is_admin(self.auth, request) - if not self.is_mine(UserID.from_string(user_id)): + if not self.hs.is_mine(UserID.from_string(user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") user = await self.store.get_user_by_id(user_id) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index f8c5bf18d4a..827d9a7946e 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -19,6 +19,7 @@ # # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple, cast @@ -494,7 +495,7 @@ def __init__(self, hs: "HomeServer"): self.admin_handler = hs.get_admin_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_POST( self, request: SynapseRequest, room_identifier: str @@ -510,7 +511,7 @@ async def on_POST( assert_params_in_dict(content, ["user_id"]) target_user = UserID.from_string(content["user_id"]) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "This endpoint can only be used with local users", @@ -587,7 +588,7 @@ def __init__(self, hs: "HomeServer"): self._state_storage_controller = hs.get_storage_controllers().state self.event_creation_handler = hs.get_event_creation_handler() self.state_handler = hs.get_state_handler() - self.is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) async def on_POST( self, request: SynapseRequest, room_identifier: str @@ -623,7 +624,7 @@ async def on_POST( # We pick the local user with the highest power. user_power = power_levels.content.get("users", {}) admin_users = [ - user_id for user_id in user_power if self.is_mine_id(user_id) + user_id for user_id in user_power if self.hs.is_mine_id(user_id) ] admin_users.sort(key=lambda user: user_power[user]) @@ -656,7 +657,7 @@ async def on_POST( # If there is no power level events then the creator has rights. pl_content = {} admin_user_id = create_event.sender - if not self.is_mine_id(admin_user_id): + if not self.hs.is_mine_id(admin_user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "No local admin user in room", @@ -784,7 +785,7 @@ class RoomEventContextServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self.clock = hs.get_clock() self.room_context_handler = hs.get_room_context_handler() self._event_serializer = hs.get_event_client_serializer() @@ -914,7 +915,7 @@ class RoomMessagesRestServlet(RestServlet): PATTERNS = admin_patterns("/rooms/(?P[^/]*)/messages$") def __init__(self, hs: "HomeServer"): - self._hs = hs + self._hs = weakref.proxy(hs) self._clock = hs.get_clock() self._pagination_handler = hs.get_pagination_handler() self._auth = hs.get_auth() diff --git a/synapse/rest/admin/server_notice_servlet.py b/synapse/rest/admin/server_notice_servlet.py index f3150e88d71..f2e6f7d546a 100644 --- a/synapse/rest/admin/server_notice_servlet.py +++ b/synapse/rest/admin/server_notice_servlet.py @@ -17,6 +17,7 @@ # [This file includes modifications made by New Vector Limited] # # +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Optional, Tuple @@ -62,7 +63,7 @@ def __init__(self, hs: "HomeServer"): self.server_notices_manager = hs.get_server_notices_manager() self.admin_handler = hs.get_admin_handler() self.txns = HttpTransactionCache(hs) - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) def register(self, json_resource: HttpServer) -> None: PATTERN = "/send_server_notice" @@ -97,7 +98,7 @@ async def _do( ) target_user = UserID.from_string(body["user_id"]) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Server notices can only be sent to local users" ) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index c1955bf9593..82f73607884 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -22,6 +22,7 @@ import hmac import logging import secrets +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union @@ -239,7 +240,7 @@ class UserRestServletV2(RestServlet): """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() self.store = hs.get_datastores().main @@ -544,7 +545,7 @@ def __init__(self, hs: "HomeServer"): self.auth_handler = hs.get_auth_handler() self.reactor = hs.get_reactor() self.nonces: Dict[str, int] = {} - self.hs = hs + self.hs = weakref.proxy(hs) self._all_user_types = hs.config.user_types.all_user_types def _clear_old_nonces(self) -> None: @@ -724,7 +725,7 @@ class WhoisRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_GET( self, request: SynapseRequest, user_id: str @@ -735,7 +736,7 @@ async def on_GET( if target_user != requester.user: await assert_user_is_admin(self.auth, requester) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only whois a local user") ret = await self.admin_handler.get_whois(target_user) @@ -749,7 +750,7 @@ class DeactivateAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self._deactivate_account_handler = hs.get_deactivate_account_handler() self.auth = hs.get_auth() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main async def on_POST( @@ -758,7 +759,7 @@ async def on_POST( requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - if not self.is_mine(UserID.from_string(target_user_id)): + if not self.hs.is_mine(UserID.from_string(target_user_id)): raise SynapseError( HTTPStatus.BAD_REQUEST, "Can only deactivate local users" ) @@ -791,7 +792,7 @@ class SuspendAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main class PutBody(RequestBodyModel): @@ -803,7 +804,7 @@ async def on_PUT( requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - if not self.is_mine(UserID.from_string(target_user_id)): + if not self.hs.is_mine(UserID.from_string(target_user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only suspend local users") if not await self.store.get_user_by_id(target_user_id): @@ -914,7 +915,7 @@ class SearchUsersRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_GET( self, request: SynapseRequest, target_user_id: str @@ -931,7 +932,7 @@ async def on_GET( # if not is_admin and target_user != auth_user: # raise AuthError(HTTPStatus.FORBIDDEN, "You are not a server admin") - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only users a local user") term = parse_string(request, "term", required=True) @@ -983,7 +984,7 @@ class UserAdminServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) async def on_GET( self, request: SynapseRequest, user_id: str @@ -992,7 +993,7 @@ async def on_GET( target_user = UserID.from_string(user_id) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be admins of this homeserver", @@ -1015,7 +1016,7 @@ async def on_PUT( assert_params_in_dict(body, ["admin"]) - if not self.is_mine(target_user): + if not self.hs.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be admins of this homeserver", @@ -1039,7 +1040,7 @@ class UserMembershipRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/joined_rooms$") def __init__(self, hs: "HomeServer"): - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main @@ -1071,7 +1072,7 @@ class PushersRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/pushers$") def __init__(self, hs: "HomeServer"): - self.is_mine = hs.is_mine + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self.auth = hs.get_auth() @@ -1080,7 +1081,7 @@ async def on_GET( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.is_mine(UserID.from_string(user_id)): + if not self.hs.is_mine(UserID.from_string(user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") if not await self.store.get_user_by_id(user_id): @@ -1116,7 +1117,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() - self.is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) async def on_POST( self, request: SynapseRequest, user_id: str @@ -1125,7 +1126,7 @@ async def on_POST( await assert_user_is_admin(self.auth, requester) auth_user = requester.user - if not self.is_mine_id(user_id): + if not self.hs.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be logged in as" ) @@ -1184,14 +1185,14 @@ class ShadowBanRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) async def on_POST( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.is_mine_id(user_id): + if not self.hs.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be shadow-banned" ) @@ -1205,7 +1206,7 @@ async def on_DELETE( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.is_mine_id(user_id): + if not self.hs.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be shadow-banned" ) @@ -1236,14 +1237,14 @@ class RateLimitRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) async def on_GET( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.is_mine_id(user_id): + if not self.hs.is_mine_id(user_id): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") if not await self.store.get_user_by_id(user_id): @@ -1274,7 +1275,7 @@ async def on_POST( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.is_mine_id(user_id): + if not self.hs.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be ratelimited" ) @@ -1322,7 +1323,7 @@ async def on_DELETE( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.is_mine_id(user_id): + if not self.hs.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be ratelimited" ) @@ -1343,14 +1344,14 @@ class AccountDataRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() self._store = hs.get_datastores().main - self._is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) async def on_GET( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self._auth, request) - if not self._is_mine_id(user_id): + if not self.hs._is_mine_id(user_id): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") if not await self._store.get_user_by_id(user_id): diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 7d6c0afd9a6..d962bf2a26b 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -21,6 +21,7 @@ # import logging import random +import weakref from typing import TYPE_CHECKING, List, Literal, Optional, Tuple from urllib.parse import urlparse @@ -75,7 +76,7 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.datastore = hs.get_datastores().main self.config = hs.config self.identity_handler = hs.get_identity_handler() @@ -149,7 +150,7 @@ class PasswordRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() self.datastore = self.hs.get_datastores().main @@ -281,7 +282,7 @@ class DeactivateAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() self._deactivate_account_handler = hs.get_deactivate_account_handler() @@ -324,7 +325,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.config = hs.config self.identity_handler = hs.get_identity_handler() self.store = self.hs.get_datastores().main @@ -406,7 +407,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): PATTERNS = client_patterns("/account/3pid/msisdn/requestToken$") def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) super().__init__() self.store = self.hs.get_datastores().main self.identity_handler = hs.get_identity_handler() @@ -586,7 +587,7 @@ class ThreepidRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() @@ -649,7 +650,7 @@ class ThreepidAddRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() @@ -699,7 +700,7 @@ class ThreepidBindRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() @@ -727,7 +728,7 @@ class ThreepidUnbindRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() self.datastore = self.hs.get_datastores().main @@ -757,7 +758,7 @@ class ThreepidDeleteRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/account_data.py b/synapse/rest/client/account_data.py index 734c9e992f5..00347bd919a 100644 --- a/synapse/rest/client/account_data.py +++ b/synapse/rest/client/account_data.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Optional, Tuple from synapse.api.constants import AccountDataTypes, ReceiptTypes @@ -67,7 +68,7 @@ class AccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main self.handler = hs.get_account_data_handler() @@ -143,7 +144,7 @@ class UnstableAccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self.auth = hs.get_auth() self.handler = hs.get_account_data_handler() @@ -180,7 +181,7 @@ class RoomAccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main self.handler = hs.get_account_data_handler() @@ -278,7 +279,7 @@ class UnstableRoomAccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self.auth = hs.get_auth() self.handler = hs.get_account_data_handler() diff --git a/synapse/rest/client/account_validity.py b/synapse/rest/client/account_validity.py index ec7836b647f..f5388257e24 100644 --- a/synapse/rest/client/account_validity.py +++ b/synapse/rest/client/account_validity.py @@ -19,6 +19,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Tuple from twisted.web.server import Request @@ -42,7 +43,7 @@ class AccountValidityRenewServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.account_activity_handler = hs.get_account_validity_handler() self.auth = hs.get_auth() self.account_renewed_template = ( @@ -83,7 +84,7 @@ class AccountValiditySendMailServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.account_activity_handler = hs.get_account_validity_handler() self.auth = hs.get_auth() self.account_validity_renew_by_email_enabled = ( diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index b8dca7c7977..80a65476d8c 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, cast from twisted.web.server import Request @@ -50,7 +51,7 @@ class AuthRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() self.registration_handler = hs.get_registration_handler() diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index a279db1cc5c..09fe0170770 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -18,6 +18,7 @@ # # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -43,7 +44,7 @@ class CapabilitiesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.config = hs.config self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 5667af20d44..d5e963c2df1 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -21,6 +21,7 @@ # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple @@ -51,7 +52,7 @@ class DevicesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self._msc3852_enabled = hs.config.experimental.msc3852_enabled @@ -87,7 +88,7 @@ class DeleteDevicesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -138,7 +139,7 @@ class DeviceRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -295,7 +296,7 @@ class DehydratedDeviceServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -354,7 +355,7 @@ class ClaimDehydratedDeviceServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -493,7 +494,7 @@ class DehydratedDeviceV2Servlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() handler = hs.get_device_handler() self.e2e_keys_handler = hs.get_e2e_keys_handler() diff --git a/synapse/rest/client/filter.py b/synapse/rest/client/filter.py index f1e881975f5..5526580c3c3 100644 --- a/synapse/rest/client/filter.py +++ b/synapse/rest/client/filter.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import AuthError, NotFoundError, StoreError, SynapseError @@ -42,7 +43,7 @@ class GetFilterRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.filtering = hs.get_filtering() @@ -81,7 +82,7 @@ class CreateFilterRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.filtering = hs.get_filtering() diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 09749b840fc..b4dd68203ca 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -22,6 +22,7 @@ import logging import re +import weakref from collections import Counter from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast @@ -375,7 +376,7 @@ class SigningKeyUploadServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.e2e_keys_handler = hs.get_e2e_keys_handler() self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index aa0aa36cd94..54a3397f494 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -21,6 +21,7 @@ import logging import re +import weakref from typing import ( TYPE_CHECKING, Any, @@ -91,7 +92,7 @@ class LoginRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self._main_store = hs.get_datastores().main # JWT configuration variables. diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 4c044ae900e..a9d97e46a87 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -22,6 +22,7 @@ import logging import re +import weakref from typing import Optional from synapse.http.server import ( @@ -134,7 +135,7 @@ def __init__( self.media_repo = media_repo self.media_storage = media_storage self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self._server_name = hs.hostname self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.thumbnailer = ThumbnailProvider(hs, media_repo, media_storage) @@ -159,7 +160,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self._is_mine_server_name(server_name): + if self.hs._is_mine_server_name(server_name): if self.dynamic_thumbnails: await self.thumbnailer.select_or_generate_local_thumbnail( request, @@ -223,7 +224,7 @@ class DownloadResource(RestServlet): def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() async def on_GET( @@ -258,7 +259,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self._is_mine_server_name(server_name): + if self.hs._is_mine_server_name(server_name): await self.media_repo.get_local_media( request, media_id, file_name, max_timeout_ms ) diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index 104d54cd890..1ee2a1b7cf9 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -22,6 +22,7 @@ """This module contains REST servlets to do with presence: /presence/""" import logging +import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import AuthError, Codes, LimitExceededError, SynapseError @@ -45,7 +46,7 @@ class PresenceStatusRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.presence_handler = hs.get_presence_handler() self.clock = hs.get_clock() self.auth = hs.get_auth() diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 243245f7393..db93d60eb00 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -22,6 +22,7 @@ """This module contains REST servlets to do with profile: /profile/""" import re +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -63,7 +64,7 @@ class ProfileRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.profile_handler = hs.get_profile_handler() self.auth = hs.get_auth() @@ -106,7 +107,7 @@ class ProfileFieldRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.profile_handler = hs.get_profile_handler() self.auth = hs.get_auth() diff --git a/synapse/rest/client/pusher.py b/synapse/rest/client/pusher.py index a455f95a263..b5b8d733bdc 100644 --- a/synapse/rest/client/pusher.py +++ b/synapse/rest/client/pusher.py @@ -21,6 +21,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import Codes, SynapseError @@ -48,7 +49,7 @@ class PushersRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self._store = hs.get_datastores().main @@ -79,7 +80,7 @@ class PushersSetRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.notifier = hs.get_notifier() self.pusher_pool = self.hs.get_pusherpool() diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 58231d2b044..6c790307568 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -21,6 +21,7 @@ # import logging import random +import weakref from typing import TYPE_CHECKING, List, Optional, Tuple from twisted.web.server import Request @@ -81,7 +82,7 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.identity_handler = hs.get_identity_handler() self.config = hs.config @@ -176,7 +177,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.identity_handler = hs.get_identity_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -256,7 +257,7 @@ class RegistrationSubmitTokenServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.config = hs.config self.clock = hs.get_clock() @@ -322,7 +323,7 @@ class UsernameAvailabilityRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.registration_handler = hs.get_registration_handler() self.ratelimiter = FederationRateLimiter( hs.get_clock(), @@ -386,7 +387,7 @@ class RegistrationTokenValidityRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self.ratelimiter = Ratelimiter( store=self.store, @@ -415,7 +416,7 @@ class RegisterRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 81faf38a7f8..9b054e6d50d 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -20,6 +20,7 @@ # import logging +import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -49,7 +50,7 @@ class ReportEventRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -128,7 +129,7 @@ class ReportRoomRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -180,7 +181,7 @@ class ReportUserRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastores().main diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 09b5de275ce..d8f8b8bc9f1 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -23,6 +23,7 @@ import logging import re +import weakref from enum import Enum from http import HTTPStatus from typing import TYPE_CHECKING, Awaitable, Dict, List, Optional, Tuple @@ -600,7 +601,7 @@ class PublicRoomListRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -800,7 +801,7 @@ class RoomMessageListRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self.clock = hs.get_clock() self.pagination_handler = hs.get_pagination_handler() self.auth = hs.get_auth() @@ -1003,7 +1004,7 @@ class RoomEventContextServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self.clock = hs.get_clock() self.room_context_handler = hs.get_room_context_handler() self._event_serializer = hs.get_event_client_serializer() @@ -1347,7 +1348,7 @@ class RoomTypingRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.presence_handler = hs.get_presence_handler() self.auth = hs.get_auth() diff --git a/synapse/rest/client/room_upgrade_rest_servlet.py b/synapse/rest/client/room_upgrade_rest_servlet.py index 130ae31619a..981c148c45f 100644 --- a/synapse/rest/client/room_upgrade_rest_servlet.py +++ b/synapse/rest/client/room_upgrade_rest_servlet.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import Codes, ShadowBanError, SynapseError @@ -65,7 +66,7 @@ class RoomUpgradeRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = hs + self._hs = weakref.proxy(hs) self._room_creation_handler = hs.get_room_creation_handler() self._auth = hs.get_auth() self._worker_lock_handler = hs.get_worker_locks_handler() diff --git a/synapse/rest/client/sendtodevice.py b/synapse/rest/client/sendtodevice.py index 2a675145609..36240e5bf0f 100644 --- a/synapse/rest/client/sendtodevice.py +++ b/synapse/rest/client/sendtodevice.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Tuple from synapse.http import servlet @@ -46,7 +47,7 @@ class SendToDeviceRestServlet(servlet.RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.txns = HttpTransactionCache(hs) self.device_message_handler = hs.get_device_message_handler() diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index c9fb9dc4d3a..296c5acfd93 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -20,6 +20,7 @@ # import itertools import logging +import weakref from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union @@ -110,7 +111,7 @@ class SyncRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.server_name = hs.hostname self.auth = hs.get_auth() self.store = hs.get_datastores().main @@ -682,7 +683,7 @@ class SlidingSyncE2eeRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.store = hs.get_datastores().main self.sync_handler = hs.get_sync_handler() @@ -691,7 +692,7 @@ def __init__(self, hs: "HomeServer"): # derived information from rooms (see how `_generate_sync_entry_for_rooms()` # prepares a bunch of data for `_generate_sync_entry_for_device_list()`). self.only_member_events_filter_collection = FilterCollection( - self.hs, + hs, { "room": { # We only care about membership events for the `device_lists`. diff --git a/synapse/rest/client/thread_subscriptions.py b/synapse/rest/client/thread_subscriptions.py index 5307132ec34..b38d2c420e8 100644 --- a/synapse/rest/client/thread_subscriptions.py +++ b/synapse/rest/client/thread_subscriptions.py @@ -25,7 +25,6 @@ class ThreadSubscriptionsRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - self.is_mine = hs.is_mine self.store = hs.get_datastores().main self.handler = hs.get_thread_subscriptions_handler() diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 1a57996aecf..e3dbbcbf472 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -23,6 +23,7 @@ to ensure idempotency when performing PUTs using the REST API.""" import logging +import weakref from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Hashable, Tuple from typing_extensions import ParamSpec @@ -48,7 +49,7 @@ class HttpTransactionCache: def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = weakref.proxy(hs) self.clock = self.hs.get_clock() # $txn_key: (ObservableDeferred<(res_code, res_json_body)>, timestamp) self.transactions: Dict[ diff --git a/synapse/rest/client/user_directory.py b/synapse/rest/client/user_directory.py index 94fcb11c0ca..217205e60bd 100644 --- a/synapse/rest/client/user_directory.py +++ b/synapse/rest/client/user_directory.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import SynapseError @@ -42,7 +43,7 @@ class UserDirectorySearchRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() self.user_directory_handler = hs.get_user_directory_handler() diff --git a/synapse/rest/client/voip.py b/synapse/rest/client/voip.py index fbed3a3bae9..3091db2a80e 100644 --- a/synapse/rest/client/voip.py +++ b/synapse/rest/client/voip.py @@ -22,6 +22,7 @@ import base64 import hashlib import hmac +import weakref from typing import TYPE_CHECKING, Tuple from synapse.http.server import HttpServer @@ -40,7 +41,7 @@ class VoipRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = hs + self.hs = weakref.proxy(hs) self.auth = hs.get_auth() async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 3961f82894a..4992f2add87 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -20,6 +20,7 @@ import hmac import logging +import weakref from hashlib import sha256 from http import HTTPStatus from os import path @@ -83,7 +84,7 @@ class ConsentResource(DirectServeHtmlResource): def __init__(self, hs: "HomeServer"): super().__init__(clock=hs.get_clock()) - self.hs = hs + self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self.registration_handler = hs.get_registration_handler() diff --git a/synapse/rest/media/download_resource.py b/synapse/rest/media/download_resource.py index 3c3f703667a..993fae93047 100644 --- a/synapse/rest/media/download_resource.py +++ b/synapse/rest/media/download_resource.py @@ -21,6 +21,7 @@ # import logging import re +import weakref from typing import TYPE_CHECKING, Optional from synapse.http.server import set_corp_headers, set_cors_headers @@ -50,7 +51,7 @@ class DownloadResource(RestServlet): def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) async def on_GET( self, @@ -82,7 +83,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self._is_mine_server_name(server_name): + if self.hs._is_mine_server_name(server_name): await self.media_repo.get_local_media( request, media_id, file_name, max_timeout_ms, allow_authenticated=False ) diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index 536fea4c32f..5aee7c7d15f 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -22,6 +22,7 @@ import logging import re +import weakref from typing import TYPE_CHECKING from synapse.http.server import set_corp_headers, set_cors_headers @@ -61,7 +62,7 @@ def __init__( self.store = hs.get_datastores().main self.media_repo = media_repo self.media_storage = media_storage - self._is_mine_server_name = hs.is_mine_server_name + self.hs = weakref.proxy(hs) self._server_name = hs.hostname self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails @@ -85,7 +86,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self._is_mine_server_name(server_name): + if self.hs._is_mine_server_name(server_name): if self.dynamic_thumbnails: await self.thumbnail_provider.select_or_generate_local_thumbnail( request, diff --git a/synapse/server.py b/synapse/server.py index 231bd149075..149844aab58 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -33,6 +33,7 @@ from typing_extensions import TypeAlias from twisted.internet.interfaces import IOpenSSLContextFactory +from twisted.internet.task import LoopingCall from twisted.internet.tcp import Port from twisted.python.threadpool import ThreadPool from twisted.web.iweb import IPolicyForHTTPS @@ -285,6 +286,7 @@ def __init__( hostname : The hostname for the server. config: The full config for the homeserver. """ + if not reactor: from twisted.internet import reactor as _reactor @@ -311,6 +313,30 @@ def __init__( # This attribute is set by the free function `refresh_certificate`. self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None + self._looping_calls: List[LoopingCall] = [] + + def __del__(self) -> None: + logger.warning("Destructing HomeServer") + + def shutdown(self) -> None: + logger.info("Received shutdown request") + + # logger.info("Unsubscribing replication callbacks") + # self.get_replication_notifier()._replication_callbacks.clear() + + logger.info("Stopping looping calls: %d", len(self._looping_calls)) + for looping_call in self._looping_calls: + looping_call.stop() + del looping_call + + self._looping_calls = [] + + logger.info("Shutting down datastores") + self.get_datastores().main.shutdown() + + def register_looping_call(self, looping_call: LoopingCall) -> None: + self._looping_calls.append(looping_call) + def register_module_web_resource(self, path: str, resource: Resource) -> None: """Allows a module to register a web resource to be served at the given path. diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 19f86b5a563..310cf967415 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -18,6 +18,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, Optional from synapse.api.constants import EventTypes, Membership, RoomCreationPreset @@ -44,7 +45,7 @@ def __init__(self, hs: "HomeServer"): self._event_creation_handler = hs.get_event_creation_handler() self._message_handler = hs.get_message_handler() self._storage_controllers = hs.get_storage_controllers() - self._is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.servernotices.server_notices_mxid @@ -148,7 +149,9 @@ async def get_or_create_notice_room_for_user(self, user_id: str) -> str: if self.server_notices_mxid is None: raise Exception("Server notices not enabled") - assert self._is_mine_id(user_id), "Cannot send server notices to remote users" + assert self.hs._is_mine_id(user_id), ( + "Cannot send server notices to remote users" + ) requester = create_requester( self.server_notices_mxid, authenticated_entity=self.server_name diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 976a98a58b4..1def1a0469e 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -20,6 +20,7 @@ # import heapq import logging +import weakref from collections import ChainMap, defaultdict from typing import ( TYPE_CHECKING, @@ -193,7 +194,7 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() # nb must be called this for @measure_func self.store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state - self.hs = hs + self.hs = weakref.proxy(hs) self._state_resolution_handler = hs.get_state_resolution_handler() self._storage_controllers = hs.get_storage_controllers() self._events_shard_config = hs.config.worker.events_shard_config diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index d55c9e18ed6..568a816151b 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -20,6 +20,7 @@ # # import logging +import weakref from abc import ABCMeta from typing import TYPE_CHECKING, Any, Collection, Dict, Iterable, Optional, Union @@ -54,7 +55,6 @@ def __init__( db_conn: LoggingDatabaseConnection, hs: "HomeServer", ): - self.hs = hs self.server_name = hs.hostname # nb must be called this for @cached self._clock = hs.get_clock() self.database_engine = database.engine diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index d170bbddaa8..825373bed40 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -20,6 +20,7 @@ # import abc import logging +import weakref from enum import Enum, IntEnum from types import TracebackType from typing import ( @@ -248,7 +249,7 @@ class BackgroundUpdater: def __init__(self, hs: "HomeServer", database: "DatabasePool"): self._clock = hs.get_clock() self.db_pool = database - self.hs = hs + self.hs = weakref.proxy(hs) self._database_name = database.name() diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 9f54430a226..e01badac4dd 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -22,6 +22,7 @@ import itertools import logging +import weakref from collections import deque from typing import ( TYPE_CHECKING, @@ -340,13 +341,12 @@ def __init__( self.server_name = hs.hostname self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() - self.is_mine_id = hs.is_mine_id self._event_persist_queue = _EventPeristenceQueue( self._process_event_persist_queue_task ) self._state_resolution_handler = hs.get_state_resolution_handler() self._state_controller = state_controller - self.hs = hs + self.hs = weakref.proxy(hs) async def _process_event_persist_queue_task( self, @@ -1085,7 +1085,7 @@ async def _prune_extremities( while events_to_check: new_events: Set[str] = set() for event_to_check in events_to_check: - if self.is_mine_id(event_to_check.sender): + if self.hs.is_mine_id(event_to_check.sender): if event_to_check.type != EventTypes.Dummy: logger.debug("Not dropping own event") return new_latest_event_ids @@ -1166,7 +1166,7 @@ async def _is_server_still_joined( """ if not any( - self.is_mine_id(state_key) + self.hs.is_mine_id(state_key) for typ, state_key in itertools.chain(delta.to_delete, delta.to_insert) if typ == EventTypes.Member ): @@ -1178,7 +1178,7 @@ async def _is_server_still_joined( # current state events_to_check = [] # Event IDs that aren't an event we're persisting for (typ, state_key), event_id in delta.to_insert.items(): - if typ != EventTypes.Member or not self.is_mine_id(state_key): + if typ != EventTypes.Member or not self.hs.is_mine_id(state_key): continue for event, _ in ev_ctx_rm: @@ -1208,7 +1208,7 @@ async def _is_server_still_joined( users_to_ignore = [ state_key for typ, state_key in itertools.chain(delta.to_insert, delta.to_delete) - if typ == EventTypes.Member and self.is_mine_id(state_key) + if typ == EventTypes.Member and self.hs.is_mine_id(state_key) ] if await self.main_store.is_local_host_in_room_ignoring_users( diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 8997f4526fc..69d496f98c0 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -19,6 +19,7 @@ # # import logging +import weakref from itertools import chain from typing import ( TYPE_CHECKING, @@ -69,7 +70,7 @@ class StateStorageController: def __init__(self, hs: "HomeServer", stores: "Databases"): self.server_name = hs.hostname # nb must be called this for @cached - self._is_mine_id = hs.is_mine_id + self.hs = weakref.proxy(hs) self._clock = hs.get_clock() self.stores = stores self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) @@ -239,7 +240,7 @@ async def get_state_for_events( state_filter = StateFilter.all() await_full_state = True - if not state_filter.must_await_full_state(self._is_mine_id): + if not state_filter.must_await_full_state(self.hs._is_mine_id): await_full_state = False event_to_groups = await self.get_state_group_for_events( @@ -300,7 +301,7 @@ async def get_state_ids_for_events( state_filter = StateFilter.all() if await_full_state and not state_filter.must_await_full_state( - self._is_mine_id + self.hs._is_mine_id ): # Full state is not required if the state filter is restrictive enough. await_full_state = False @@ -602,7 +603,7 @@ async def get_current_state_ids( if state_filter is None: state_filter = StateFilter.all() - if await_full_state and state_filter.must_await_full_state(self._is_mine_id): + if await_full_state and state_filter.must_await_full_state(self.hs._is_mine_id): await self._partial_state_room_tracker.await_full_state(room_id) if state_filter is not None and not state_filter.is_full(): diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 61881956149..f30737c510b 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -23,6 +23,7 @@ import logging import time import types +import weakref from collections import defaultdict from time import monotonic as monotonic_time from typing import ( @@ -560,7 +561,7 @@ def __init__( database_config: DatabaseConnectionConfig, engine: BaseDatabaseEngine, ): - self.hs = hs + self.hs = weakref.proxy(hs) self._clock = hs.get_clock() self._txn_limit = database_config.config.get("txn_limit", 0) self._database_config = database_config diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index de55c452aea..65cf81544e0 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -20,6 +20,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast import attr @@ -170,12 +171,15 @@ def __init__( db_conn: LoggingDatabaseConnection, hs: "HomeServer", ): - self.hs = hs + self.hs = weakref.proxy(hs) self._clock = hs.get_clock() self.database_engine = database.engine super().__init__(database, db_conn, hs) + def shutdown(self) -> None: + super().shutdown() + async def get_users_paginate( self, start: int, diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 8e623bf0617..7224d57fff9 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -21,6 +21,7 @@ import datetime import itertools import logging +import weakref from queue import Empty, PriorityQueue from typing import ( TYPE_CHECKING, @@ -139,7 +140,7 @@ def __init__( ): super().__init__(database, db_conn, hs) - self.hs = hs + self.hs = weakref.proxy(hs) if hs.config.worker.run_background_tasks: hs.get_clock().looping_call( diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index f42023418e2..430a75b3f65 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -94,6 +94,7 @@ ) import attr +from twisted.internet.task import LoopingCall from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes from synapse.metrics.background_process_metrics import wrap_as_background_process @@ -254,6 +255,8 @@ def _deserialize_action(actions: str, is_highlight: bool) -> List[Union[dict, st class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBaseStore): + _background_tasks: List[LoopingCall] = [] + def __init__( self, database: DatabasePool, @@ -280,8 +283,10 @@ def __init__( self._rotate_count = 10000 self._doing_notif_rotation = False if hs.config.worker.run_background_tasks: - self._rotate_notif_loop = self._clock.looping_call( - self._rotate_notifs, 30 * 1000 + self._background_tasks.append( + self._clock.looping_call( + self._rotate_notifs, 30 * 1000 + ) ) self._clear_old_staging_loop = self._clock.looping_call( @@ -325,6 +330,11 @@ def __init__( columns=["room_id"], ) + def __del__(self) -> None: + logger.warning("Stopping background tasks") + for task in self._background_tasks: + task.stop() + async def _background_drop_null_thread_id_indexes( self, progress: JsonDict, batch_size: int ) -> int: @@ -1917,6 +1927,8 @@ def __init__( where_clause="highlight=0", ) + def __del__(self) -> None: + logger.warning("Destructor") def _action_has_highlight(actions: Collection[Union[Mapping, str]]) -> bool: for action in actions: diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index c0299cb62e8..bdb90d8473c 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -22,6 +22,7 @@ import collections import itertools import logging +import weakref from collections import OrderedDict from typing import ( TYPE_CHECKING, @@ -236,7 +237,7 @@ def __init__( main_data_store: "DataStore", db_conn: LoggingDatabaseConnection, ): - self.hs = hs + self.hs = weakref.proxy(hs) self.db_pool = db self.store = main_data_store self.database_engine = db.engine @@ -244,7 +245,6 @@ def __init__( self._instance_name = hs.get_instance_name() self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages - self.is_mine_id = hs.is_mine_id # This should only exist on instances that are configured to write assert hs.get_instance_name() in hs.config.worker.writers.events, ( @@ -553,7 +553,7 @@ async def _calculate_sliding_sync_table_changes( user_ids_to_delete_membership_snapshots = [ state_key for event_type, state_key in to_delete - if event_type == EventTypes.Member and self.is_mine_id(state_key) + if event_type == EventTypes.Member and self.hs.is_mine_id(state_key) ] membership_snapshot_shared_insert_values: SlidingSyncMembershipSnapshotSharedInsertValues = {} @@ -563,7 +563,9 @@ async def _calculate_sliding_sync_table_changes( if to_insert: membership_event_id_to_user_id_map: Dict[str, str] = {} for state_key, event_id in to_insert.items(): - if state_key[0] == EventTypes.Member and self.is_mine_id(state_key[1]): + if state_key[0] == EventTypes.Member and self.hs.is_mine_id( + state_key[1] + ): membership_event_id_to_user_id_map[event_id] = state_key[1] membership_event_map: Dict[str, EventBase] = {} @@ -1928,7 +1930,7 @@ def _update_current_state_txn( [ (room_id, state_key) for etype, state_key in itertools.chain(to_delete, to_insert) - if etype == EventTypes.Member and self.is_mine_id(state_key) + if etype == EventTypes.Member and self.hs.is_mine_id(state_key) ], ) @@ -1945,7 +1947,7 @@ def _update_current_state_txn( [ (room_id, key[1], ev_id, ev_id, ev_id) for key, ev_id in to_insert.items() - if key[0] == EventTypes.Member and self.is_mine_id(key[1]) + if key[0] == EventTypes.Member and self.hs.is_mine_id(key[1]) ], ) @@ -2997,7 +2999,7 @@ def _store_room_members_txn( # unless its an outlier, and an outlier is only "current" if it's an "out of # band membership", like a remote invite or a rejection of a remote invite. if ( - self.is_mine_id(event.state_key) + self.hs.is_mine_id(event.state_key) and not inhibit_local_membership_updates and event.internal_metadata.is_outlier() and event.internal_metadata.is_out_of_band_membership() diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index 9ce1100b5ce..dadf1739e61 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -81,6 +81,9 @@ def __init__( # Used in _generate_user_daily_visits to keep track of progress self._last_user_visit_update = self._get_start_of_day() + def __del__(self) -> None: + logger.warning("Destructor") + @wrap_as_background_process("read_forward_extremities") async def _read_forward_extremities(self) -> None: def fetch(txn: LoggingTransaction) -> List[Tuple[int, int]]: @@ -395,6 +398,8 @@ async def generate_user_daily_visits(self) -> None: Generates daily visit data for use in cohort/ retention analysis """ + logger.info("generate user daily visits called") + def _generate_user_daily_visits(txn: LoggingTransaction) -> None: logger.info("Calling _generate_user_daily_visits") today_start = self._get_start_of_day() diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index f5a6b98be71..5ca11731978 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -18,6 +18,7 @@ # # import logging +import weakref from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, cast from synapse.metrics.background_process_metrics import wrap_as_background_process @@ -50,7 +51,7 @@ def __init__( ): super().__init__(database, db_conn, hs) self._clock = hs.get_clock() - self.hs = hs + self.hs = weakref.proxy(hs) if hs.config.redis.redis_enabled: # If we're using Redis, we can shift this update process off to diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index 12cff1d3522..26981fa2ce2 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import weakref from typing import ( TYPE_CHECKING, Any, @@ -97,7 +98,7 @@ def __init__( writers=hs.config.worker.writers.presence, ) - self.hs = hs + self.hs = weakref.proxy(hs) self._presence_on_startup = self._get_active_presence(db_conn) presence_cache_prefill, min_presence_val = self.db_pool.get_cache_dict( diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index ce77e0b0d64..ef325167f0e 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -39,6 +39,8 @@ import attr +from twisted.internet.task import LoopingCall + from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS @@ -108,9 +110,11 @@ def __init__( and self.hs.config.metrics.metrics_flags.known_servers ): self._known_servers_count = 1 - self.hs.get_clock().looping_call( - self._count_known_servers, - 60 * 1000, + self._count_known_servers_task: Optional[LoopingCall] = ( + self.hs.get_clock().looping_call( + self._count_known_servers, + 6 * 1000, + ) ) self.hs.get_clock().call_later( 1, @@ -123,6 +127,13 @@ def __init__( lambda: self._known_servers_count, ) + def shutdown(self) -> None: + logger.info("Shutting down RoomMemberWorkerStore") + # print(self.hs.get_reactor().getDelayedCalls()) + if self._count_known_servers_task is not None: + self._count_known_servers_task.stop() + self._count_known_servers_task = None + @wrap_as_background_process("_count_known_servers") async def _count_known_servers(self) -> int: """ @@ -157,6 +168,7 @@ def _transact(txn: LoggingTransaction) -> int: # We always know about ourselves, even if we have nothing in # room_memberships (for example, the server is new). self._known_servers_count = max([count, 1]) + logger.info("Known servers: %s", self._known_servers_count) return self._known_servers_count @cached(max_entries=100000, iterable=True) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 5169656c73c..a2d21d7184b 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -20,6 +20,7 @@ # import logging +import weakref from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional, Set, Tuple from twisted.python.failure import Failure @@ -100,7 +101,7 @@ class TaskScheduler: OCCASIONAL_REPORT_INTERVAL_MS = 5 * 60 * 1000 # 5 minutes def __init__(self, hs: "HomeServer"): - self._hs = hs + self._hs = weakref.proxy(hs) self._store = hs.get_datastores().main self._clock = hs.get_clock() self._running_tasks: Set[str] = set() diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 25cf5269b8e..e0ae2c7a5f3 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -19,6 +19,7 @@ # # +import weakref from typing import Dict, Iterable, List, Optional from unittest.mock import AsyncMock, Mock @@ -414,7 +415,7 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = hs + self.hs = weakref.proxy(hs) # Mock the ApplicationServiceScheduler's _TransactionController's send method so that # we can track any outgoing ephemeral events self.send_mock = AsyncMock() diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 26642c18eac..8070dcdf22c 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -12,6 +12,7 @@ # . # # +import weakref from typing import Optional from unittest import mock @@ -51,7 +52,7 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: ) def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = hs + self.hs = weakref.proxy(hs) self.handler = hs.get_room_policy_handler() main_store = self.hs.get_datastores().main diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index bf18c1e72a5..5baa71fab8e 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import weakref from typing import Any, Dict, Iterable, List, Optional, Set, Tuple from unittest import mock @@ -127,7 +128,7 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = hs + self.hs = weakref.proxy(hs) self.handler = self.hs.get_room_summary_handler() # Create a user. @@ -1145,7 +1146,7 @@ class RoomSummaryTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = hs + self.hs = weakref.proxy(hs) self.handler = self.hs.get_room_summary_handler() # Create a user. diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index 18039a07e20..498212075bb 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -19,6 +19,7 @@ # # import json +import weakref from contextlib import contextmanager from typing import Generator, List, Set, Tuple from unittest import mock @@ -53,7 +54,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = hs + self.hs = weakref.proxy(hs) self.store: EventsWorkerStore = hs.get_datastores().main self.user = self.register_user("user", "pass") From c283db8a06d50df2030680ae5204197132929e4a Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 6 Aug 2025 14:54:20 -0600 Subject: [PATCH 002/181] More shutdown cleanup --- synapse/api/auth_blocking.py | 2 +- synapse/api/ratelimiting.py | 10 ++- synapse/app/homeserver.py | 13 ++- synapse/app/phone_stats_home.py | 9 +- synapse/appservice/scheduler.py | 2 +- synapse/federation/federation_client.py | 4 +- synapse/federation/federation_server.py | 2 +- synapse/federation/persistence.py | 3 +- synapse/federation/send_queue.py | 6 +- synapse/federation/sender/__init__.py | 18 ++-- synapse/federation/transport/server/_base.py | 2 +- synapse/handlers/account_validity.py | 2 +- synapse/handlers/auth.py | 6 +- synapse/handlers/delayed_events.py | 1 + synapse/handlers/device.py | 13 +-- synapse/handlers/devicemessage.py | 1 + synapse/handlers/identity.py | 2 + synapse/handlers/message.py | 5 +- synapse/handlers/pagination.py | 4 +- synapse/handlers/presence.py | 19 ++-- synapse/handlers/register.py | 16 +++- synapse/handlers/reports.py | 1 + synapse/handlers/room_member.py | 7 ++ synapse/handlers/room_summary.py | 1 + synapse/handlers/sync.py | 2 + synapse/handlers/typing.py | 4 +- synapse/handlers/worker_lock.py | 2 +- synapse/http/server.py | 5 +- synapse/http/site.py | 4 +- synapse/media/media_repository.py | 9 +- synapse/media/url_previewer.py | 5 +- synapse/metrics/__init__.py | 5 ++ synapse/metrics/common_usage_metrics.py | 6 +- synapse/notifier.py | 15 ++-- synapse/replication/http/devices.py | 3 +- synapse/replication/tcp/client.py | 2 +- synapse/replication/tcp/handler.py | 15 ++-- synapse/replication/tcp/protocol.py | 3 + synapse/replication/tcp/redis.py | 5 +- synapse/replication/tcp/resource.py | 5 +- synapse/replication/tcp/streams/__init__.py | 2 + synapse/replication/tcp/streams/_base.py | 9 +- synapse/replication/tcp/streams/events.py | 3 +- synapse/replication/tcp/streams/federation.py | 1 + synapse/rest/client/login.py | 2 + synapse/rest/client/login_token_request.py | 1 + synapse/rest/client/presence.py | 1 + synapse/rest/client/register.py | 1 + synapse/rest/client/sync.py | 1 + synapse/rest/client/transactions.py | 2 +- synapse/rest/media/create_resource.py | 1 + synapse/server.py | 88 ++++++++++++++++--- synapse/state/__init__.py | 3 +- synapse/storage/_base.py | 1 + synapse/storage/background_updates.py | 6 +- synapse/storage/controllers/persist_events.py | 2 +- synapse/storage/controllers/purge_events.py | 4 +- synapse/storage/controllers/state.py | 7 +- synapse/storage/database.py | 10 ++- synapse/storage/databases/main/__init__.py | 5 +- synapse/storage/databases/main/cache.py | 1 + .../storage/databases/main/censor_events.py | 2 +- synapse/storage/databases/main/client_ips.py | 6 +- synapse/storage/databases/main/deviceinbox.py | 5 +- synapse/storage/databases/main/devices.py | 4 +- .../databases/main/event_federation.py | 6 +- .../databases/main/event_push_actions.py | 18 ++-- synapse/storage/databases/main/events.py | 4 +- .../storage/databases/main/events_worker.py | 4 +- synapse/storage/databases/main/lock.py | 4 +- synapse/storage/databases/main/metrics.py | 5 +- .../storage/databases/main/registration.py | 8 +- synapse/storage/databases/main/roommember.py | 15 +--- synapse/storage/databases/main/session.py | 2 +- .../storage/databases/main/transactions.py | 2 +- synapse/storage/util/id_generators.py | 3 +- synapse/util/batching_queue.py | 4 +- synapse/util/caches/expiringcache.py | 8 +- synapse/util/caches/lrucache.py | 4 +- synapse/util/task_scheduler.py | 15 ++-- tests/federation/test_federation_catch_up.py | 1 + tests/util/test_expiring_cache.py | 4 + 82 files changed, 329 insertions(+), 185 deletions(-) diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index 5734416a6f4..fbc442518df 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -36,7 +36,7 @@ class AuthBlocking: def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self._server_notices_mxid = hs.config.servernotices.server_notices_mxid self._hs_disabled = hs.config.server.hs_disabled diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index 509ef6b2c18..47a75e0ef63 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -21,6 +21,7 @@ # from typing import TYPE_CHECKING, Dict, Hashable, Optional, Tuple +import weakref from synapse.api.errors import LimitExceededError from synapse.config.ratelimiting import RatelimitSettings @@ -33,6 +34,7 @@ from synapse.module_api.callbacks.ratelimit_callbacks import ( RatelimitModuleApiCallbacks, ) + from synapse.server import HomeServer class Ratelimiter: @@ -75,6 +77,7 @@ class Ratelimiter: def __init__( self, + hs: "HomeServer", store: DataStore, clock: Clock, cfg: RatelimitSettings, @@ -83,7 +86,7 @@ def __init__( self.clock = clock self.rate_hz = cfg.per_second self.burst_count = cfg.burst_count - self.store = store + self.store = weakref.proxy(store) self._limiter_name = cfg.key self._ratelimit_callbacks = ratelimit_callbacks @@ -94,7 +97,7 @@ def __init__( # * The rate_hz (leak rate) of this particular bucket. self.actions: Dict[Hashable, Tuple[float, float, float]] = {} - self.clock.looping_call(self._prune_message_counts, 60 * 1000) + hs.register_looping_call(self.clock.looping_call(self._prune_message_counts, 60 * 1000)) def _get_key( self, requester: Optional[Requester], key: Optional[Hashable] @@ -348,6 +351,7 @@ async def ratelimit( class RequestRatelimiter: def __init__( self, + hs: "HomeServer", store: DataStore, clock: Clock, rc_message: RatelimitSettings, @@ -358,6 +362,7 @@ def __init__( # The rate_hz and burst_count are overridden on a per-user basis self.request_ratelimiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=RatelimitSettings(key=rc_message.key, per_second=0, burst_count=0), @@ -368,6 +373,7 @@ def __init__( # by the presence of rate limits in the config if rc_admin_redaction: self.admin_redaction_ratelimiter: Optional[Ratelimiter] = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=rc_admin_redaction, diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e027b5eaea2..42bd6007178 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -69,6 +69,7 @@ from synapse.rest.well_known import well_known_resource from synapse.server import HomeServer from synapse.storage import DataStore +from synapse.types import ISynapseReactor from synapse.util.check_dependencies import VERSION, check_requirements from synapse.util.httpresourcetree import create_resource_tree from synapse.util.module_loader import load_module @@ -83,6 +84,14 @@ def gz_wrap(r: Resource) -> Resource: class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore + def shutdown(self) -> None: + super().shutdown() + for listener in self._listening_services: + listener.loseConnection() + self._listening_services.clear() + + self._reactor.stop() + def _listener_http( self, config: HomeServerConfig, @@ -308,7 +317,8 @@ def start_listening(self) -> None: logger.warning("Unrecognized listener type: %s", listener.type) -def setup(config_options: List[str]) -> SynapseHomeServer: +from typing import Optional +def setup(config_options: List[str], reactor: Optional[ISynapseReactor]=None) -> SynapseHomeServer: """ Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. @@ -365,6 +375,7 @@ def setup(config_options: List[str]) -> SynapseHomeServer: config.server.server_name, config=config, version_string=f"Synapse/{VERSION}", + reactor=reactor, ) synapse.config.logger.setup_logging(hs, config, use_worker_options=False) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index c6935f5089c..4dd3c4a6914 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -206,11 +206,12 @@ def performance_stats_init() -> None: ) # monthly active user limiting functionality - clock.looping_call( + hs.register_looping_call(clock.looping_call( hs.get_datastores().main.reap_monthly_active_users, ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, - ) - hs.get_datastores().main.reap_monthly_active_users() + )) + # TODO: (devon) how does this hold onto a DB pool reference? + #hs.register_background_process(hs.get_datastores().main.reap_monthly_active_users()) @wrap_as_background_process("generate_monthly_active_users") async def generate_monthly_active_users() -> None: @@ -234,7 +235,7 @@ async def generate_monthly_active_users() -> None: if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: generate_monthly_active_users() - clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000) + hs.register_looping_call(clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)) # End of monthly active user settings if hs.config.metrics.report_stats: diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 9d7fc0995a9..d90dc2d3e1a 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -523,7 +523,7 @@ def force_retry(self) -> None: if self.scheduled_recovery: self.clock.cancel_call_later(self.scheduled_recovery) # Run a retry, which will resechedule a recovery if it fails. - run_as_background_process( + deferred = run_as_background_process( "retry", self.retry, ) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 78f912e4d4e..4acc8cd8b6d 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -133,7 +133,7 @@ def __init__(self, hs: "HomeServer"): super().__init__(hs) self.pdu_destination_tried: Dict[str, Dict[str, int]] = {} - self._clock.looping_call(self._clear_tried_cache, 60 * 1000) + hs.register_looping_call(self._clock.looping_call(self._clear_tried_cache, 60 * 1000)) self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() @@ -143,6 +143,7 @@ def __init__(self, hs: "HomeServer"): # Cache mapping `event_id` to a tuple of the event itself and the `pull_origin` # (which server we pulled the event from) self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache( + hs=hs, cache_name="get_pdu_cache", server_name=self.server_name, clock=self._clock, @@ -162,6 +163,7 @@ def __init__(self, hs: "HomeServer"): Tuple[str, bool], Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]], ] = ExpiringCache( + hs=hs, cache_name="get_room_hierarchy_cache", server_name=self.server_name, clock=self._clock, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 6d70f277cbd..87d237f2add 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -300,7 +300,7 @@ async def on_incoming_transaction( # Start a periodic check for old staged events. This is to handle # the case where locks time out, e.g. if another process gets killed # without dropping its locks. - self._clock.looping_call(self._handle_old_staged_events, 60 * 1000) + self.hs.register_looping_call(self._clock.looping_call(self._handle_old_staged_events, 60 * 1000)) # keep this as early as possible to make the calculated origin ts as # accurate as possible. diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 8340b485031..108811481aa 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -28,6 +28,7 @@ import logging from typing import Optional, Tuple +import weakref from synapse.federation.units import Transaction from synapse.storage.databases.main import DataStore @@ -40,7 +41,7 @@ class TransactionActions: """Defines persistence actions that relate to handling Transactions.""" def __init__(self, datastore: DataStore): - self.store = datastore + self.store = weakref.proxy(datastore) async def have_responded( self, origin: str, transaction: Transaction diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index d004f63b024..064c4e7da1c 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -112,12 +112,12 @@ def __init__(self, hs: "HomeServer"): # lambda binds to the queue rather than to the name of the queue which # changes. ARGH. def register(name: str, queue: Sized) -> None: - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_federation_send_queue_%s_size" % (queue_name,), "", [], lambda: len(queue), - ) + )) for queue_name in [ "presence_map", @@ -129,7 +129,7 @@ def register(name: str, queue: Sized) -> None: ]: register(queue_name, getattr(self, queue_name)) - self.clock.looping_call(self._clear_queue, 30 * 1000) + hs.register_looping_call(self.clock.looping_call(self._clear_queue, 30 * 1000)) def _next_pos(self) -> int: pos = self.pos diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 8b9dced7493..8e890c19005 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -389,7 +389,7 @@ def __init__(self, hs: "HomeServer"): # map from destination to PerDestinationQueue self._per_destination_queues: Dict[str, PerDestinationQueue] = {} - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_federation_transaction_queue_pending_destinations", "", [], @@ -398,24 +398,24 @@ def __init__(self, hs: "HomeServer"): for d in self._per_destination_queues.values() if d.transmission_loop_running ), - ) + )) - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_federation_transaction_queue_pending_pdus", "", [], lambda: sum( d.pending_pdu_count() for d in self._per_destination_queues.values() ), - ) - LaterGauge( + )) + hs.register_later_gauge(LaterGauge( "synapse_federation_transaction_queue_pending_edus", "", [], lambda: sum( d.pending_edu_count() for d in self._per_destination_queues.values() ), - ) + )) self._is_processing = False self._last_poked_id = -1 @@ -426,16 +426,16 @@ def __init__(self, hs: "HomeServer"): 1.0 / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second ) self._destination_wakeup_queue = _DestinationWakeupQueue( - self, self.clock, max_delay_s=rr_txn_interval_per_room_s + weakref.proxy(self), self.clock, max_delay_s=rr_txn_interval_per_room_s ) # Regularly wake up destinations that have outstanding PDUs to be caught up - self.clock.looping_call_now( + hs.register_looping_call(self.clock.looping_call_now( run_as_background_process, WAKEUP_RETRY_PERIOD_SEC * 1000.0, "wake_destinations_needing_catchup", self._wake_destinations_needing_catchup, - ) + )) def _get_per_destination_queue( self, destination: str diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 667103318b5..2204c8cf49b 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -66,7 +66,7 @@ def __init__(self, hs: "HomeServer"): self.keyring = hs.get_keyring() self.server_name = hs.hostname self.hs = weakref.proxy(hs) - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self.federation_domain_whitelist = ( hs.config.federation.federation_domain_whitelist ) diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 055ff2b60aa..6aaeff5abe0 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -73,7 +73,7 @@ def __init__(self, hs: "HomeServer"): # Check the renewal emails to send and send them every 30min. if hs.config.worker.run_background_tasks: - self.clock.looping_call(self._send_renewal_emails, 30 * 60 * 1000) + hs.register_looping_call(self.clock.looping_call(self._send_renewal_emails, 30 * 60 * 1000)) async def is_user_expired(self, user_id: str) -> bool: """Checks if a user has expired against third-party modules. diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 9d6ff12a194..da72ea029b2 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -227,6 +227,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for failed auth during UIA. Uses same ratelimit config # as per `rc_login.failed_attempts`. self._failed_uia_attempts_ratelimiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=self.hs.config.ratelimiting.rc_login_failed_attempts, @@ -237,6 +238,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for failed /login attempts self._failed_login_attempts_ratelimiter = Ratelimiter( + hs=hs, store=self.store, clock=hs.get_clock(), cfg=self.hs.config.ratelimiting.rc_login_failed_attempts, @@ -246,12 +248,12 @@ def __init__(self, hs: "HomeServer"): # Expire old UI auth sessions after a period of time. if hs.config.worker.run_background_tasks: - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( run_as_background_process, 5 * 60 * 1000, "expire_old_sessions", self._expire_old_sessions, - ) + )) # Load the SSO HTML templates. diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index beb0e819c2b..7f040c02d30 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -67,6 +67,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for management of existing delayed events, # keyed by the sending user ID & device ID. self._delayed_event_mgmt_ratelimiter = Ratelimiter( + hs=hs, store=self._store, clock=self._clock, cfg=self._config.ratelimiting.rc_delayed_event_mgmt, diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 7e416c6daa5..aaef53865ce 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -127,7 +127,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self.hs = weakref.proxy(hs) - self.store = cast("GenericWorkerStore", hs.get_datastores().main) + self.store = weakref.proxy(cast("GenericWorkerStore", hs.get_datastores().main)) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() self._appservice_handler = hs.get_application_service_handler() @@ -191,12 +191,12 @@ def __init__(self, hs: "HomeServer"): hs.config.worker.run_background_tasks and self._delete_stale_devices_after is not None ): - self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( run_as_background_process, DELETE_STALE_DEVICES_INTERVAL_MS, "delete_stale_devices", self._delete_stale_devices, - ) + )) async def _delete_stale_devices(self) -> None: """Background task that deletes devices which haven't been accessed for more than @@ -1327,7 +1327,7 @@ class DeviceListWorkerUpdater: "Handles incoming device list updates from federation and contacts the main device list writer over replication" def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self._notifier = hs.get_notifier() # On which instance the DeviceListUpdater is running # Must be kept in sync with DeviceHandler @@ -1458,6 +1458,7 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): # but they're useful to have them about to reduce the number of spurious # resyncs. self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache( + hs=hs, cache_name="device_update_edu", server_name=self.server_name, clock=self.clock, @@ -1468,12 +1469,12 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): # Attempt to resync out of sync device lists every 30s. self._resync_retry_lock = Lock() - self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( run_as_background_process, 30 * 1000, func=self._maybe_retry_device_resync, desc="_maybe_retry_device_resync", - ) + )) @trace async def incoming_device_list_update( diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index a79f557fe3f..bf4f3ff08e3 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -81,6 +81,7 @@ def __init__(self, hs: "HomeServer"): # a rate limiter for room key requests. The keys are # (sending_user_id, sending_device_id). self._ratelimiter = Ratelimiter( + hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_key_requests, diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index d193c435f83..31888e0f1f9 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -74,11 +74,13 @@ def __init__(self, hs: "HomeServer"): # Ratelimiters for `/requestToken` endpoints. self._3pid_validation_ratelimiter_ip = Ratelimiter( + hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_3pid_validation, ) self._3pid_validation_ratelimiter_address = Ratelimiter( + hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_3pid_validation, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6e282c22782..f4e1df76087 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -539,13 +539,13 @@ def __init__(self, hs: "HomeServer"): self.config.worker.run_background_tasks and self.config.server.cleanup_extremities_with_dummy_events ): - self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( lambda: run_as_background_process( "send_dummy_events_to_fill_extremities", self._send_dummy_events_to_fill_extremities, ), 5 * 60 * 1000, - ) + )) self._message_handler = hs.get_message_handler() @@ -559,6 +559,7 @@ def __init__(self, hs: "HomeServer"): self._external_cache_joined_hosts_updates: Optional[ExpiringCache] = None if self._external_cache.is_enabled(): self._external_cache_joined_hosts_updates = ExpiringCache( + hs=hs, cache_name="_external_cache_joined_hosts_updates", server_name=self.server_name, clock=self.clock, diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 0258a3aa535..94f825783ed 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -116,14 +116,14 @@ def __init__(self, hs: "HomeServer"): for job in hs.config.retention.retention_purge_jobs: logger.info("Setting up purge job with config: %s", job) - self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( run_as_background_process, job.interval, "purge_history_for_rooms_in_range", self.purge_history_for_rooms_in_range, job.shortest_max_lifetime, job.longest_max_lifetime, - ) + )) self._task_scheduler.register_action( self._purge_history, PURGE_HISTORY_ACTION_NAME diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 2629dc73785..a1b4f4150b0 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -195,7 +195,7 @@ class BasePresenceHandler(abc.ABC): def __init__(self, hs: "HomeServer"): self.hs = weakref.proxy(hs) self.clock = hs.get_clock() - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self._storage_controllers = hs.get_storage_controllers() self.presence_router = hs.get_presence_router() self.state = hs.get_state_handler() @@ -508,9 +508,9 @@ def __init__(self, hs: "HomeServer"): self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs) self._set_state_client = ReplicationPresenceSetState.make_client(hs) - self._send_stop_syncing_loop = self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( self.send_stop_syncing, UPDATE_SYNCING_USERS_MS - ) + )) hs.get_reactor().addSystemEventTrigger( "before", @@ -757,12 +757,12 @@ def __init__(self, hs: "HomeServer"): EduTypes.PRESENCE, self.incoming_presence ) - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_handlers_presence_user_to_current_state_size", "", [], lambda: len(self.user_to_current_state), - ) + )) # The per-device presence state, maps user to devices to per-device presence state. self._user_to_device_to_current_state: Dict[ @@ -810,6 +810,7 @@ def __init__(self, hs: "HomeServer"): # have not yet been persisted self.unpersisted_users_changes: Set[str] = set() + # TODO: (devon) how do I remove this binding? hs.get_reactor().addSystemEventTrigger( "before", "shutdown", @@ -859,12 +860,12 @@ def __init__(self, hs: "HomeServer"): 60 * 1000, ) - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_handlers_presence_wheel_timer_size", "", [], lambda: len(self.wheel_timer), - ) + )) # Used to handle sending of presence to newly joined users/servers if self._track_presence: @@ -2376,7 +2377,7 @@ def __init__(self, hs: "HomeServer", presence_handler: BasePresenceHandler): self._clock = hs.get_clock() self._notifier = hs.get_notifier() self._instance_name = hs.get_instance_name() - self._presence_handler = presence_handler + self._presence_handler = weakref.proxy(presence_handler) self._repl_client = ReplicationGetStreamUpdates.make_client(hs) # Should we keep a queue of recent presence updates? We only bother if @@ -2411,7 +2412,7 @@ def __init__(self, hs: "HomeServer", presence_handler: BasePresenceHandler): self._current_tokens: Dict[str, int] = {} if self._queue_presence_updates: - self._clock.looping_call(self._clear_queue, self._CLEAR_ITEMS_EVERY_MS) + hs.register_looping_call(self._clock.looping_call(self._clear_queue, self._CLEAR_ITEMS_EVERY_MS)) def _clear_queue(self) -> None: """Clear out older entries from the queue.""" diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 0bc4cdbf52f..b48a1883a6e 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -98,7 +98,7 @@ class LoginDict(TypedDict): class RegistrationHandler: def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() self.hs = weakref.proxy(hs) @@ -127,7 +127,19 @@ def __init__(self, hs: "HomeServer"): ) else: self.device_handler = hs.get_device_handler() - self._register_device_client = self.register_device_inner + self_ref = weakref.proxy(self) + async def do_it( + user_id: str, + device_id: Optional[str], + initial_display_name: Optional[str], + is_guest: bool = False, + is_appservice_ghost: bool = False, + should_issue_refresh_token: bool = False, + auth_provider_id: Optional[str] = None, + auth_provider_session_id: Optional[str] = None, + ): + return RegistrationHandler.register_device_inner(self_ref, user_id, device_id,initial_display_name, is_guest,is_appservice_ghost,should_issue_refresh_token, auth_provider_id,auth_provider_session_id) + self._register_device_client = do_it self.pusher_pool = hs.get_pusherpool() self.session_lifetime = hs.config.registration.session_lifetime diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py index fe68927c0e8..fd7009fc666 100644 --- a/synapse/handlers/reports.py +++ b/synapse/handlers/reports.py @@ -39,6 +39,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for management of existing delayed events, # keyed by the requesting user ID. self._reports_ratelimiter = Ratelimiter( + hs=hs, store=self._store, clock=self._clock, cfg=hs.config.ratelimiting.rc_reports, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 97a9fd73e33..4cd96269152 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -128,6 +128,7 @@ def __init__(self, hs: "HomeServer"): self.allow_per_room_profiles = self.config.server.allow_per_room_profiles self._join_rate_limiter_local = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_joins_local, @@ -136,6 +137,7 @@ def __init__(self, hs: "HomeServer"): # I.e. joins this server makes by requesting /make_join /send_join from # another server. self._join_rate_limiter_remote = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_joins_remote, @@ -149,6 +151,7 @@ def __init__(self, hs: "HomeServer"): # other homeservers # I wonder if a homeserver-wide collection of rate limiters might be cleaner? self._join_rate_per_room_limiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_joins_per_room, @@ -157,6 +160,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for invites, keyed by room (across all issuers, all # recipients). self._invites_per_room_limiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_room, @@ -166,6 +170,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for invites, keyed by recipient (across all rooms, all # issuers). self._invites_per_recipient_limiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_user, @@ -175,6 +180,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for invites, keyed by issuer (across all rooms, all # recipients). self._invites_per_issuer_limiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_issuer, @@ -182,6 +188,7 @@ def __init__(self, hs: "HomeServer"): ) self._third_party_invite_limiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_third_party_invite, diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 838fee6a303..9eb6bd75a4b 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -104,6 +104,7 @@ def __init__(self, hs: "HomeServer"): self._server_name = hs.hostname self._federation_client = hs.get_federation_client() self._ratelimiter = Ratelimiter( + hs=hs, store=self._store, clock=hs.get_clock(), cfg=RatelimitSettings("", per_second=5, burst_count=10), diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 69064e751aa..60c529467ed 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,6 +37,7 @@ Union, overload, ) +import weakref import attr from prometheus_client import Counter @@ -363,6 +364,7 @@ def __init__(self, hs: "HomeServer"): self.lazy_loaded_members_cache: ExpiringCache[ Tuple[str, Optional[str]], LruCache[str, str] ] = ExpiringCache( + hs=hs, cache_name="lazy_loaded_members_cache", server_name=self.server_name, clock=self.clock, diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index e84c1017d17..1c483622368 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -106,8 +106,8 @@ def __init__(self, hs: "HomeServer"): self._rooms_updated: Set[str] = set() - self.clock.looping_call(self._handle_timeouts, 5000) - self.clock.looping_call(self._prune_old_typing, FORGET_TIMEOUT) + hs.register_looping_call(self.clock.looping_call(self._handle_timeouts, 5000)) + hs.register_looping_call(self.clock.looping_call(self._prune_old_typing, FORGET_TIMEOUT)) def _reset(self) -> None: """Reset the typing handler's data caches.""" diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 3077e9e463d..d9e61962e51 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -78,7 +78,7 @@ def __init__(self, hs: "HomeServer") -> None: Tuple[str, str], WeakSet[Union[WaitingLock, WaitingMultiLock]] ] = {} - self._clock.looping_call(self._cleanup_locks, 30_000) + hs.register_looping_call(self._clock.looping_call(self._cleanup_locks, 30_000)) self._notifier.add_lock_released_callback(self._on_lock_released) diff --git a/synapse/http/server.py b/synapse/http/server.py index 92e3a97575c..efcec958caf 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -503,12 +503,9 @@ def register_paths( for path_pattern in path_patterns: logger.debug("Registering for %s %s", method, path_pattern.pattern) self._routes.setdefault(path_pattern, {})[method_bytes] = _PathEntry( - callback, servlet_classname + weakref.proxy(callback), servlet_classname ) - def unregister_paths(self) -> None: - self._routes.clear() - def _get_handler_for_request( self, request: "SynapseRequest" ) -> Tuple[ServletCallback, str, Dict[str, str]]: diff --git a/synapse/http/site.py b/synapse/http/site.py index e83a4447b2f..2eac2a15380 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -23,6 +23,7 @@ import time from http import HTTPStatus from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, Union +import weakref import attr from zope.interface import implementer @@ -701,10 +702,11 @@ def __init__( request_id_header = config.http_options.request_id_header + self_ref = weakref.proxy(self) def request_factory(channel: HTTPChannel, queued: bool) -> Request: return request_class( channel, - self, + self_ref, max_request_body_size=max_request_body_size, queued=queued, request_id_header=request_id_header, diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 5899ff68b62..f4da86acf17 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -120,6 +120,7 @@ def __init__(self, hs: "HomeServer"): self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.download_ratelimiter = Ratelimiter( + hs=hs, store=hs.get_storage_controllers().main, clock=hs.get_clock(), cfg=hs.config.ratelimiting.remote_media_downloads, @@ -147,9 +148,9 @@ def __init__(self, hs: "HomeServer"): hs, self.primary_base_path, self.filepaths, storage_providers ) - self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( self._start_update_recently_accessed, UPDATE_RECENTLY_ACCESSED_TS - ) + )) # Media retention configuration options self._media_retention_local_media_lifetime_ms = ( @@ -166,10 +167,10 @@ def __init__(self, hs: "HomeServer"): ): # Run the background job to apply media retention rules routinely, # with the duration between runs dictated by the homeserver config. - self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( self._start_apply_media_retention_rules, MEDIA_RETENTION_CHECK_PERIOD_MS, - ) + )) if hs.config.media.url_preview_enabled: self.url_previewer: Optional[UrlPreviewer] = UrlPreviewer( diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index eb0104e5432..f60899eaf59 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -199,6 +199,7 @@ def __init__( # memory cache mapping urls to an ObservableDeferred returning # JSON-encoded OG metadata self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache( + hs=hs, cache_name="url_previews", server_name=self.server_name, clock=self.clock, @@ -207,9 +208,9 @@ def __init__( ) if self._worker_run_media_background_jobs: - self._cleaner_loop = self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( self._start_expire_url_cache_data, 10 * 1000 - ) + )) async def preview(self, url: str, user: UserID, ts: int) -> bytes: # the in-memory cache: diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index de750a5de2d..151cd1c7688 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -196,6 +196,11 @@ def _register(self) -> None: REGISTRY.register(self) all_gauges[self.name] = self + def unregister(self) -> None: + if self.name in all_gauges.keys(): + logger.warning("%s unregistering", self.name) + REGISTRY.unregister(all_gauges.pop(self.name)) + # `MetricsEntry` only makes sense when it is a `Protocol`, # but `Protocol` can't be used as a `TypeVar` bound. diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index 970367e9e05..feeec49425c 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -19,6 +19,7 @@ # # from typing import TYPE_CHECKING +import weakref import attr @@ -49,6 +50,7 @@ class CommonUsageMetricsManager: def __init__(self, hs: "HomeServer") -> None: self._store = hs.get_datastores().main self._clock = hs.get_clock() + self.hs = weakref.proxy(hs) async def get_metrics(self) -> CommonUsageMetrics: """Get the CommonUsageMetrics object. If no collection has happened yet, do it @@ -64,12 +66,12 @@ async def setup(self) -> None: run_as_background_process( desc="common_usage_metrics_update_gauges", func=self._update_gauges ) - self._clock.looping_call( + self.hs.register_looping_call(self._clock.looping_call( run_as_background_process, 5 * 60 * 1000, desc="common_usage_metrics_update_gauges", func=self._update_gauges, - ) + )) async def _collect(self) -> CommonUsageMetrics: """Collect the common metrics and either create the CommonUsageMetrics object to diff --git a/synapse/notifier.py b/synapse/notifier.py index f12863371c3..bc5688f8231 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -247,13 +247,14 @@ def __init__(self, hs: "HomeServer"): self.federation_sender = None if hs.should_send_federation(): + # TODO: (devon) why is this one already a weakref proxy? self.federation_sender = hs.get_federation_sender() self.state_handler = hs.get_state_handler() - self.clock.looping_call( + hs.register_looping_call(self.clock.looping_call( self.remove_expired_streams, self.UNUSED_STREAM_EXPIRY_MS - ) + )) # This is not a very cheap test to perform, but it's only executed # when rendering the metrics page, which is likely once per minute at @@ -268,17 +269,17 @@ def count_listeners() -> int: return sum(stream.count_listeners() for stream in all_user_streams) - LaterGauge("synapse_notifier_listeners", "", [], count_listeners) + hs.register_later_gauge(LaterGauge("synapse_notifier_listeners", "", [], count_listeners)) - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_notifier_rooms", "", [], lambda: count(bool, list(self.room_to_user_streams.values())), - ) - LaterGauge( + )) + hs.register_later_gauge(LaterGauge( "synapse_notifier_users", "", [], lambda: len(self.user_to_user_stream) - ) + )) def add_replication_callback(self, cb: Callable[[], None]) -> None: """Add a callback that will be called when some new data is available. diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py index 974d83bb8bf..d0e43a52176 100644 --- a/synapse/replication/http/devices.py +++ b/synapse/replication/http/devices.py @@ -20,6 +20,7 @@ import logging from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +import weakref from twisted.web.server import Request @@ -161,7 +162,7 @@ def __init__(self, hs: "HomeServer"): self.device_list_updater = hs.get_device_handler().device_list_updater - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self.clock = hs.get_clock() @staticmethod diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 6d53c6d1936..9cef4cfedb8 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -77,7 +77,7 @@ class ReplicationDataHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self.notifier = hs.get_notifier() self._reactor = hs.get_reactor() self._clock = hs.get_clock() diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 3611c678c20..bc173efe948 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -35,6 +35,7 @@ TypeVar, Union, ) +import weakref from prometheus_client import Counter @@ -229,12 +230,12 @@ def __init__(self, hs: "HomeServer"): # outgoing replication commands to.) self._connections: List[IReplicationConnection] = [] - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_replication_tcp_resource_total_connections", "", [], lambda: len(self._connections), - ) + )) # When POSITION or RDATA commands arrive, we stick them in a queue and process # them in order in a separate background process. @@ -252,7 +253,7 @@ def __init__(self, hs: "HomeServer"): # from that connection. self._streams_by_connection: Dict[IReplicationConnection, Set[str]] = {} - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_replication_tcp_command_queue", "Number of inbound RDATA/POSITION commands queued for processing", ["stream_name"], @@ -260,7 +261,7 @@ def __init__(self, hs: "HomeServer"): (stream_name,): len(queue) for stream_name, queue in self._command_queues_by_stream.items() }, - ) + )) self._is_master = hs.config.worker.worker_app is None @@ -400,7 +401,7 @@ def start_replication(self, hs: "HomeServer") -> None: reactor = hs.get_reactor() redis_config = hs.config.redis if redis_config.redis_path is not None: - reactor.connectUNIX( + self._connection = reactor.connectUNIX( redis_config.redis_path, self._factory, timeout=30, @@ -409,7 +410,7 @@ def start_replication(self, hs: "HomeServer") -> None: elif hs.config.redis.redis_use_tls: ssl_context_factory = ClientContextFactory(hs.config.redis) - reactor.connectSSL( + self._connection = reactor.connectSSL( redis_config.redis_host, redis_config.redis_port, self._factory, @@ -418,7 +419,7 @@ def start_replication(self, hs: "HomeServer") -> None: bindAddress=None, ) else: - reactor.connectTCP( + self._connection = reactor.connectTCP( redis_config.redis_host, redis_config.redis_port, self._factory, diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index fb9c539122b..892836a4f30 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -105,6 +105,9 @@ class IReplicationConnection(Interface): def send_command(cmd: Command) -> None: """Send the command down the connection""" + def close(self) -> None: + """Close the connection""" + @implementer(IReplicationConnection) class BaseReplicationStreamProtocol(LineOnlyReceiver): diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index c4601a61410..180daee9196 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -127,6 +127,9 @@ def connectionMade(self) -> None: super().connectionMade() run_as_background_process("subscribe-replication", self._send_subscribe) + def close(self) -> None: + pass + async def _send_subscribe(self) -> None: # it's important to make sure that we only send the REPLICATE command once we # have successfully subscribed to the stream - otherwise we might miss the @@ -275,7 +278,7 @@ def __init__( convertNumbers=convertNumbers, ) - hs.get_clock().looping_call(self._send_ping, 30 * 1000) + hs.register_looping_call(hs.get_clock().looping_call(self._send_ping, 30 * 1000)) @wrap_as_background_process("redis_ping") async def _send_ping(self) -> None: diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 0080a76f6f1..35401c90951 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -23,6 +23,7 @@ import logging import random from typing import TYPE_CHECKING, List, Optional, Tuple +import weakref from prometheus_client import Counter @@ -79,7 +80,7 @@ class ReplicationStreamer: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) self.clock = hs.get_clock() self.notifier = hs.get_notifier() self._instance_name = hs.get_instance_name() @@ -112,7 +113,7 @@ def __init__(self, hs: "HomeServer"): # # Note that if the position hasn't advanced then we won't send anything. if any(EventsStream.NAME == s.NAME for s in self.streams): - self.clock.looping_call(self.on_notifier_poke, 1000) + hs.register_looping_call(self.clock.looping_call(self.on_notifier_poke, 1000)) def on_notifier_poke(self) -> None: """Checks if there is actually any new data and sends it to the diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 25c15e5d486..87ac0a5ae17 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -77,6 +77,7 @@ __all__ = [ "STREAMS_MAP", "Stream", + "EventsStream", "BackfillStream", "PresenceStream", "PresenceFederationStream", @@ -87,6 +88,7 @@ "CachesStream", "DeviceListsStream", "ToDeviceStream", + "FederationStream", "AccountDataStream", "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 9694fff4fee..48f429d7f0a 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -31,6 +31,7 @@ Tuple, TypeVar, ) +import weakref import attr @@ -128,7 +129,7 @@ def __init__( update_function: callback go get stream updates, as above """ self.local_instance_name = local_instance_name - self.update_function = update_function + self.update_function = weakref.proxy(update_function) # The token from which we last asked for updates self.last_token = self.current_token(self.local_instance_name) @@ -561,7 +562,7 @@ class DeviceListsStreamRow: ROW_TYPE = DeviceListsStreamRow def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) super().__init__( hs.get_instance_name(), self._update_function, @@ -648,7 +649,7 @@ class AccountDataStreamRow: ROW_TYPE = AccountDataStreamRow def __init__(self, hs: "HomeServer"): - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) super().__init__( hs.get_instance_name(), self._update_function, @@ -740,7 +741,7 @@ class ThreadSubscriptionsStreamRow: ROW_TYPE = ThreadSubscriptionsStreamRow def __init__(self, hs: Any): - self.store = hs.get_datastores().main + self.store = weakref.proxy(hs.get_datastores().main) super().__init__( hs.get_instance_name(), self._update_function, diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py index 05b55fb0338..daece31f2e4 100644 --- a/synapse/replication/tcp/streams/events.py +++ b/synapse/replication/tcp/streams/events.py @@ -21,6 +21,7 @@ import heapq from collections import defaultdict from typing import TYPE_CHECKING, Iterable, Optional, Tuple, Type, TypeVar, cast +import weakref import attr @@ -151,7 +152,7 @@ class EventsStream(_StreamFromIdGen): NAME = "events" def __init__(self, hs: "HomeServer"): - self._store = hs.get_datastores().main + self._store = weakref.proxy(hs.get_datastores().main) super().__init__( hs.get_instance_name(), self._update_function, self._store._stream_id_gen ) diff --git a/synapse/replication/tcp/streams/federation.py b/synapse/replication/tcp/streams/federation.py index 1c2ffe86b7b..014c9c36861 100644 --- a/synapse/replication/tcp/streams/federation.py +++ b/synapse/replication/tcp/streams/federation.py @@ -19,6 +19,7 @@ # # from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Tuple +import weakref import attr diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 54a3397f494..f76d23ea56c 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -127,11 +127,13 @@ def __init__(self, hs: "HomeServer"): self._well_known_builder = WellKnownBuilder(hs) self._address_ratelimiter = Ratelimiter( + hs=hs, store=self._main_store, clock=hs.get_clock(), cfg=self.hs.config.ratelimiting.rc_login_address, ) self._account_ratelimiter = Ratelimiter( + hs=hs, store=self._main_store, clock=hs.get_clock(), cfg=self.hs.config.ratelimiting.rc_login_account, diff --git a/synapse/rest/client/login_token_request.py b/synapse/rest/client/login_token_request.py index a053db8e551..d86b87e190a 100644 --- a/synapse/rest/client/login_token_request.py +++ b/synapse/rest/client/login_token_request.py @@ -79,6 +79,7 @@ def __init__(self, hs: "HomeServer"): # This endpoint can be used to spawn additional sessions and could be # abused by a malicious client to create many sessions. self._ratelimiter = Ratelimiter( + hs=hs, store=self._main_store, clock=hs.get_clock(), cfg=RatelimitSettings( diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index 1ee2a1b7cf9..0428012ee43 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -54,6 +54,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for presence updates, keyed by requester. self._presence_per_user_limiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_presence_per_user, diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 6c790307568..d17eb771964 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -390,6 +390,7 @@ def __init__(self, hs: "HomeServer"): self.hs = weakref.proxy(hs) self.store = hs.get_datastores().main self.ratelimiter = Ratelimiter( + hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_registration_token_validity, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 296c5acfd93..5c3d076173c 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -132,6 +132,7 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for presence updates, keyed by requester. self._presence_per_user_limiter = Ratelimiter( + hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_presence_per_user, diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index e3dbbcbf472..fc541cce222 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -57,7 +57,7 @@ def __init__(self, hs: "HomeServer"): ] = {} # Try to clean entries every 30 mins. This means entries will exist # for at *LEAST* 30 mins, and at *MOST* 60 mins. - self.cleaner = self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS) + hs.register_looping_call(self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS)) def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hashable: """A helper function which returns a transaction key that can be used diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index e45df11c9fa..b832e4bc0e4 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -49,6 +49,7 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): # A rate limiter for creating new media IDs. self._create_media_rate_limiter = Ratelimiter( + hs=hs, store=hs.get_datastores().main, clock=self.clock, cfg=hs.config.ratelimiting.rc_media_create, diff --git a/synapse/server.py b/synapse/server.py index 149844aab58..d187748f806 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1,6 +1,4 @@ # -# This file is licensed under the Affero General Public License (AGPL) version 3. -# # Copyright 2021 The Matrix.org Foundation C.I.C. # Copyright (C) 2023-2024 New Vector, Ltd # @@ -29,7 +27,9 @@ import functools import logging from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type, TypeVar, cast +import weakref +from twisted.internet import defer from typing_extensions import TypeAlias from twisted.internet.interfaces import IOpenSSLContextFactory @@ -129,7 +129,7 @@ ) from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.media.media_repository import MediaRepository -from synapse.metrics import register_threadpool +from synapse.metrics import LaterGauge, register_threadpool from synapse.metrics.common_usage_metrics import CommonUsageMetricsManager from synapse.module_api import ModuleApi from synapse.module_api.callbacks import ModuleApiCallbacks @@ -216,7 +216,15 @@ def cache_in_self(builder: F) -> F: @functools.wraps(builder) def _get(self: "HomeServer") -> T: try: - return getattr(self, depname) + dep = getattr(self, depname) + try: + return_dep = weakref.proxy(dep) + except: + if isinstance(dep, dict): + return_dep = {k: weakref.proxy(v) for k, v in dep.items()} + else: + logger.warning("Failed creating proxy for %s", depname) + return return_dep except AttributeError: pass @@ -230,8 +238,17 @@ def _get(self: "HomeServer") -> T: setattr(self, depname, dep) finally: building[0] = False + + return_dep = dep + try: + return_dep = weakref.proxy(dep) + except: + if isinstance(dep, dict): + return_dep = {k: weakref.proxy(v) for k, v in dep.items()} + else: + logger.warning("Failed creating proxy for %s", depname) - return dep + return return_dep return cast(F, _get) @@ -314,29 +331,70 @@ def __init__( self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None self._looping_calls: List[LoopingCall] = [] - - def __del__(self) -> None: - logger.warning("Destructing HomeServer") + self._later_gauges: List[LaterGauge] = [] + self._background_processes: List[defer.Deferred] = [] def shutdown(self) -> None: logger.info("Received shutdown request") - # logger.info("Unsubscribing replication callbacks") - # self.get_replication_notifier()._replication_callbacks.clear() + logger.info("Unsubscribing replication callbacks") + self.get_replication_notifier()._replication_callbacks.clear() + + logger.info("Stopping replication") + for connection in self.get_replication_command_handler()._connections: + connection.close() + self.get_replication_command_handler()._connections.clear() + + logger.info("Stopping replication streams") + # self.get_replication_command_handler()._factory.continueTrying = False + # self.get_replication_command_handler()._factory.stopFactory() + # self.get_replication_command_handler()._factory.stopTrying() + # self.get_replication_command_handler()._connection.disconnect() + # self.get_replication_command_handler()._factory.synapse_outbound_redis_connection.disconnect() + # self.get_replication_command_handler()._factory = None + self.get_replication_command_handler()._streams.clear() + self.get_replication_command_handler()._streams_to_replicate.clear() + STREAMS_MAP.clear() + + from synapse.util.batching_queue import number_of_keys + number_of_keys.clear() + from synapse.util.batching_queue import number_queued + number_queued.clear() + + # TODO: unregister all metrics that bind to HS resources! + logger.info("Stopping later gauges: %d", len(self._later_gauges)) + for later_gauge in self._later_gauges: + later_gauge.unregister() + self._later_gauges.clear() + + logger.info("Unregistering db background updates") + for db in self.get_datastores().databases: + db.stop_background_updates() logger.info("Stopping looping calls: %d", len(self._looping_calls)) for looping_call in self._looping_calls: looping_call.stop() - del looping_call + self._looping_calls.clear() + + self._looping_calls.clear() - self._looping_calls = [] + logger.info("Stopping background processes: %d", len(self._background_processes)) + for process in self._background_processes: + process.cancel() + self._background_processes.clear() - logger.info("Shutting down datastores") - self.get_datastores().main.shutdown() + for call in self.get_reactor().getDelayedCalls(): + call.cancel() def register_looping_call(self, looping_call: LoopingCall) -> None: self._looping_calls.append(looping_call) + def register_later_gauge(self, later_gauge: LaterGauge) -> None: + self._later_gauges.append(later_gauge) + + def register_background_process(self, process: defer.Deferred) -> None: + self._background_processes.append(process) + def register_module_web_resource(self, path: str, resource: Resource) -> None: """Allows a module to register a web resource to be served at the given path. @@ -454,6 +512,7 @@ def get_distributor(self) -> Distributor: @cache_in_self def get_registration_ratelimiter(self) -> Ratelimiter: return Ratelimiter( + hs=self, store=self.get_datastores().main, clock=self.get_clock(), cfg=self.config.ratelimiting.rc_registration, @@ -970,6 +1029,7 @@ def should_send_federation(self) -> bool: @cache_in_self def get_request_ratelimiter(self) -> RequestRatelimiter: return RequestRatelimiter( + self, self.get_datastores().main, self.get_clock(), self.config.ratelimiting.rc_message, diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 1def1a0469e..31e122ad942 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -641,6 +641,7 @@ def __init__(self, hs: "HomeServer"): # dict of set of event_ids -> _StateCacheEntry. self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( ExpiringCache( + hs=hs, cache_name="state_cache", server_name=self.server_name, clock=self.clock, @@ -660,7 +661,7 @@ def __init__(self, hs: "HomeServer"): _StateResMetrics ) - self.clock.looping_call(self._report_metrics, 120 * 1000) + hs.register_looping_call(self.clock.looping_call(self._report_metrics, 120 * 1000)) async def resolve_state_groups( self, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 568a816151b..e912bdca472 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -55,6 +55,7 @@ def __init__( db_conn: LoggingDatabaseConnection, hs: "HomeServer", ): + self.hs = weakref.proxy(hs) self.server_name = hs.hostname # nb must be called this for @cached self._clock = hs.get_clock() self.database_engine = database.engine diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 825373bed40..5f8a9433c9b 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -248,7 +248,7 @@ class BackgroundUpdater: def __init__(self, hs: "HomeServer", database: "DatabasePool"): self._clock = hs.get_clock() - self.db_pool = database + self.db_pool = weakref.proxy(database) self.hs = weakref.proxy(hs) self._database_name = database.name() @@ -395,9 +395,9 @@ def start_doing_background_updates(self) -> None: # if we start a new background update, not all updates are done. self._all_done = False sleep = self.sleep_enabled - run_as_background_process( + self.hs.register_background_process(run_as_background_process( "background_updates", self.run_background_updates, sleep - ) + )) async def run_background_updates(self, sleep: bool) -> None: if self._running or not self.enabled: diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index e01badac4dd..d3ab02a8539 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -198,7 +198,7 @@ def __init__( """ self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {} self._currently_persisting_rooms: Set[str] = set() - self._per_item_callback = per_item_callback + self._per_item_callback = weakref.proxy(per_item_callback) async def add_to_queue( self, diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index df3f264b062..8acc98f0b63 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -49,9 +49,9 @@ def __init__(self, hs: "HomeServer", stores: Databases): self.stores = stores if hs.config.worker.run_background_tasks: - self._delete_state_loop_call = hs.get_clock().looping_call( + hs.register_looping_call(hs.get_clock().looping_call( self._delete_state_groups_loop, 60 * 1000 - ) + )) self.stores.state.db_pool.updates.register_background_update_handler( _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE, diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 69d496f98c0..771272e59de 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -38,6 +38,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase +from synapse.http import proxy from synapse.logging.opentracing import tag_args, trace from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.roommember import ProfileInfo @@ -72,9 +73,9 @@ def __init__(self, hs: "HomeServer", stores: "Databases"): self.server_name = hs.hostname # nb must be called this for @cached self.hs = weakref.proxy(hs) self._clock = hs.get_clock() - self.stores = stores - self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) - self._partial_state_room_tracker = PartialCurrentStateTracker(stores.main) + self.stores = weakref.proxy(stores) + self._partial_state_events_tracker = weakref.proxy(PartialStateEventsTracker(stores.main)) + self._partial_state_room_tracker = weakref.proxy(PartialCurrentStateTracker(stores.main)) # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index f30737c510b..e624cd85104 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -568,12 +568,12 @@ def __init__( self._db_pool = make_pool(hs.get_reactor(), database_config, engine) self.updates = BackgroundUpdater(hs, self) - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_background_update_status", "Background update status", [], self.updates.get_status, - ) + )) self._previous_txn_total_time = 0.0 self._current_txn_total_time = 0.0 @@ -606,6 +606,10 @@ def __init__( self._check_safe_to_upsert, ) + def stop_background_updates(self) -> None: + self.updates.enabled = False + self.updates._background_update_handlers.clear() + def name(self) -> str: "Return the name of this database" return self._database_config.name @@ -669,7 +673,7 @@ def loop() -> None: "Total database time: %.3f%% {%s}", ratio * 100, top_three_counters ) - self._clock.looping_call(loop, 10000) + self.hs.register_looping_call(self._clock.looping_call(loop, 10000)) def new_transaction( self, diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 65cf81544e0..8a1b5757292 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -173,13 +173,10 @@ def __init__( ): self.hs = weakref.proxy(hs) self._clock = hs.get_clock() - self.database_engine = database.engine + self.database_engine = weakref.proxy(database.engine) super().__init__(database, db_conn, hs) - def shutdown(self) -> None: - super().shutdown() - async def get_users_paginate( self, start: int, diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index dc37f671101..7bee8a1a42c 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -794,6 +794,7 @@ async def _clean_up_cache_invalidation_wrapper(self) -> None: else: next_interval = REGULAR_CLEANUP_INTERVAL_MS + # TODO: (devon) what is the best way to track these manually looping calls? self.hs.get_clock().call_later( next_interval / 1000, self._clean_up_cache_invalidation_wrapper ) diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py index 5b15fd707d2..1c03874ab41 100644 --- a/synapse/storage/databases/main/censor_events.py +++ b/synapse/storage/databases/main/censor_events.py @@ -54,7 +54,7 @@ def __init__( hs.config.worker.run_background_tasks and self.hs.config.server.redaction_retention_period is not None ): - hs.get_clock().looping_call(self._censor_redactions, 5 * 60 * 1000) + hs.register_looping_call(hs.get_clock().looping_call(self._censor_redactions, 5 * 60 * 1000)) @wrap_as_background_process("_censor_redactions") async def _censor_redactions(self) -> None: diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index cf7bc4ac693..35f4bda7a9f 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -441,7 +441,7 @@ def __init__( ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: - self._clock.looping_call(self._prune_old_user_ips, 5 * 1000) + hs.register_looping_call(self._clock.looping_call(self._prune_old_user_ips, 5 * 1000)) if self._update_on_this_worker: # This is the designated worker that can write to the client IP @@ -452,9 +452,9 @@ def __init__( Tuple[str, str, str], Tuple[str, Optional[str], int] ] = {} - self._client_ip_looper = self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._update_client_ips_batch, 5 * 1000 - ) + )) self.hs.get_reactor().addSystemEventTrigger( "before", "shutdown", self._update_client_ips_batch ) diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index da10afbebec..18484c10d8f 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -93,6 +93,7 @@ def __init__( self._last_device_delete_cache: ExpiringCache[ Tuple[str, Optional[str]], int ] = ExpiringCache( + hs=hs, cache_name="last_device_delete_cache", server_name=self.server_name, clock=self._clock, @@ -152,12 +153,12 @@ def __init__( ) if hs.config.worker.run_background_tasks: - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( run_as_background_process, DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS, "_delete_old_federation_inbox_rows", self._delete_old_federation_inbox_rows, - ) + )) def process_replication_rows( self, diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 6ed9f858003..81a06e44dc0 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -194,9 +194,9 @@ def __init__( ) if hs.config.worker.run_background_tasks: - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._prune_old_outbound_device_pokes, 60 * 60 * 1000 - ) + )) def process_replication_rows( self, stream_name: str, instance_name: str, token: int, rows: Iterable[Any] diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 7224d57fff9..c6b70721bcf 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -143,9 +143,9 @@ def __init__( self.hs = weakref.proxy(hs) if hs.config.worker.run_background_tasks: - hs.get_clock().looping_call( + hs.register_looping_call(hs.get_clock().looping_call( self._delete_old_forward_extrem_cache, 60 * 60 * 1000 - ) + )) # Cache of event ID to list of auth event IDs and their depths. self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache( @@ -159,7 +159,7 @@ def __init__( # index. self.tests_allow_no_chain_cover_index = True - self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000) + hs.register_looping_call(self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000)) if isinstance(self.database_engine, PostgresEngine): self.db_pool.updates.register_background_validate_constraint_and_delete_rows( diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 430a75b3f65..32a7d51692f 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -276,22 +276,22 @@ def __init__( self._find_stream_orderings_for_times_txn(cur) cur.close() - self.find_stream_orderings_looping_call = self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._find_stream_orderings_for_times, 10 * 60 * 1000 - ) + )) self._rotate_count = 10000 self._doing_notif_rotation = False if hs.config.worker.run_background_tasks: - self._background_tasks.append( + hs.register_looping_call( self._clock.looping_call( self._rotate_notifs, 30 * 1000 ) ) - self._clear_old_staging_loop = self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._clear_old_push_actions_staging, 30 * 60 * 1000 - ) + )) self.db_pool.updates.register_background_index_update( "event_push_summary_unique_index2", @@ -330,11 +330,6 @@ def __init__( columns=["room_id"], ) - def __del__(self) -> None: - logger.warning("Stopping background tasks") - for task in self._background_tasks: - task.stop() - async def _background_drop_null_thread_id_indexes( self, progress: JsonDict, batch_size: int ) -> int: @@ -1927,9 +1922,6 @@ def __init__( where_clause="highlight=0", ) - def __del__(self) -> None: - logger.warning("Destructor") - def _action_has_highlight(actions: Collection[Union[Mapping, str]]) -> bool: for action in actions: if not isinstance(action, dict): diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index bdb90d8473c..92e1829e7f6 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -238,8 +238,8 @@ def __init__( db_conn: LoggingDatabaseConnection, ): self.hs = weakref.proxy(hs) - self.db_pool = db - self.store = main_data_store + self.db_pool = weakref.proxy(db) + self.store = weakref.proxy(main_data_store) self.database_engine = db.engine self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 7927842d0f0..b4cc57a28cc 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -277,10 +277,10 @@ def __init__( if hs.config.worker.run_background_tasks: # We periodically clean out old transaction ID mappings - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._cleanup_old_transaction_ids, 5 * 60 * 1000, - ) + )) self._get_event_cache: AsyncLruCache[Tuple[str], EventCacheEntry] = ( AsyncLruCache( diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index 8277ad8c33c..9069a48df04 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -103,9 +103,9 @@ def __init__( self._acquiring_locks: Set[Tuple[str, str]] = set() - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._reap_stale_read_write_locks, _LOCK_TIMEOUT_MS / 10.0 - ) + )) @wrap_as_background_process("LockStore._on_shutdown") async def _on_shutdown(self) -> None: diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index dadf1739e61..73be6dc573d 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -76,14 +76,11 @@ def __init__( # Read the extrems every 60 minutes if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000) + hs.register_looping_call(self._clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000)) # Used in _generate_user_daily_visits to keep track of progress self._last_user_visit_update = self._get_start_of_day() - def __del__(self) -> None: - logger.warning("Destructor") - @wrap_as_background_process("read_forward_extremities") async def _read_forward_extremities(self) -> None: def fetch(txn: LoggingTransaction) -> List[Tuple[int, int]]: diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 320d29e4748..8140832aee4 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -226,9 +226,9 @@ def __init__( # Create a background job for culling expired 3PID validity tokens if hs.config.worker.run_background_tasks: - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self.cull_expired_threepid_validation_tokens, THIRTY_MINUTES_IN_MS - ) + )) async def register_user( self, @@ -2727,9 +2727,9 @@ def __init__( # Create a background job for removing expired login tokens if hs.config.worker.run_background_tasks: - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._delete_expired_login_tokens, THIRTY_MINUTES_IN_MS - ) + )) async def add_access_token_to_user( self, diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index ef325167f0e..2c4f1379d2d 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -111,28 +111,21 @@ def __init__( ): self._known_servers_count = 1 self._count_known_servers_task: Optional[LoopingCall] = ( - self.hs.get_clock().looping_call( + hs.register_looping_call(self.hs.get_clock().looping_call( self._count_known_servers, 6 * 1000, - ) + )) ) self.hs.get_clock().call_later( 1, self._count_known_servers, ) - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_federation_known_servers", "", [], lambda: self._known_servers_count, - ) - - def shutdown(self) -> None: - logger.info("Shutting down RoomMemberWorkerStore") - # print(self.hs.get_reactor().getDelayedCalls()) - if self._count_known_servers_task is not None: - self._count_known_servers_task.stop() - self._count_known_servers_task = None + )) @wrap_as_background_process("_count_known_servers") async def _count_known_servers(self) -> int: diff --git a/synapse/storage/databases/main/session.py b/synapse/storage/databases/main/session.py index 8a1331d4c8e..419b58c8f67 100644 --- a/synapse/storage/databases/main/session.py +++ b/synapse/storage/databases/main/session.py @@ -55,7 +55,7 @@ def __init__( # Create a background job for culling expired sessions. if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000) + hs.register_looping_call(self._clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000)) async def create_session( self, session_type: str, value: JsonDict, expiry_ms: int diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index bfc324b80d2..fd9e9001bce 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -81,7 +81,7 @@ def __init__( super().__init__(database, db_conn, hs) if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000) + hs.register_looping_call(self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000)) @wrap_as_background_process("cleanup_transactions") async def _cleanup_transactions(self) -> None: diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index 026a0517d20..fe7508a13e5 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -41,6 +41,7 @@ Union, cast, ) +import weakref import attr from sortedcontainers import SortedList, SortedSet @@ -220,7 +221,7 @@ def __init__( writers: List[str], positive: bool = True, ) -> None: - self._db = db + self._db = weakref.proxy(db) self._notifier = notifier self._stream_name = stream_name self._instance_name = instance_name diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 3fb697751fc..2e0b7803e41 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -31,6 +31,7 @@ Tuple, TypeVar, ) +import weakref from prometheus_client import Gauge @@ -107,12 +108,13 @@ def __init__( self._next_values: Dict[Hashable, List[Tuple[V, defer.Deferred]]] = {} # The function to call with batches of values. - self._process_batch_callback = process_batch_callback + self._process_batch_callback = weakref.proxy(process_batch_callback) number_queued.labels(self._name).set_function( lambda: sum(len(q) for q in self._next_values.values()) ) + # TODO: (devon) what do with this global? number_of_keys.labels(self._name).set_function(lambda: len(self._next_values)) self._number_in_flight_metric: Gauge = number_in_flight.labels(self._name) diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 4be4c6f01b1..c29781cd3d4 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -21,7 +21,7 @@ import logging from collections import OrderedDict -from typing import Any, Generic, Iterable, Literal, Optional, TypeVar, Union, overload +from typing import TYPE_CHECKING, Any, Generic, Iterable, Literal, Optional, TypeVar, Union, overload import attr @@ -32,6 +32,9 @@ from synapse.util import Clock from synapse.util.caches import EvictionReason, register_cache +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -47,6 +50,7 @@ class ExpiringCache(Generic[KT, VT]): def __init__( self, *, + hs: "HomeServer", cache_name: str, server_name: str, clock: Clock, @@ -101,7 +105,7 @@ def __init__( def f() -> "defer.Deferred[None]": return run_as_background_process("prune_cache", self._prune_cache) - self._clock.looping_call(f, self._expiry_ms / 2) + hs.register_looping_call(self._clock.looping_call(f, self._expiry_ms / 2)) def __setitem__(self, key: KT, value: VT) -> None: now = self._clock.time_msec() diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 466362e79ce..1ba1c5e650a 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -235,13 +235,13 @@ def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: USE_GLOBAL_LIST = True clock = hs.get_clock() - clock.looping_call( + hs.register_looping_call(clock.looping_call( _expire_old_entries, 30 * 1000, clock, expiry_time, hs.config.caches.cache_autotuning, - ) + )) class _Node(Generic[KT, VT]): diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index a2d21d7184b..4d7a3749242 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -102,7 +102,7 @@ class TaskScheduler: def __init__(self, hs: "HomeServer"): self._hs = weakref.proxy(hs) - self._store = hs.get_datastores().main + self._store = weakref.proxy(hs.get_datastores().main) self._clock = hs.get_clock() self._running_tasks: Set[str] = set() # A map between action names and their registered function @@ -119,21 +119,21 @@ def __init__(self, hs: "HomeServer"): self._launching_new_tasks = False if self._run_background_tasks: - self._clock.looping_call( + hs.register_looping_call(self._clock.looping_call( self._launch_scheduled_tasks, TaskScheduler.SCHEDULE_INTERVAL_MS, - ) - self._clock.looping_call( + )) + hs.register_looping_call(self._clock.looping_call( self._clean_scheduled_tasks, TaskScheduler.SCHEDULE_INTERVAL_MS, - ) + )) - LaterGauge( + hs.register_later_gauge(LaterGauge( "synapse_scheduler_running_tasks", "The number of concurrent running tasks handled by the TaskScheduler", labels=None, caller=lambda: len(self._running_tasks), - ) + )) def register_action( self, @@ -436,6 +436,7 @@ async def wrapper() -> None: log_context, start_time, ) + self._hs.register_looping_call(occasional_status_call) try: (status, result, error) = await function(task) except Exception: diff --git a/tests/federation/test_federation_catch_up.py b/tests/federation/test_federation_catch_up.py index 1e1ed8e6426..cab769cf789 100644 --- a/tests/federation/test_federation_catch_up.py +++ b/tests/federation/test_federation_catch_up.py @@ -1,6 +1,7 @@ from typing import Callable, Collection, List, Optional, Tuple from unittest import mock from unittest.mock import AsyncMock, Mock +import weakref from twisted.test.proto_helpers import MemoryReactor diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index 75bf50e644b..7a862bfec56 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -33,6 +33,7 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_get_set(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( + hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), @@ -46,6 +47,7 @@ def test_get_set(self) -> None: def test_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( + hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), @@ -65,6 +67,7 @@ def test_eviction(self) -> None: def test_iterable_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, List[int]] = ExpiringCache( + hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), @@ -89,6 +92,7 @@ def test_iterable_eviction(self) -> None: def test_time_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, int] = ExpiringCache( + hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), From 61508a6c26a89875431627feabd41f72a7725ef8 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 6 Aug 2025 15:18:48 -0600 Subject: [PATCH 003/181] Temporarily disable all tests that call generate_latest --- tests/metrics/test_metrics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index e92d5f6dfa1..ada59531850 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -130,8 +130,10 @@ def get_metrics_from_gauge( return results +RUN_TEST = False class BuildInfoTests(unittest.TestCase): + @unittest.skip_unless(RUN_TEST, "must be enabled") def test_get_build(self) -> None: """ The synapse_build_info metric reports the OS version, Python version, @@ -150,6 +152,7 @@ def test_get_build(self) -> None: class CacheMetricsTests(unittest.HomeserverTestCase): + @unittest.skip_unless(RUN_TEST, "must be enabled") def test_cache_metric(self) -> None: """ Caches produce metrics reflecting their state when scraped. From df7b437f128903644c4bc6cc950e4ae9c350ef54 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 6 Aug 2025 15:21:25 -0600 Subject: [PATCH 004/181] Revert "Temporarily disable all tests that call generate_latest" This reverts commit 61508a6c26a89875431627feabd41f72a7725ef8. --- tests/metrics/test_metrics.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index ada59531850..e92d5f6dfa1 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -130,10 +130,8 @@ def get_metrics_from_gauge( return results -RUN_TEST = False class BuildInfoTests(unittest.TestCase): - @unittest.skip_unless(RUN_TEST, "must be enabled") def test_get_build(self) -> None: """ The synapse_build_info metric reports the OS version, Python version, @@ -152,7 +150,6 @@ def test_get_build(self) -> None: class CacheMetricsTests(unittest.HomeserverTestCase): - @unittest.skip_unless(RUN_TEST, "must be enabled") def test_cache_metric(self) -> None: """ Caches produce metrics reflecting their state when scraped. From 5078451bdc4d8268378d785d89842e088b93118b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 14 Aug 2025 14:26:30 -0600 Subject: [PATCH 005/181] Cleanup SynapseHomeServer without weakrefs --- synapse/api/auth/base.py | 3 +- synapse/api/auth_blocking.py | 5 +- synapse/api/filtering.py | 5 +- synapse/api/ratelimiting.py | 3 +- synapse/app/_base.py | 15 +++-- synapse/app/homeserver.py | 2 +- synapse/crypto/keyring.py | 3 +- synapse/federation/federation_base.py | 3 +- synapse/federation/persistence.py | 3 +- synapse/federation/send_queue.py | 3 +- synapse/federation/sender/__init__.py | 5 +- synapse/federation/transport/client.py | 3 +- .../federation/transport/server/__init__.py | 3 +- synapse/federation/transport/server/_base.py | 7 +- synapse/handlers/account.py | 3 +- synapse/handlers/account_validity.py | 3 +- synapse/handlers/admin.py | 3 +- synapse/handlers/appservice.py | 3 +- synapse/handlers/auth.py | 5 +- synapse/handlers/cas.py | 3 +- synapse/handlers/deactivate_account.py | 3 +- synapse/handlers/device.py | 7 +- synapse/handlers/devicemessage.py | 3 +- synapse/handlers/directory.py | 3 +- synapse/handlers/e2e_keys.py | 3 +- synapse/handlers/event_auth.py | 3 +- synapse/handlers/events.py | 3 +- synapse/handlers/federation.py | 3 +- synapse/handlers/federation_event.py | 3 +- synapse/handlers/identity.py | 3 +- synapse/handlers/initial_sync.py | 3 +- synapse/handlers/jwt.py | 3 +- synapse/handlers/message.py | 3 +- synapse/handlers/pagination.py | 3 +- synapse/handlers/presence.py | 27 ++------ synapse/handlers/profile.py | 3 +- synapse/handlers/receipts.py | 3 +- synapse/handlers/register.py | 8 +-- synapse/handlers/reports.py | 3 +- synapse/handlers/room.py | 7 +- synapse/handlers/room_list.py | 3 +- synapse/handlers/room_member.py | 5 +- synapse/handlers/room_policy.py | 3 +- synapse/handlers/search.py | 3 +- synapse/handlers/send_email.py | 3 +- synapse/handlers/sliding_sync/room_lists.py | 3 +- synapse/handlers/sso.py | 3 +- synapse/handlers/stats.py | 3 +- synapse/handlers/sync.py | 1 - synapse/handlers/typing.py | 7 +- synapse/handlers/ui_auth/checkers.py | 5 +- synapse/handlers/user_directory.py | 3 +- synapse/http/client.py | 3 +- synapse/http/matrixfederationclient.py | 3 +- synapse/http/server.py | 5 +- synapse/http/site.py | 3 +- synapse/media/media_repository.py | 3 +- synapse/media/media_storage.py | 5 +- synapse/media/storage_provider.py | 3 +- synapse/media/thumbnailer.py | 3 +- synapse/metrics/common_usage_metrics.py | 3 +- synapse/module_api/__init__.py | 5 +- synapse/notifier.py | 3 +- synapse/push/__init__.py | 3 +- synapse/push/bulk_push_rule_evaluator.py | 3 +- synapse/push/mailer.py | 3 +- synapse/push/pusher.py | 3 +- synapse/push/pusherpool.py | 3 +- synapse/replication/http/_base.py | 3 +- synapse/replication/http/devices.py | 3 +- synapse/replication/tcp/client.py | 5 +- synapse/replication/tcp/handler.py | 1 - synapse/replication/tcp/resource.py | 3 +- synapse/replication/tcp/streams/_base.py | 9 ++- synapse/replication/tcp/streams/events.py | 3 +- synapse/replication/tcp/streams/federation.py | 1 - synapse/rest/admin/devices.py | 7 +- synapse/rest/admin/experimental_features.py | 3 +- synapse/rest/admin/media.py | 5 +- synapse/rest/admin/rooms.py | 9 ++- synapse/rest/admin/server_notice_servlet.py | 3 +- synapse/rest/admin/users.py | 27 ++++---- synapse/rest/client/account.py | 21 +++--- synapse/rest/client/account_data.py | 9 ++- synapse/rest/client/account_validity.py | 5 +- synapse/rest/client/auth.py | 3 +- synapse/rest/client/capabilities.py | 3 +- synapse/rest/client/devices.py | 13 ++-- synapse/rest/client/filter.py | 5 +- synapse/rest/client/keys.py | 3 +- synapse/rest/client/login.py | 3 +- synapse/rest/client/media.py | 5 +- synapse/rest/client/presence.py | 3 +- synapse/rest/client/profile.py | 5 +- synapse/rest/client/pusher.py | 5 +- synapse/rest/client/register.py | 13 ++-- synapse/rest/client/reporting.py | 7 +- synapse/rest/client/room.py | 9 ++- .../rest/client/room_upgrade_rest_servlet.py | 3 +- synapse/rest/client/sendtodevice.py | 3 +- synapse/rest/client/sync.py | 5 +- synapse/rest/client/transactions.py | 3 +- synapse/rest/client/user_directory.py | 3 +- synapse/rest/client/voip.py | 3 +- synapse/rest/consent/consent_resource.py | 3 +- synapse/rest/media/download_resource.py | 3 +- synapse/rest/media/thumbnail_resource.py | 3 +- synapse/server.py | 64 +++++++++++++------ .../server_notices/server_notices_manager.py | 3 +- synapse/state/__init__.py | 3 +- synapse/storage/_base.py | 3 +- synapse/storage/background_updates.py | 5 +- synapse/storage/controllers/persist_events.py | 5 +- synapse/storage/controllers/state.py | 9 ++- synapse/storage/database.py | 3 +- synapse/storage/databases/main/__init__.py | 5 +- synapse/storage/databases/main/client_ips.py | 4 +- .../databases/main/event_federation.py | 3 +- synapse/storage/databases/main/events.py | 7 +- synapse/storage/databases/main/lock.py | 6 +- .../databases/main/monthly_active_users.py | 3 +- synapse/storage/databases/main/presence.py | 3 +- synapse/storage/util/id_generators.py | 3 +- synapse/util/batching_queue.py | 3 +- synapse/util/metrics.py | 3 + synapse/util/task_scheduler.py | 5 +- tests/federation/test_federation_catch_up.py | 1 - tests/handlers/test_appservice.py | 3 +- tests/handlers/test_room_policy.py | 3 +- tests/handlers/test_room_summary.py | 5 +- .../databases/main/test_events_worker.py | 3 +- 131 files changed, 268 insertions(+), 386 deletions(-) diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 51cddd8fb03..f97a71caf7c 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -19,7 +19,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, Optional, Tuple from netaddr import IPAddress @@ -53,7 +52,7 @@ class BaseAuth: """Common base class for all auth implementations.""" def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index fbc442518df..19a46028997 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Optional from synapse.api.constants import LimitBlockingTypes, UserTypes @@ -36,7 +35,7 @@ class AuthBlocking: def __init__(self, hs: "HomeServer"): - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main self._server_notices_mxid = hs.config.servernotices.server_notices_mxid self._hs_disabled = hs.config.server.hs_disabled @@ -47,7 +46,7 @@ def __init__(self, hs: "HomeServer"): self._mau_limits_reserved_threepids = ( hs.config.server.mau_limits_reserved_threepids ) - self.hs = weakref.proxy(hs) + self.hs = hs self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips async def check_auth_blocking( diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 1dcce813838..34dd12368a0 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -21,7 +21,6 @@ # # import json -import weakref from typing import ( TYPE_CHECKING, Awaitable, @@ -153,7 +152,7 @@ def matrix_user_id_validator(user_id: object) -> bool: class Filtering: def __init__(self, hs: "HomeServer"): - self._hs = weakref.proxy(hs) + self._hs = hs self.store = hs.get_datastores().main self.DEFAULT_FILTER_COLLECTION = FilterCollection(hs, {}) @@ -321,7 +320,7 @@ def blocks_all_room_timeline(self) -> bool: class Filter: def __init__(self, hs: "HomeServer", filter_json: JsonMapping): - self._hs = weakref.proxy(hs) + self._hs = hs self._store = hs.get_datastores().main self.filter_json = filter_json diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index 47a75e0ef63..79d1d1cce62 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -21,7 +21,6 @@ # from typing import TYPE_CHECKING, Dict, Hashable, Optional, Tuple -import weakref from synapse.api.errors import LimitExceededError from synapse.config.ratelimiting import RatelimitSettings @@ -86,7 +85,7 @@ def __init__( self.clock = clock self.rate_hz = cfg.per_second self.burst_count = cfg.burst_count - self.store = weakref.proxy(store) + self.store = store self._limiter_name = cfg.key self._ratelimit_callbacks = ratelimit_callbacks diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 89eb5d0e23a..6472c426e16 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -552,6 +552,7 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: signal.signal(signal.SIGHUP, run_sighup) + # TODO: FIXME, weakref needed here register_sighup(refresh_certificate, weakref.proxy(hs)) register_sighup(reload_cache_config, hs.config) @@ -608,13 +609,13 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: # rest of time. Doing so means less work each GC (hopefully). # # PyPy does not (yet?) implement gc.freeze() - if hasattr(gc, "freeze"): - gc.collect() - gc.freeze() - - # Speed up shutdowns by freezing all allocated objects. This moves everything - # into the permanent generation and excludes them from the final GC. - atexit.register(gc.freeze) + # if hasattr(gc, "freeze"): + # gc.collect() + # gc.freeze() + # + # # Speed up shutdowns by freezing all allocated objects. This moves everything + # # into the permanent generation and excludes them from the final GC. + # atexit.register(gc.freeze) def reload_cache_config(config: HomeServerConfig) -> None: diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 42bd6007178..57a15ef2a65 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -90,7 +90,7 @@ def shutdown(self) -> None: listener.loseConnection() self._listening_services.clear() - self._reactor.stop() + #self._reactor.stop() def _listener_http( self, diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 41f327f4b05..df607533cb3 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -21,7 +21,6 @@ import abc import logging -import weakref from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple import attr @@ -176,7 +175,7 @@ def __init__( process_batch_callback=self._inner_fetch_key_requests, ) - self.hs = weakref.proxy(hs) + self.hs = hs # build a FetchKeyResult for each of our own keys, to shortcircuit the # fetcher. diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index a95379f4fd9..5d7e678221a 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -20,7 +20,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional, Sequence from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership @@ -57,7 +56,7 @@ def __init__(self, message: str, event_id: str): class FederationBase: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.keyring = hs.get_keyring() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker diff --git a/synapse/federation/persistence.py b/synapse/federation/persistence.py index 108811481aa..8340b485031 100644 --- a/synapse/federation/persistence.py +++ b/synapse/federation/persistence.py @@ -28,7 +28,6 @@ import logging from typing import Optional, Tuple -import weakref from synapse.federation.units import Transaction from synapse.storage.databases.main import DataStore @@ -41,7 +40,7 @@ class TransactionActions: """Defines persistence actions that relate to handling Transactions.""" def __init__(self, datastore: DataStore): - self.store = weakref.proxy(datastore) + self.store = datastore async def have_responded( self, origin: str, transaction: Transaction diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 064c4e7da1c..ae94a328cf2 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -37,7 +37,6 @@ """ import logging -import weakref from typing import ( TYPE_CHECKING, Dict, @@ -75,7 +74,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.notifier = hs.get_notifier() - self.hs = weakref.proxy(hs) + self.hs = hs # We may have multiple federation sender instances, so we need to track # their positions separately. diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 8e890c19005..b38f9680208 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -131,7 +131,6 @@ import abc import logging -import weakref from collections import OrderedDict from typing import ( TYPE_CHECKING, @@ -370,7 +369,7 @@ async def _handle(self) -> None: class FederationSender(AbstractFederationSender): def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.server_name = hs.hostname self.store = hs.get_datastores().main @@ -426,7 +425,7 @@ def __init__(self, hs: "HomeServer"): 1.0 / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second ) self._destination_wakeup_queue = _DestinationWakeupQueue( - weakref.proxy(self), self.clock, max_delay_s=rr_txn_interval_per_room_s + self, self.clock, max_delay_s=rr_txn_interval_per_room_s ) # Regularly wake up destinations that have outstanding PDUs to be caught up diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 8c73a8cb397..12bc2ed8d81 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -22,7 +22,6 @@ import logging import urllib -import weakref from typing import ( TYPE_CHECKING, Any, @@ -69,7 +68,7 @@ class TransportLayerClient: def __init__(self, hs: "HomeServer"): self.client = hs.get_federation_http_client() - self.hs = weakref.proxy(hs) + self.hs = hs async def get_room_state_ids( self, destination: str, room_id: str, event_id: str diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index a43367b9518..dbdbd90e2bd 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -20,7 +20,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Literal, Optional, Tuple, Type from synapse.api.errors import FederationDeniedError, SynapseError @@ -64,7 +63,7 @@ def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None) servlet_groups: List of servlet groups to register. Defaults to ``DEFAULT_SERVLET_GROUPS``. """ - self.hs = weakref.proxy(hs) + self.hs = hs self.clock = hs.get_clock() self.servlet_groups = servlet_groups diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index 2204c8cf49b..c215e346246 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -23,7 +23,6 @@ import logging import re import time -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast @@ -65,8 +64,8 @@ def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self.keyring = hs.get_keyring() self.server_name = hs.hostname - self.hs = weakref.proxy(hs) - self.store = weakref.proxy(hs.get_datastores().main) + self.hs = hs + self.store = hs.get_datastores().main self.federation_domain_whitelist = ( hs.config.federation.federation_domain_whitelist ) @@ -278,7 +277,7 @@ def __init__( ratelimiter: FederationRateLimiter, server_name: str, ): - self.hs = weakref.proxy(hs) + self.hs = hs self.authenticator = authenticator self.ratelimiter = ratelimiter self.server_name = server_name diff --git a/synapse/handlers/account.py b/synapse/handlers/account.py index 72eda9d8675..58d2988c3f5 100644 --- a/synapse/handlers/account.py +++ b/synapse/handlers/account.py @@ -19,7 +19,6 @@ # # -import weakref from typing import TYPE_CHECKING, Dict, List, Tuple from synapse.api.errors import Codes, SynapseError @@ -32,7 +31,7 @@ class AccountHandler: def __init__(self, hs: "HomeServer"): self._main_store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs self._federation_client = hs.get_federation_client() self._use_account_validity_in_account_status = ( hs.config.server.use_account_validity_in_account_status diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 6aaeff5abe0..64f9c314337 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -21,7 +21,6 @@ import email.mime.multipart import email.utils import logging -import weakref from typing import TYPE_CHECKING, List, Optional, Tuple from synapse.api.errors import AuthError, StoreError, SynapseError @@ -38,7 +37,7 @@ class AccountValidityHandler: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.config = hs.config self.store = hs.get_datastores().main self.send_email_handler = hs.get_send_email_handler() diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index 794fbbf865d..e90d675b59c 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -21,7 +21,6 @@ import abc import logging -import weakref from typing import ( TYPE_CHECKING, Any, @@ -74,7 +73,7 @@ def __init__(self, hs: "HomeServer"): self._redact_all_events, REDACT_ALL_EVENTS_ACTION_NAME ) - self.hs = weakref.proxy(hs) + self.hs = hs async def get_redact_task(self, redact_id: str) -> Optional[ScheduledTask]: """Get the current status of an active redaction process diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 4b5a94f1232..28a6158f9e6 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -19,7 +19,6 @@ # # import logging -import weakref from typing import ( TYPE_CHECKING, Collection, @@ -76,7 +75,7 @@ class ApplicationServicesHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs self.appservice_api = hs.get_application_service_api() self.scheduler = hs.get_application_service_scheduler() self.started_scheduler = False diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index da72ea029b2..dd7a001475e 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -24,7 +24,6 @@ import time import unicodedata import urllib.parse -import weakref from binascii import crc32 from http import HTTPStatus from typing import ( @@ -214,9 +213,7 @@ def __init__(self, hs: "HomeServer"): self.password_auth_provider = hs.get_password_auth_provider() - self.hs = weakref.proxy( - hs - ) # FIXME better possibility to access registrationHandler later? + self.hs = hs # FIXME better possibility to access registrationHandler later? self.macaroon_gen = hs.get_macaroon_generator() self._password_enabled_for_login = hs.config.auth.password_enabled_for_login self._password_enabled_for_reauth = hs.config.auth.password_enabled_for_reauth diff --git a/synapse/handlers/cas.py b/synapse/handlers/cas.py index c8c1ff68cc3..fbe79c2e4c3 100644 --- a/synapse/handlers/cas.py +++ b/synapse/handlers/cas.py @@ -20,7 +20,6 @@ # import logging import urllib.parse -import weakref from typing import TYPE_CHECKING, Dict, List, Optional from xml.etree import ElementTree as ET @@ -67,7 +66,7 @@ class CasHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self._hostname = hs.hostname self._store = hs.get_datastores().main self._auth_handler = hs.get_auth_handler() diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 4868b70fcbf..305363892fa 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -20,7 +20,6 @@ # import itertools import logging -import weakref from typing import TYPE_CHECKING, Optional from synapse.api.constants import Membership @@ -39,7 +38,7 @@ class DeactivateAccountHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() self._room_member_handler = hs.get_room_member_handler() diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index aaef53865ce..f64dfe6e60d 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -21,7 +21,6 @@ # import logging import random -import weakref from threading import Lock from typing import ( TYPE_CHECKING, @@ -126,8 +125,8 @@ class DeviceHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func - self.hs = weakref.proxy(hs) - self.store = weakref.proxy(cast("GenericWorkerStore", hs.get_datastores().main)) + self.hs = hs + self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() self._appservice_handler = hs.get_application_service_handler() @@ -1327,7 +1326,7 @@ class DeviceListWorkerUpdater: "Handles incoming device list updates from federation and contacts the main device list writer over replication" def __init__(self, hs: "HomeServer"): - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main self._notifier = hs.get_notifier() # On which instance the DeviceListUpdater is running # Must be kept in sync with DeviceHandler diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index bf4f3ff08e3..06584a32409 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -20,7 +20,6 @@ # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Any, Dict, Optional @@ -53,7 +52,7 @@ def __init__(self, hs: "HomeServer"): """ self.store = hs.get_datastores().main self.notifier = hs.get_notifier() - self.hs = weakref.proxy(hs) + self.hs = hs self.device_handler = hs.get_device_handler() if hs.config.experimental.msc3814_enabled: self.event_sources = hs.get_event_sources() diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index f43c4ab24f4..5d4982ea11b 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -21,7 +21,6 @@ import logging import string -import weakref from typing import TYPE_CHECKING, Iterable, List, Literal, Optional, Sequence from synapse.api.constants import MAX_ALIAS_LENGTH, EventTypes @@ -48,7 +47,7 @@ class DirectoryHandler: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.hostname = hs.hostname self.state = hs.get_state_handler() diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index c70a2dcddc4..00708c7a4c6 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -20,7 +20,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Mapping, Optional, Tuple import attr @@ -69,7 +68,7 @@ def __init__(self, hs: "HomeServer"): self.federation = hs.get_federation_client() self.device_handler = hs.get_device_handler() self._appservice_handler = hs.get_application_service_handler() - self.hs = weakref.proxy(hs) + self.hs = hs self.clock = hs.get_clock() self._worker_lock_handler = hs.get_worker_locks_handler() self._task_scheduler = hs.get_task_scheduler() diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index 49c5e930264..7cad3aecab1 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -19,7 +19,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, List, Mapping, Optional, Union from synapse import event_auth @@ -55,7 +54,7 @@ def __init__(self, hs: "HomeServer"): self._store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state self._server_name = hs.hostname - self.hs = weakref.proxy(hs) + self.hs = hs async def check_auth_rules_from_context( self, diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 6a92054bb19..3f46032a43a 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -21,7 +21,6 @@ import logging import random -import weakref from typing import TYPE_CHECKING, Iterable, List, Optional from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState @@ -45,7 +44,7 @@ class EventStreamHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs self.notifier = hs.get_notifier() self.state = hs.get_state_handler() diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 210a7e45b31..87afb0a056d 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -25,7 +25,6 @@ import enum import itertools import logging -import weakref from enum import Enum from http import HTTPStatus from typing import ( @@ -134,7 +133,7 @@ class FederationHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.clock = hs.get_clock() self.store = hs.get_datastores().main diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index d370174e3ea..93ef8c7dfef 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -22,7 +22,6 @@ import collections import itertools import logging -import weakref from http import HTTPStatus from typing import ( TYPE_CHECKING, @@ -167,7 +166,7 @@ def __init__(self, hs: "HomeServer"): ) self._notifier = hs.get_notifier() - self.hs = weakref.proxy(hs) + self.hs = hs self._server_name = hs.hostname self._instance_name = hs.get_instance_name() diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 31888e0f1f9..ccb265d2bee 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -24,7 +24,6 @@ import logging import urllib.parse -import weakref from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional, Tuple import attr @@ -68,7 +67,7 @@ def __init__(self, hs: "HomeServer"): ip_allowlist=hs.config.server.federation_ip_range_allowlist, ) self.federation_http_client = hs.get_federation_http_client() - self.hs = weakref.proxy(hs) + self.hs = hs self._web_client_location = hs.config.email.invite_client_location diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 9d24a8ef3f0..75d64d2d50b 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, List, Optional, Tuple from synapse.api.constants import ( @@ -65,7 +64,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() self.state_handler = hs.get_state_handler() - self.hs = weakref.proxy(hs) + self.hs = hs self.state = hs.get_state_handler() self.clock = hs.get_clock() self.validator = EventValidator() diff --git a/synapse/handlers/jwt.py b/synapse/handlers/jwt.py index d6e7ee50020..400f3a59aa1 100644 --- a/synapse/handlers/jwt.py +++ b/synapse/handlers/jwt.py @@ -18,7 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import weakref from typing import TYPE_CHECKING, Optional, Tuple from authlib.jose import JsonWebToken, JWTClaims @@ -33,7 +32,7 @@ class JwtHandler: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.jwt_secret = hs.config.jwt.jwt_secret self.jwt_subject_claim = hs.config.jwt.jwt_subject_claim diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index f4e1df76087..a4d94eccd9d 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -21,7 +21,6 @@ # import logging import random -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple @@ -476,7 +475,7 @@ async def _expire_event(self, event_id: str) -> None: class EventCreationHandler: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.validator = EventValidator() self.event_builder_factory = hs.get_event_builder_factory() self.server_name = hs.hostname # nb must be called this for @measure_func diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 94f825783ed..8eb2cf31ce0 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -19,7 +19,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, List, Optional, Set, Tuple, cast from twisted.python.failure import Failure @@ -79,7 +78,7 @@ class PaginationHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index a1b4f4150b0..1853f917eb1 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -76,7 +76,6 @@ import contextlib import itertools import logging -import weakref from bisect import bisect from contextlib import contextmanager from types import TracebackType @@ -193,9 +192,9 @@ class BasePresenceHandler(abc.ABC): writer""" def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.clock = hs.get_clock() - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.presence_router = hs.get_presence_router() self.state = hs.get_state_handler() @@ -512,14 +511,9 @@ def __init__(self, hs: "HomeServer"): self.send_stop_syncing, UPDATE_SYNCING_USERS_MS )) - hs.get_reactor().addSystemEventTrigger( - "before", - "shutdown", - run_as_background_process, - "generic_presence.on_shutdown", - self._on_shutdown, - ) + hs.register_shutdown_handler("generic_presence.on_shutdown", self._on_shutdown) + @wrap_as_background_process("WorkerPresenceHandler._on_shutdown") async def _on_shutdown(self) -> None: if self._track_presence: self.hs.get_replication_command_handler().send_command( @@ -810,14 +804,7 @@ def __init__(self, hs: "HomeServer"): # have not yet been persisted self.unpersisted_users_changes: Set[str] = set() - # TODO: (devon) how do I remove this binding? - hs.get_reactor().addSystemEventTrigger( - "before", - "shutdown", - run_as_background_process, - "presence.on_shutdown", - self._on_shutdown, - ) + hs.register_shutdown_handler("presence.on_shutdown", self._on_shutdown) # Keeps track of the number of *ongoing* syncs on this process. While # this is non zero a user will never go offline. @@ -1766,7 +1753,7 @@ def __init__(self, hs: "HomeServer"): # # AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler self.server_name = hs.hostname - self.hs = weakref.proxy(hs) + self.hs = hs self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -2377,7 +2364,7 @@ def __init__(self, hs: "HomeServer", presence_handler: BasePresenceHandler): self._clock = hs.get_clock() self._notifier = hs.get_notifier() self._instance_name = hs.get_instance_name() - self._presence_handler = weakref.proxy(presence_handler) + self._presence_handler = presence_handler self._repl_client = ReplicationGetStreamUpdates.make_client(hs) # Should we keep a queue of recent presence updates? We only bother if diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 8fa03e278ea..d0a2143a300 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -20,7 +20,6 @@ # import logging import random -import weakref from typing import TYPE_CHECKING, List, Optional, Union from synapse.api.constants import ProfileFields @@ -59,7 +58,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self.clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs self.federation = hs.get_federation_client() hs.get_federation_registry().register_query_handler( diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index 4f4301d61eb..c776654d12c 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -19,7 +19,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Tuple from synapse.api.constants import EduTypes, ReceiptTypes @@ -50,7 +49,7 @@ def __init__(self, hs: "HomeServer"): self.event_handler = hs.get_event_handler() self._storage_controllers = hs.get_storage_controllers() - self.hs = weakref.proxy(hs) + self.hs = hs # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b48a1883a6e..c874c01aa02 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -23,7 +23,6 @@ """Contains functions for registering clients.""" import logging -import weakref from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, TypedDict from prometheus_client import Counter @@ -98,10 +97,10 @@ class LoginDict(TypedDict): class RegistrationHandler: def __init__(self, hs: "HomeServer"): - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.auth_blocking = hs.get_auth_blocking() self._auth_handler = hs.get_auth_handler() @@ -127,7 +126,6 @@ def __init__(self, hs: "HomeServer"): ) else: self.device_handler = hs.get_device_handler() - self_ref = weakref.proxy(self) async def do_it( user_id: str, device_id: Optional[str], @@ -138,7 +136,7 @@ async def do_it( auth_provider_id: Optional[str] = None, auth_provider_session_id: Optional[str] = None, ): - return RegistrationHandler.register_device_inner(self_ref, user_id, device_id,initial_display_name, is_guest,is_appservice_ghost,should_issue_refresh_token, auth_provider_id,auth_provider_session_id) + return RegistrationHandler.register_device_inner(self, user_id, device_id,initial_display_name, is_guest,is_appservice_ghost,should_issue_refresh_token, auth_provider_id,auth_provider_session_id) self._register_device_client = do_it self.pusher_pool = hs.get_pusherpool() diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py index fd7009fc666..116685e6de5 100644 --- a/synapse/handlers/reports.py +++ b/synapse/handlers/reports.py @@ -14,7 +14,6 @@ # # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING @@ -32,7 +31,7 @@ class ReportsHandler: def __init__(self, hs: "HomeServer"): - self._hs = weakref.proxy(hs) + self._hs = hs self._store = hs.get_datastores().main self._clock = hs.get_clock() diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 58d7d60ee40..d8c4d0c20e7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -26,7 +26,6 @@ import math import random import string -import weakref from collections import OrderedDict from http import HTTPStatus from typing import ( @@ -126,7 +125,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.auth_blocking = hs.get_auth_blocking() self.clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self.event_creation_handler = hs.get_event_creation_handler() self.room_member_handler = hs.get_room_member_handler() @@ -1442,7 +1441,7 @@ async def _generate_and_create_room_id( class RoomContextHandler: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -1807,7 +1806,7 @@ class RoomShutdownHandler: DEFAULT_ROOM_NAME = "Content Violation Notification" def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.room_member_handler = hs.get_room_member_handler() self._room_creation_handler = hs.get_room_creation_handler() self._replication = hs.get_replication_data_handler() diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 6b36765ee2a..9d4307fb078 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Any, List, Optional, Tuple import attr @@ -65,7 +64,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.hs = weakref.proxy(hs) + self.hs = hs self.enable_room_list_search = hs.config.roomdirectory.enable_room_list_search self.response_cache: ResponseCache[ Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]] diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 4cd96269152..d7e9a85c029 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -22,7 +22,6 @@ import abc import logging import random -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Iterable, List, Optional, Set, Tuple @@ -90,7 +89,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): # ought to be separated out a lot better. def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.auth = hs.get_auth() @@ -2171,7 +2170,7 @@ class RoomForgetterHandler(StateDeltasHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) - self._hs = weakref.proxy(hs) + self._hs = hs self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._clock = hs.get_clock() diff --git a/synapse/handlers/room_policy.py b/synapse/handlers/room_policy.py index 8695fbc7975..170c477d6f4 100644 --- a/synapse/handlers/room_policy.py +++ b/synapse/handlers/room_policy.py @@ -15,7 +15,6 @@ # import logging -import weakref from typing import TYPE_CHECKING from synapse.events import EventBase @@ -30,7 +29,7 @@ class RoomPolicyHandler: def __init__(self, hs: "HomeServer"): - self._hs = weakref.proxy(hs) + self._hs = hs self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._event_auth_handler = hs.get_event_auth_handler() diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 93a8b4fd4fb..1a71135d5fa 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -21,7 +21,6 @@ import itertools import logging -import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple import attr @@ -61,7 +60,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.state_handler = hs.get_state_handler() self.clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs self._event_serializer = hs.get_event_client_serializer() self._relations_handler = hs.get_relations_handler() self._storage_controllers = hs.get_storage_controllers() diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 8027fd19235..92fed980e60 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -21,7 +21,6 @@ import email.utils import logging -import weakref from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from io import BytesIO @@ -159,7 +158,7 @@ async def _sendmail( class SendEmailHandler: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self._reactor = hs.get_reactor() diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index b495e99c0ea..200111a4015 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -15,7 +15,6 @@ import enum import logging -import weakref from itertools import chain from typing import ( TYPE_CHECKING, @@ -188,7 +187,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.storage_controllers = hs.get_storage_controllers() self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync - self.hs = weakref.proxy(hs) + self.hs = hs async def compute_interested_rooms( self, diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index e555ff603cd..5dc6b49dd54 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -22,7 +22,6 @@ import hashlib import io import logging -import weakref from typing import ( TYPE_CHECKING, Any, @@ -204,7 +203,7 @@ def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self._store = hs.get_datastores().main self._server_name = hs.hostname - self.hs = weakref.proxy(hs) + self.hs = hs self._registration_handler = hs.get_registration_handler() self._auth_handler = hs.get_auth_handler() self._device_handler = hs.get_device_handler() diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 7a547e983ae..d80bb157125 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -20,7 +20,6 @@ # # import logging -import weakref from collections import Counter from typing import ( TYPE_CHECKING, @@ -54,7 +53,7 @@ class StatsHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 60c529467ed..44068941732 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -37,7 +37,6 @@ Union, overload, ) -import weakref import attr from prometheus_client import Counter diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 1c483622368..dec0f47311a 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -20,7 +20,6 @@ # import logging import random -import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple import attr @@ -83,7 +82,7 @@ def __init__(self, hs: "HomeServer"): self._storage_controllers = hs.get_storage_controllers() self.server_name = hs.config.server.server_name self.clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs self.federation = None if hs.should_send_federation(): @@ -268,7 +267,7 @@ def __init__(self, hs: "HomeServer"): self.notifier = hs.get_notifier() self.event_auth_handler = hs.get_event_auth_handler() - self.hs = weakref.proxy(hs) + self.hs = hs hs.get_federation_registry().register_edu_handler( EduTypes.TYPING, self._recv_edu @@ -509,7 +508,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self._main_store = hs.get_datastores().main self.clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs def _make_event_for(self, room_id: str) -> JsonMapping: typing = self.hs.get_typing_handler()._room_typing[room_id] diff --git a/synapse/handlers/ui_auth/checkers.py b/synapse/handlers/ui_auth/checkers.py index e3f876b2108..32dca8c43ba 100644 --- a/synapse/handlers/ui_auth/checkers.py +++ b/synapse/handlers/ui_auth/checkers.py @@ -20,7 +20,6 @@ # import logging -import weakref from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, ClassVar, Sequence, Type @@ -160,7 +159,7 @@ async def check_auth(self, authdict: dict, clientip: str) -> Any: class _BaseThreepidAuthChecker: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main async def _check_threepid(self, medium: str, authdict: dict) -> dict: @@ -261,7 +260,7 @@ class RegistrationTokenAuthChecker(UserInteractiveAuthChecker): def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.hs = weakref.proxy(hs) + self.hs = hs self._enabled = bool( hs.config.registration.registration_requires_token ) or bool(hs.config.registration.enable_registration_token_3pid_bypass) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 13870829fac..48d8c988a56 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -20,7 +20,6 @@ # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Set, Tuple @@ -113,7 +112,7 @@ def __init__(self, hs: "HomeServer"): ) self.show_locked_users = hs.config.userdirectory.show_locked_users self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker - self._hs = weakref.proxy(hs) + self._hs = hs # The current position in the current_state_delta stream self.pos: Optional[int] = None diff --git a/synapse/http/client.py b/synapse/http/client.py index c28d020c6d3..dcaaafe45db 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -20,7 +20,6 @@ # import logging import urllib.parse -import weakref from http import HTTPStatus from io import BytesIO from typing import ( @@ -346,7 +345,7 @@ def __init__( hs: "HomeServer", treq_args: Optional[Dict[str, Any]] = None, ): - self.hs = weakref.proxy(hs) + self.hs = hs self.reactor = hs.get_reactor() self._extra_treq_args = treq_args or {} diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 7c89e1c7e67..31bde39c71a 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -24,7 +24,6 @@ import random import sys import urllib.parse -import weakref from http import HTTPStatus from io import BytesIO, StringIO from typing import ( @@ -402,7 +401,7 @@ def __init__( hs: "HomeServer", tls_client_options_factory: Optional[FederationPolicyForHTTPS], ): - self.hs = weakref.proxy(hs) + self.hs = hs self.signing_key = hs.signing_key self.server_name = hs.hostname diff --git a/synapse/http/server.py b/synapse/http/server.py index efcec958caf..395d82fd168 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -25,7 +25,6 @@ import types import urllib import urllib.parse -import weakref from http import HTTPStatus from http.client import FOUND from inspect import isawaitable @@ -473,7 +472,7 @@ def __init__( super().__init__(canonical_json, extract_context, clock=self.clock) # Map of path regex -> method -> callback. self._routes: Dict[Pattern[str], Dict[bytes, _PathEntry]] = {} - self.hs = weakref.proxy(hs) + self.hs = hs def register_paths( self, @@ -503,7 +502,7 @@ def register_paths( for path_pattern in path_patterns: logger.debug("Registering for %s %s", method, path_pattern.pattern) self._routes.setdefault(path_pattern, {})[method_bytes] = _PathEntry( - weakref.proxy(callback), servlet_classname + callback, servlet_classname ) def _get_handler_for_request( diff --git a/synapse/http/site.py b/synapse/http/site.py index 2eac2a15380..5087f38455d 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -23,7 +23,6 @@ import time from http import HTTPStatus from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, Union -import weakref import attr from zope.interface import implementer @@ -702,7 +701,7 @@ def __init__( request_id_header = config.http_options.request_id_header - self_ref = weakref.proxy(self) + self_ref = self def request_factory(channel: HTTPChannel, queued: bool) -> Request: return request_class( channel, diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index f4da86acf17..bc6d6c574b0 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -23,7 +23,6 @@ import logging import os import shutil -import weakref from io import BytesIO from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple @@ -90,7 +89,7 @@ class MediaRepository: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.client = hs.get_federation_client() self.clock = hs.get_clock() diff --git a/synapse/media/media_storage.py b/synapse/media/media_storage.py index 2ff6b5f637a..afd33c02a12 100644 --- a/synapse/media/media_storage.py +++ b/synapse/media/media_storage.py @@ -24,7 +24,6 @@ import logging import os import shutil -import weakref from contextlib import closing from io import BytesIO from types import TracebackType @@ -171,7 +170,7 @@ def __init__( filepaths: MediaFilePaths, storage_providers: Sequence["StorageProvider"], ): - self.hs = weakref.proxy(hs) + self.hs = hs self.reactor = hs.get_reactor() self.local_media_directory = local_media_directory self.filepaths = filepaths @@ -416,7 +415,7 @@ class FileResponder(Responder): """ def __init__(self, hs: "HomeServer", open_file: BinaryIO): - self.hs = weakref.proxy(hs) + self.hs = hs self.open_file = open_file def write_to_consumer(self, consumer: IConsumer) -> Deferred: diff --git a/synapse/media/storage_provider.py b/synapse/media/storage_provider.py index 277c7752505..300952025a3 100644 --- a/synapse/media/storage_provider.py +++ b/synapse/media/storage_provider.py @@ -23,7 +23,6 @@ import logging import os import shutil -import weakref from typing import TYPE_CHECKING, Callable, Optional from synapse.config._base import Config @@ -145,7 +144,7 @@ class FileStorageProviderBackend(StorageProvider): """ def __init__(self, hs: "HomeServer", config: str): - self.hs = weakref.proxy(hs) + self.hs = hs self.reactor = hs.get_reactor() self.cache_directory = hs.config.media.media_store_path self.base_directory = config diff --git a/synapse/media/thumbnailer.py b/synapse/media/thumbnailer.py index 8a7c86897bd..5d9afda3229 100644 --- a/synapse/media/thumbnailer.py +++ b/synapse/media/thumbnailer.py @@ -20,7 +20,6 @@ # # import logging -import weakref from io import BytesIO from types import TracebackType from typing import TYPE_CHECKING, List, Optional, Tuple, Type @@ -265,7 +264,7 @@ def __init__( media_repo: "MediaRepository", media_storage: MediaStorage, ): - self.hs = weakref.proxy(hs) + self.hs = hs self.reactor = hs.get_reactor() self.media_repo = media_repo self.media_storage = media_storage diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index feeec49425c..8d7229d9aae 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -19,7 +19,6 @@ # # from typing import TYPE_CHECKING -import weakref import attr @@ -50,7 +49,7 @@ class CommonUsageMetricsManager: def __init__(self, hs: "HomeServer") -> None: self._store = hs.get_datastores().main self._clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs async def get_metrics(self) -> CommonUsageMetrics: """Get the CommonUsageMetrics object. If no collection has happened yet, do it diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 67b2f810f75..f039cd54c3e 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -20,7 +20,6 @@ # import email.utils import logging -import weakref from typing import ( TYPE_CHECKING, Any, @@ -259,7 +258,7 @@ class ModuleApi: """ def __init__(self, hs: "HomeServer", auth_handler: AuthHandler) -> None: - self._hs = weakref.proxy(hs) + self._hs = hs # TODO: Fix this type hint once the types for the data stores have been ironed # out. @@ -1944,7 +1943,7 @@ class AccountDataManager: """ def __init__(self, hs: "HomeServer") -> None: - self._hs = weakref.proxy(hs) + self._hs = hs self._store = hs.get_datastores().main self._handler = hs.get_account_data_handler() diff --git a/synapse/notifier.py b/synapse/notifier.py index bc5688f8231..07e5ea47132 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import ( TYPE_CHECKING, Awaitable, @@ -224,7 +223,7 @@ def __init__(self, hs: "HomeServer"): self.user_to_user_stream: Dict[str, _NotifierUserStream] = {} self.room_to_user_streams: Dict[str, Set[_NotifierUserStream]] = {} - self.hs = weakref.proxy(hs) + self.hs = hs self._storage_controllers = hs.get_storage_controllers() self.event_sources = hs.get_event_sources() self.store = hs.get_datastores().main diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py index 8aeaf19e4a8..7bc99bd7857 100644 --- a/synapse/push/__init__.py +++ b/synapse/push/__init__.py @@ -94,7 +94,6 @@ """ import abc -import weakref from typing import TYPE_CHECKING, Any, Dict, Optional import attr @@ -158,7 +157,7 @@ class ThrottleParams: class Pusher(metaclass=abc.ABCMeta): def __init__(self, hs: "HomeServer", pusher_config: PusherConfig): - self.hs = weakref.proxy(hs) + self.hs = hs self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 9d643a36228..fed99319300 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import ( TYPE_CHECKING, Any, @@ -128,7 +127,7 @@ class BulkPushRuleEvaluator: """ def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.server_name = hs.hostname self.store = hs.get_datastores().main self.server_name = hs.hostname # nb must be called this for @measure_func diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 71004b2feca..fadba480dd8 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -21,7 +21,6 @@ import logging import urllib.parse -import weakref from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, TypeVar import bleach @@ -123,7 +122,7 @@ def __init__( template_html: jinja2.Template, template_text: jinja2.Template, ): - self.hs = weakref.proxy(hs) + self.hs = hs self.template_html = template_html self.template_text = template_text diff --git a/synapse/push/pusher.py b/synapse/push/pusher.py index 445ca56eaa0..9a5dd7a9d4b 100644 --- a/synapse/push/pusher.py +++ b/synapse/push/pusher.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Callable, Dict, Optional from synapse.push import Pusher, PusherConfig @@ -36,7 +35,7 @@ class PusherFactory: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.config = hs.config self.pusher_types: Dict[str, Callable[[HomeServer, PusherConfig], Pusher]] = { diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index cb4388af241..0a7541b4c70 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Dict, Iterable, Optional from prometheus_client import Gauge @@ -65,7 +64,7 @@ class PusherPool: """ def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.pusher_factory = PusherFactory(hs) self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 9cdd474097b..4e6f206854b 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -24,7 +24,6 @@ import urllib.parse from inspect import signature from typing import TYPE_CHECKING, Any, Awaitable, Callable, ClassVar, Dict, List, Tuple -import weakref from prometheus_client import Counter, Gauge @@ -206,7 +205,7 @@ def make_client(cls, hs: "HomeServer") -> Callable: parameter to specify which instance to hit (the instance must be in the `instance_map` config). """ - _hs = weakref.proxy(hs) + _hs = hs clock = hs.get_clock() client = hs.get_replication_client() local_instance_name = hs.get_instance_name() diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py index d0e43a52176..974d83bb8bf 100644 --- a/synapse/replication/http/devices.py +++ b/synapse/replication/http/devices.py @@ -20,7 +20,6 @@ import logging from typing import TYPE_CHECKING, Dict, List, Optional, Tuple -import weakref from twisted.web.server import Request @@ -162,7 +161,7 @@ def __init__(self, hs: "HomeServer"): self.device_list_updater = hs.get_device_handler().device_list_updater - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main self.clock = hs.get_clock() @staticmethod diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 9cef4cfedb8..177b183aaa9 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -21,7 +21,6 @@ """A replication client for use by synapse workers.""" import logging -import weakref from typing import TYPE_CHECKING, Dict, Iterable, Optional, Set, Tuple from sortedcontainers import SortedList @@ -77,7 +76,7 @@ class ReplicationDataHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self._reactor = hs.get_reactor() self._clock = hs.get_clock() @@ -415,7 +414,7 @@ def __init__(self, hs: "HomeServer"): assert hs.should_send_federation() self.store = hs.get_datastores().main - self._hs = weakref.proxy(hs) + self._hs = hs # We need to make a temporary value to ensure that mypy picks up the # right type. We know we should have a federation sender instance since diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index bc173efe948..9ffa1484ba8 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -35,7 +35,6 @@ TypeVar, Union, ) -import weakref from prometheus_client import Counter diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 35401c90951..559b785a2d3 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -23,7 +23,6 @@ import logging import random from typing import TYPE_CHECKING, List, Optional, Tuple -import weakref from prometheus_client import Counter @@ -80,7 +79,7 @@ class ReplicationStreamer: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main self.clock = hs.get_clock() self.notifier = hs.get_notifier() self._instance_name = hs.get_instance_name() diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 48f429d7f0a..9694fff4fee 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -31,7 +31,6 @@ Tuple, TypeVar, ) -import weakref import attr @@ -129,7 +128,7 @@ def __init__( update_function: callback go get stream updates, as above """ self.local_instance_name = local_instance_name - self.update_function = weakref.proxy(update_function) + self.update_function = update_function # The token from which we last asked for updates self.last_token = self.current_token(self.local_instance_name) @@ -562,7 +561,7 @@ class DeviceListsStreamRow: ROW_TYPE = DeviceListsStreamRow def __init__(self, hs: "HomeServer"): - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main super().__init__( hs.get_instance_name(), self._update_function, @@ -649,7 +648,7 @@ class AccountDataStreamRow: ROW_TYPE = AccountDataStreamRow def __init__(self, hs: "HomeServer"): - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main super().__init__( hs.get_instance_name(), self._update_function, @@ -741,7 +740,7 @@ class ThreadSubscriptionsStreamRow: ROW_TYPE = ThreadSubscriptionsStreamRow def __init__(self, hs: Any): - self.store = weakref.proxy(hs.get_datastores().main) + self.store = hs.get_datastores().main super().__init__( hs.get_instance_name(), self._update_function, diff --git a/synapse/replication/tcp/streams/events.py b/synapse/replication/tcp/streams/events.py index daece31f2e4..05b55fb0338 100644 --- a/synapse/replication/tcp/streams/events.py +++ b/synapse/replication/tcp/streams/events.py @@ -21,7 +21,6 @@ import heapq from collections import defaultdict from typing import TYPE_CHECKING, Iterable, Optional, Tuple, Type, TypeVar, cast -import weakref import attr @@ -152,7 +151,7 @@ class EventsStream(_StreamFromIdGen): NAME = "events" def __init__(self, hs: "HomeServer"): - self._store = weakref.proxy(hs.get_datastores().main) + self._store = hs.get_datastores().main super().__init__( hs.get_instance_name(), self._update_function, self._store._stream_id_gen ) diff --git a/synapse/replication/tcp/streams/federation.py b/synapse/replication/tcp/streams/federation.py index 014c9c36861..1c2ffe86b7b 100644 --- a/synapse/replication/tcp/streams/federation.py +++ b/synapse/replication/tcp/streams/federation.py @@ -19,7 +19,6 @@ # # from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Tuple -import weakref import attr diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index b5c80ce9f80..37364a587a3 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -19,7 +19,6 @@ # # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -53,7 +52,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, request: SynapseRequest, user_id: str, device_id: str @@ -125,7 +124,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_worker_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, request: SynapseRequest, user_id: str @@ -198,7 +197,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs async def on_POST( self, request: SynapseRequest, user_id: str diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index bb993c216d7..d6dd85fb698 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -20,7 +20,6 @@ # -import weakref from enum import Enum from http import HTTPStatus from typing import TYPE_CHECKING, Dict, Tuple @@ -69,7 +68,7 @@ def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() self.store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index f767c492b4a..d872fa69358 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -20,7 +20,6 @@ # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Optional, Tuple @@ -267,7 +266,7 @@ class DeleteMediaByID(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = weakref.proxy(hs) + self.hs = hs self.media_repository = hs.get_media_repository() async def on_DELETE( @@ -360,7 +359,7 @@ class UserMediaRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/media$") def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main self.media_repository = hs.get_media_repository() diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 827d9a7946e..96be843f108 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -19,7 +19,6 @@ # # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple, cast @@ -495,7 +494,7 @@ def __init__(self, hs: "HomeServer"): self.admin_handler = hs.get_admin_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_POST( self, request: SynapseRequest, room_identifier: str @@ -588,7 +587,7 @@ def __init__(self, hs: "HomeServer"): self._state_storage_controller = hs.get_storage_controllers().state self.event_creation_handler = hs.get_event_creation_handler() self.state_handler = hs.get_state_handler() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_POST( self, request: SynapseRequest, room_identifier: str @@ -785,7 +784,7 @@ class RoomEventContextServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self.clock = hs.get_clock() self.room_context_handler = hs.get_room_context_handler() self._event_serializer = hs.get_event_client_serializer() @@ -915,7 +914,7 @@ class RoomMessagesRestServlet(RestServlet): PATTERNS = admin_patterns("/rooms/(?P[^/]*)/messages$") def __init__(self, hs: "HomeServer"): - self._hs = weakref.proxy(hs) + self._hs = hs self._clock = hs.get_clock() self._pagination_handler = hs.get_pagination_handler() self._auth = hs.get_auth() diff --git a/synapse/rest/admin/server_notice_servlet.py b/synapse/rest/admin/server_notice_servlet.py index f2e6f7d546a..9f51a5a873f 100644 --- a/synapse/rest/admin/server_notice_servlet.py +++ b/synapse/rest/admin/server_notice_servlet.py @@ -17,7 +17,6 @@ # [This file includes modifications made by New Vector Limited] # # -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Optional, Tuple @@ -63,7 +62,7 @@ def __init__(self, hs: "HomeServer"): self.server_notices_manager = hs.get_server_notices_manager() self.admin_handler = hs.get_admin_handler() self.txns = HttpTransactionCache(hs) - self.hs = weakref.proxy(hs) + self.hs = hs def register(self, json_resource: HttpServer) -> None: PATTERN = "/send_server_notice" diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 82f73607884..920c27f2a8c 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -22,7 +22,6 @@ import hmac import logging import secrets -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union @@ -240,7 +239,7 @@ class UserRestServletV2(RestServlet): """ def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() self.store = hs.get_datastores().main @@ -545,7 +544,7 @@ def __init__(self, hs: "HomeServer"): self.auth_handler = hs.get_auth_handler() self.reactor = hs.get_reactor() self.nonces: Dict[str, int] = {} - self.hs = weakref.proxy(hs) + self.hs = hs self._all_user_types = hs.config.user_types.all_user_types def _clear_old_nonces(self) -> None: @@ -725,7 +724,7 @@ class WhoisRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, request: SynapseRequest, user_id: str @@ -750,7 +749,7 @@ class DeactivateAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self._deactivate_account_handler = hs.get_deactivate_account_handler() self.auth = hs.get_auth() - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main async def on_POST( @@ -792,7 +791,7 @@ class SuspendAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main class PutBody(RequestBodyModel): @@ -915,7 +914,7 @@ class SearchUsersRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, request: SynapseRequest, target_user_id: str @@ -984,7 +983,7 @@ class UserAdminServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, request: SynapseRequest, user_id: str @@ -1040,7 +1039,7 @@ class UserMembershipRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/joined_rooms$") def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main @@ -1072,7 +1071,7 @@ class PushersRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/pushers$") def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main self.auth = hs.get_auth() @@ -1117,7 +1116,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_POST( self, request: SynapseRequest, user_id: str @@ -1185,7 +1184,7 @@ class ShadowBanRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_POST( self, request: SynapseRequest, user_id: str @@ -1237,7 +1236,7 @@ class RateLimitRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, request: SynapseRequest, user_id: str @@ -1344,7 +1343,7 @@ class AccountDataRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() self._store = hs.get_datastores().main - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, request: SynapseRequest, user_id: str diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index d962bf2a26b..7d6c0afd9a6 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -21,7 +21,6 @@ # import logging import random -import weakref from typing import TYPE_CHECKING, List, Literal, Optional, Tuple from urllib.parse import urlparse @@ -76,7 +75,7 @@ class EmailPasswordRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.datastore = hs.get_datastores().main self.config = hs.config self.identity_handler = hs.get_identity_handler() @@ -150,7 +149,7 @@ class PasswordRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() self.datastore = self.hs.get_datastores().main @@ -282,7 +281,7 @@ class DeactivateAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() self._deactivate_account_handler = hs.get_deactivate_account_handler() @@ -325,7 +324,7 @@ class EmailThreepidRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.config = hs.config self.identity_handler = hs.get_identity_handler() self.store = self.hs.get_datastores().main @@ -407,7 +406,7 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): PATTERNS = client_patterns("/account/3pid/msisdn/requestToken$") def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs super().__init__() self.store = self.hs.get_datastores().main self.identity_handler = hs.get_identity_handler() @@ -587,7 +586,7 @@ class ThreepidRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() @@ -650,7 +649,7 @@ class ThreepidAddRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() @@ -700,7 +699,7 @@ class ThreepidBindRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() @@ -728,7 +727,7 @@ class ThreepidUnbindRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.identity_handler = hs.get_identity_handler() self.auth = hs.get_auth() self.datastore = self.hs.get_datastores().main @@ -758,7 +757,7 @@ class ThreepidDeleteRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/account_data.py b/synapse/rest/client/account_data.py index 00347bd919a..734c9e992f5 100644 --- a/synapse/rest/client/account_data.py +++ b/synapse/rest/client/account_data.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Optional, Tuple from synapse.api.constants import AccountDataTypes, ReceiptTypes @@ -68,7 +67,7 @@ class AccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main self.handler = hs.get_account_data_handler() @@ -144,7 +143,7 @@ class UnstableAccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self.auth = hs.get_auth() self.handler = hs.get_account_data_handler() @@ -181,7 +180,7 @@ class RoomAccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main self.handler = hs.get_account_data_handler() @@ -279,7 +278,7 @@ class UnstableRoomAccountDataServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self.auth = hs.get_auth() self.handler = hs.get_account_data_handler() diff --git a/synapse/rest/client/account_validity.py b/synapse/rest/client/account_validity.py index f5388257e24..ec7836b647f 100644 --- a/synapse/rest/client/account_validity.py +++ b/synapse/rest/client/account_validity.py @@ -19,7 +19,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Tuple from twisted.web.server import Request @@ -43,7 +42,7 @@ class AccountValidityRenewServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.account_activity_handler = hs.get_account_validity_handler() self.auth = hs.get_auth() self.account_renewed_template = ( @@ -84,7 +83,7 @@ class AccountValiditySendMailServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.account_activity_handler = hs.get_account_validity_handler() self.auth = hs.get_auth() self.account_validity_renew_by_email_enabled = ( diff --git a/synapse/rest/client/auth.py b/synapse/rest/client/auth.py index 80a65476d8c..b8dca7c7977 100644 --- a/synapse/rest/client/auth.py +++ b/synapse/rest/client/auth.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, cast from twisted.web.server import Request @@ -51,7 +50,7 @@ class AuthRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() self.registration_handler = hs.get_registration_handler() diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 09fe0170770..a279db1cc5c 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -18,7 +18,6 @@ # # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -44,7 +43,7 @@ class CapabilitiesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.config = hs.config self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index d5e963c2df1..5667af20d44 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -21,7 +21,6 @@ # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple @@ -52,7 +51,7 @@ class DevicesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self._msc3852_enabled = hs.config.experimental.msc3852_enabled @@ -88,7 +87,7 @@ class DeleteDevicesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -139,7 +138,7 @@ class DeviceRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -296,7 +295,7 @@ class DehydratedDeviceServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -355,7 +354,7 @@ class ClaimDehydratedDeviceServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() self.device_handler = handler @@ -494,7 +493,7 @@ class DehydratedDeviceV2Servlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() self.e2e_keys_handler = hs.get_e2e_keys_handler() diff --git a/synapse/rest/client/filter.py b/synapse/rest/client/filter.py index 5526580c3c3..f1e881975f5 100644 --- a/synapse/rest/client/filter.py +++ b/synapse/rest/client/filter.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import AuthError, NotFoundError, StoreError, SynapseError @@ -43,7 +42,7 @@ class GetFilterRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.filtering = hs.get_filtering() @@ -82,7 +81,7 @@ class CreateFilterRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.filtering = hs.get_filtering() diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index b4dd68203ca..09749b840fc 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -22,7 +22,6 @@ import logging import re -import weakref from collections import Counter from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, cast @@ -376,7 +375,7 @@ class SigningKeyUploadServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.e2e_keys_handler = hs.get_e2e_keys_handler() self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index f76d23ea56c..921268c9331 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -21,7 +21,6 @@ import logging import re -import weakref from typing import ( TYPE_CHECKING, Any, @@ -92,7 +91,7 @@ class LoginRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self._main_store = hs.get_datastores().main # JWT configuration variables. diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index a9d97e46a87..80f184ea7af 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -22,7 +22,6 @@ import logging import re -import weakref from typing import Optional from synapse.http.server import ( @@ -135,7 +134,7 @@ def __init__( self.media_repo = media_repo self.media_storage = media_storage self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails - self.hs = weakref.proxy(hs) + self.hs = hs self._server_name = hs.hostname self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.thumbnailer = ThumbnailProvider(hs, media_repo, media_storage) @@ -224,7 +223,7 @@ class DownloadResource(RestServlet): def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() async def on_GET( diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index 0428012ee43..f439da8e52b 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -22,7 +22,6 @@ """This module contains REST servlets to do with presence: /presence/""" import logging -import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import AuthError, Codes, LimitExceededError, SynapseError @@ -46,7 +45,7 @@ class PresenceStatusRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.presence_handler = hs.get_presence_handler() self.clock = hs.get_clock() self.auth = hs.get_auth() diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index db93d60eb00..243245f7393 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -22,7 +22,6 @@ """This module contains REST servlets to do with profile: /profile/""" import re -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -64,7 +63,7 @@ class ProfileRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.profile_handler = hs.get_profile_handler() self.auth = hs.get_auth() @@ -107,7 +106,7 @@ class ProfileFieldRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.profile_handler = hs.get_profile_handler() self.auth = hs.get_auth() diff --git a/synapse/rest/client/pusher.py b/synapse/rest/client/pusher.py index b5b8d733bdc..a455f95a263 100644 --- a/synapse/rest/client/pusher.py +++ b/synapse/rest/client/pusher.py @@ -21,7 +21,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import Codes, SynapseError @@ -49,7 +48,7 @@ class PushersRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self._store = hs.get_datastores().main @@ -80,7 +79,7 @@ class PushersSetRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.notifier = hs.get_notifier() self.pusher_pool = self.hs.get_pusherpool() diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index d17eb771964..68857c724ba 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -21,7 +21,6 @@ # import logging import random -import weakref from typing import TYPE_CHECKING, List, Optional, Tuple from twisted.web.server import Request @@ -82,7 +81,7 @@ class EmailRegisterRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.identity_handler = hs.get_identity_handler() self.config = hs.config @@ -177,7 +176,7 @@ class MsisdnRegisterRequestTokenRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.identity_handler = hs.get_identity_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -257,7 +256,7 @@ class RegistrationSubmitTokenServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.config = hs.config self.clock = hs.get_clock() @@ -323,7 +322,7 @@ class UsernameAvailabilityRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.registration_handler = hs.get_registration_handler() self.ratelimiter = FederationRateLimiter( hs.get_clock(), @@ -387,7 +386,7 @@ class RegistrationTokenValidityRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main self.ratelimiter = Ratelimiter( hs=hs, @@ -417,7 +416,7 @@ class RegisterRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main self.auth_handler = hs.get_auth_handler() diff --git a/synapse/rest/client/reporting.py b/synapse/rest/client/reporting.py index 9b054e6d50d..81faf38a7f8 100644 --- a/synapse/rest/client/reporting.py +++ b/synapse/rest/client/reporting.py @@ -20,7 +20,6 @@ # import logging -import weakref from http import HTTPStatus from typing import TYPE_CHECKING, Tuple @@ -50,7 +49,7 @@ class ReportEventRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -129,7 +128,7 @@ class ReportRoomRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastores().main @@ -181,7 +180,7 @@ class ReportUserRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.clock = hs.get_clock() self.store = hs.get_datastores().main diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index d8f8b8bc9f1..09b5de275ce 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -23,7 +23,6 @@ import logging import re -import weakref from enum import Enum from http import HTTPStatus from typing import TYPE_CHECKING, Awaitable, Dict, List, Optional, Tuple @@ -601,7 +600,7 @@ class PublicRoomListRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -801,7 +800,7 @@ class RoomMessageListRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self.clock = hs.get_clock() self.pagination_handler = hs.get_pagination_handler() self.auth = hs.get_auth() @@ -1004,7 +1003,7 @@ class RoomEventContextServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self.clock = hs.get_clock() self.room_context_handler = hs.get_room_context_handler() self._event_serializer = hs.get_event_client_serializer() @@ -1348,7 +1347,7 @@ class RoomTypingRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.presence_handler = hs.get_presence_handler() self.auth = hs.get_auth() diff --git a/synapse/rest/client/room_upgrade_rest_servlet.py b/synapse/rest/client/room_upgrade_rest_servlet.py index 981c148c45f..130ae31619a 100644 --- a/synapse/rest/client/room_upgrade_rest_servlet.py +++ b/synapse/rest/client/room_upgrade_rest_servlet.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import Codes, ShadowBanError, SynapseError @@ -66,7 +65,7 @@ class RoomUpgradeRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self._hs = weakref.proxy(hs) + self._hs = hs self._room_creation_handler = hs.get_room_creation_handler() self._auth = hs.get_auth() self._worker_lock_handler = hs.get_worker_locks_handler() diff --git a/synapse/rest/client/sendtodevice.py b/synapse/rest/client/sendtodevice.py index 36240e5bf0f..2a675145609 100644 --- a/synapse/rest/client/sendtodevice.py +++ b/synapse/rest/client/sendtodevice.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Tuple from synapse.http import servlet @@ -47,7 +46,7 @@ class SendToDeviceRestServlet(servlet.RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.txns = HttpTransactionCache(hs) self.device_message_handler = hs.get_device_message_handler() diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 5c3d076173c..11bb93947c9 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -20,7 +20,6 @@ # import itertools import logging -import weakref from collections import defaultdict from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union @@ -111,7 +110,7 @@ class SyncRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.server_name = hs.hostname self.auth = hs.get_auth() self.store = hs.get_datastores().main @@ -684,7 +683,7 @@ class SlidingSyncE2eeRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.store = hs.get_datastores().main self.sync_handler = hs.get_sync_handler() diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index fc541cce222..49de9976f6f 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -23,7 +23,6 @@ to ensure idempotency when performing PUTs using the REST API.""" import logging -import weakref from typing import TYPE_CHECKING, Awaitable, Callable, Dict, Hashable, Tuple from typing_extensions import ParamSpec @@ -49,7 +48,7 @@ class HttpTransactionCache: def __init__(self, hs: "HomeServer"): - self.hs = weakref.proxy(hs) + self.hs = hs self.clock = self.hs.get_clock() # $txn_key: (ObservableDeferred<(res_code, res_json_body)>, timestamp) self.transactions: Dict[ diff --git a/synapse/rest/client/user_directory.py b/synapse/rest/client/user_directory.py index 217205e60bd..94fcb11c0ca 100644 --- a/synapse/rest/client/user_directory.py +++ b/synapse/rest/client/user_directory.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Tuple from synapse.api.errors import SynapseError @@ -43,7 +42,7 @@ class UserDirectorySearchRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() self.user_directory_handler = hs.get_user_directory_handler() diff --git a/synapse/rest/client/voip.py b/synapse/rest/client/voip.py index 3091db2a80e..fbed3a3bae9 100644 --- a/synapse/rest/client/voip.py +++ b/synapse/rest/client/voip.py @@ -22,7 +22,6 @@ import base64 import hashlib import hmac -import weakref from typing import TYPE_CHECKING, Tuple from synapse.http.server import HttpServer @@ -41,7 +40,7 @@ class VoipRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() - self.hs = weakref.proxy(hs) + self.hs = hs self.auth = hs.get_auth() async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 4992f2add87..3961f82894a 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -20,7 +20,6 @@ import hmac import logging -import weakref from hashlib import sha256 from http import HTTPStatus from os import path @@ -84,7 +83,7 @@ class ConsentResource(DirectServeHtmlResource): def __init__(self, hs: "HomeServer"): super().__init__(clock=hs.get_clock()) - self.hs = weakref.proxy(hs) + self.hs = hs self.store = hs.get_datastores().main self.registration_handler = hs.get_registration_handler() diff --git a/synapse/rest/media/download_resource.py b/synapse/rest/media/download_resource.py index 993fae93047..12b1e6701a4 100644 --- a/synapse/rest/media/download_resource.py +++ b/synapse/rest/media/download_resource.py @@ -21,7 +21,6 @@ # import logging import re -import weakref from typing import TYPE_CHECKING, Optional from synapse.http.server import set_corp_headers, set_cors_headers @@ -51,7 +50,7 @@ class DownloadResource(RestServlet): def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo - self.hs = weakref.proxy(hs) + self.hs = hs async def on_GET( self, diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index 5aee7c7d15f..b24533677fc 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -22,7 +22,6 @@ import logging import re -import weakref from typing import TYPE_CHECKING from synapse.http.server import set_corp_headers, set_cors_headers @@ -62,7 +61,7 @@ def __init__( self.store = hs.get_datastores().main self.media_repo = media_repo self.media_storage = media_storage - self.hs = weakref.proxy(hs) + self.hs = hs self._server_name = hs.hostname self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails diff --git a/synapse/server.py b/synapse/server.py index d187748f806..85d69e71cb5 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -26,8 +26,8 @@ import abc import functools import logging -from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Type, TypeVar, cast -import weakref +from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type, TypeVar, cast +from attr import dataclass from twisted.internet import defer from typing_extensions import TypeAlias @@ -130,6 +130,7 @@ from synapse.http.matrixfederationclient import MatrixFederationHttpClient from synapse.media.media_repository import MediaRepository from synapse.metrics import LaterGauge, register_threadpool +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.metrics.common_usage_metrics import CommonUsageMetricsManager from synapse.module_api import ModuleApi from synapse.module_api.callbacks import ModuleApiCallbacks @@ -139,6 +140,7 @@ from synapse.replication.tcp.client import ReplicationDataHandler from synapse.replication.tcp.external_cache import ExternalCache from synapse.replication.tcp.handler import ReplicationCommandHandler +from synapse.replication.tcp.protocol import connected_connections from synapse.replication.tcp.resource import ReplicationStreamer from synapse.replication.tcp.streams import STREAMS_MAP, Stream from synapse.rest.media.media_repository_resource import MediaRepositoryResource @@ -217,14 +219,7 @@ def cache_in_self(builder: F) -> F: def _get(self: "HomeServer") -> T: try: dep = getattr(self, depname) - try: - return_dep = weakref.proxy(dep) - except: - if isinstance(dep, dict): - return_dep = {k: weakref.proxy(v) for k, v in dep.items()} - else: - logger.warning("Failed creating proxy for %s", depname) - return return_dep + return dep except AttributeError: pass @@ -239,20 +234,18 @@ def _get(self: "HomeServer") -> T: finally: building[0] = False - return_dep = dep - try: - return_dep = weakref.proxy(dep) - except: - if isinstance(dep, dict): - return_dep = {k: weakref.proxy(v) for k, v in dep.items()} - else: - logger.warning("Failed creating proxy for %s", depname) - - return return_dep + return dep return cast(F, _get) +@dataclass +class ShutdownInfo: + desc: str + func: Callable[..., Any] + trigger_id: Any + + class HomeServer(metaclass=abc.ABCMeta): """A basic homeserver object without lazy component builders. @@ -333,6 +326,7 @@ def __init__( self._looping_calls: List[LoopingCall] = [] self._later_gauges: List[LaterGauge] = [] self._background_processes: List[defer.Deferred] = [] + self._shutdown_handlers: List[ShutdownInfo] = [] def shutdown(self) -> None: logger.info("Received shutdown request") @@ -344,6 +338,11 @@ def shutdown(self) -> None: for connection in self.get_replication_command_handler()._connections: connection.close() self.get_replication_command_handler()._connections.clear() + # NOTE: Dont trample on other tenants + # for connection in connected_connections: + # logger.info("Closing replication connection...") + # connection.close() + # connected_connections.clear() logger.info("Stopping replication streams") # self.get_replication_command_handler()._factory.continueTrying = False @@ -354,6 +353,7 @@ def shutdown(self) -> None: # self.get_replication_command_handler()._factory = None self.get_replication_command_handler()._streams.clear() self.get_replication_command_handler()._streams_to_replicate.clear() + # XXX: Does this trample on other tenants? STREAMS_MAP.clear() from synapse.util.batching_queue import number_of_keys @@ -383,9 +383,32 @@ def shutdown(self) -> None: process.cancel() self._background_processes.clear() + from synapse.util.caches import CACHE_METRIC_REGISTRY + logger.info("Clearing cache metrics: %d", len(CACHE_METRIC_REGISTRY._pre_update_hooks)) + # TODO: Do this better, ie. don't clear metrics for other tenants + # only clear them for this server + CACHE_METRIC_REGISTRY.clear() + + for shutdown_handler in self._shutdown_handlers: + logger.info("Shutting down %s", shutdown_handler.desc) + self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) + # TODO: we should probably run these + #yield defer.ensureDeferred(shutdown_handler.func()) + self._shutdown_handlers.clear() + for call in self.get_reactor().getDelayedCalls(): call.cancel() + def register_shutdown_handler(self, desc: "LiteralString", shutdown_func: Callable[..., Any]) -> None: + id = self.get_reactor().addSystemEventTrigger( + "before", + "shutdown", + run_as_background_process, + desc, + shutdown_func, + ) + self._shutdown_handlers.append(ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id)) + def register_looping_call(self, looping_call: LoopingCall) -> None: self._looping_calls.append(looping_call) @@ -1061,6 +1084,7 @@ def get_media_sender_thread_pool(self) -> ThreadPool: ) media_threadpool.start() + # TODO: what do. it's different since it uses during instead of before self.get_reactor().addSystemEventTrigger( "during", "shutdown", media_threadpool.stop ) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 310cf967415..d10274eddd8 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -18,7 +18,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, Optional from synapse.api.constants import EventTypes, Membership, RoomCreationPreset @@ -45,7 +44,7 @@ def __init__(self, hs: "HomeServer"): self._event_creation_handler = hs.get_event_creation_handler() self._message_handler = hs.get_message_handler() self._storage_controllers = hs.get_storage_controllers() - self.hs = weakref.proxy(hs) + self.hs = hs self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.servernotices.server_notices_mxid diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 31e122ad942..677c8659eb7 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -20,7 +20,6 @@ # import heapq import logging -import weakref from collections import ChainMap, defaultdict from typing import ( TYPE_CHECKING, @@ -194,7 +193,7 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() # nb must be called this for @measure_func self.store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state - self.hs = weakref.proxy(hs) + self.hs = hs self._state_resolution_handler = hs.get_state_resolution_handler() self._storage_controllers = hs.get_storage_controllers() self._events_shard_config = hs.config.worker.events_shard_config diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index e912bdca472..d55c9e18ed6 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -20,7 +20,6 @@ # # import logging -import weakref from abc import ABCMeta from typing import TYPE_CHECKING, Any, Collection, Dict, Iterable, Optional, Union @@ -55,7 +54,7 @@ def __init__( db_conn: LoggingDatabaseConnection, hs: "HomeServer", ): - self.hs = weakref.proxy(hs) + self.hs = hs self.server_name = hs.hostname # nb must be called this for @cached self._clock = hs.get_clock() self.database_engine = database.engine diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 5f8a9433c9b..564a95ca87c 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -20,7 +20,6 @@ # import abc import logging -import weakref from enum import Enum, IntEnum from types import TracebackType from typing import ( @@ -248,8 +247,8 @@ class BackgroundUpdater: def __init__(self, hs: "HomeServer", database: "DatabasePool"): self._clock = hs.get_clock() - self.db_pool = weakref.proxy(database) - self.hs = weakref.proxy(hs) + self.db_pool = database + self.hs = hs self._database_name = database.name() diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index d3ab02a8539..293dee8b8fa 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -22,7 +22,6 @@ import itertools import logging -import weakref from collections import deque from typing import ( TYPE_CHECKING, @@ -198,7 +197,7 @@ def __init__( """ self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {} self._currently_persisting_rooms: Set[str] = set() - self._per_item_callback = weakref.proxy(per_item_callback) + self._per_item_callback = per_item_callback async def add_to_queue( self, @@ -346,7 +345,7 @@ def __init__( ) self._state_resolution_handler = hs.get_state_resolution_handler() self._state_controller = state_controller - self.hs = weakref.proxy(hs) + self.hs = hs async def _process_event_persist_queue_task( self, diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 771272e59de..95b4adf9ed4 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -19,7 +19,6 @@ # # import logging -import weakref from itertools import chain from typing import ( TYPE_CHECKING, @@ -71,11 +70,11 @@ class StateStorageController: def __init__(self, hs: "HomeServer", stores: "Databases"): self.server_name = hs.hostname # nb must be called this for @cached - self.hs = weakref.proxy(hs) + self.hs = hs self._clock = hs.get_clock() - self.stores = weakref.proxy(stores) - self._partial_state_events_tracker = weakref.proxy(PartialStateEventsTracker(stores.main)) - self._partial_state_room_tracker = weakref.proxy(PartialCurrentStateTracker(stores.main)) + self.stores = stores + self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) + self._partial_state_room_tracker = PartialCurrentStateTracker(stores.main) # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index e624cd85104..9267edc724c 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -23,7 +23,6 @@ import logging import time import types -import weakref from collections import defaultdict from time import monotonic as monotonic_time from typing import ( @@ -561,7 +560,7 @@ def __init__( database_config: DatabaseConnectionConfig, engine: BaseDatabaseEngine, ): - self.hs = weakref.proxy(hs) + self.hs = hs self._clock = hs.get_clock() self._txn_limit = database_config.config.get("txn_limit", 0) self._database_config = database_config diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 8a1b5757292..de55c452aea 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -20,7 +20,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast import attr @@ -171,9 +170,9 @@ def __init__( db_conn: LoggingDatabaseConnection, hs: "HomeServer", ): - self.hs = weakref.proxy(hs) + self.hs = hs self._clock = hs.get_clock() - self.database_engine = weakref.proxy(database.engine) + self.database_engine = database.engine super().__init__(database, db_conn, hs) diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 35f4bda7a9f..e87d75c8850 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -455,9 +455,7 @@ def __init__( hs.register_looping_call(self._clock.looping_call( self._update_client_ips_batch, 5 * 1000 )) - self.hs.get_reactor().addSystemEventTrigger( - "before", "shutdown", self._update_client_ips_batch - ) + hs.register_shutdown_handler("ClientIpWorkerStore _update_client_ips_batch", self._on_shutdown) @wrap_as_background_process("prune_old_user_ips") async def _prune_old_user_ips(self) -> None: diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index c6b70721bcf..697985215b7 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -21,7 +21,6 @@ import datetime import itertools import logging -import weakref from queue import Empty, PriorityQueue from typing import ( TYPE_CHECKING, @@ -140,7 +139,7 @@ def __init__( ): super().__init__(database, db_conn, hs) - self.hs = weakref.proxy(hs) + self.hs = hs if hs.config.worker.run_background_tasks: hs.register_looping_call(hs.get_clock().looping_call( diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 92e1829e7f6..5ddaef49212 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -22,7 +22,6 @@ import collections import itertools import logging -import weakref from collections import OrderedDict from typing import ( TYPE_CHECKING, @@ -237,9 +236,9 @@ def __init__( main_data_store: "DataStore", db_conn: LoggingDatabaseConnection, ): - self.hs = weakref.proxy(hs) - self.db_pool = weakref.proxy(db) - self.store = weakref.proxy(main_data_store) + self.hs = hs + self.db_pool = db + self.store = main_data_store self.database_engine = db.engine self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index 9069a48df04..42cd7eeeef8 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -95,11 +95,7 @@ def __init__( # lead to a race, as we may drop the lock while we are still processing. # However, a) it should be a small window, b) the lock is best effort # anyway and c) we want to really avoid leaking locks when we restart. - hs.get_reactor().addSystemEventTrigger( - "before", - "shutdown", - self._on_shutdown, - ) + hs.register_shutdown_handler("LockStore _on_shutdown", self._on_shutdown) self._acquiring_locks: Set[Tuple[str, str]] = set() diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index 5ca11731978..f5a6b98be71 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -18,7 +18,6 @@ # # import logging -import weakref from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, cast from synapse.metrics.background_process_metrics import wrap_as_background_process @@ -51,7 +50,7 @@ def __init__( ): super().__init__(database, db_conn, hs) self._clock = hs.get_clock() - self.hs = weakref.proxy(hs) + self.hs = hs if hs.config.redis.redis_enabled: # If we're using Redis, we can shift this update process off to diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index 26981fa2ce2..12cff1d3522 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -18,7 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import weakref from typing import ( TYPE_CHECKING, Any, @@ -98,7 +97,7 @@ def __init__( writers=hs.config.worker.writers.presence, ) - self.hs = weakref.proxy(hs) + self.hs = hs self._presence_on_startup = self._get_active_presence(db_conn) presence_cache_prefill, min_presence_val = self.db_pool.get_cache_dict( diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index fe7508a13e5..026a0517d20 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -41,7 +41,6 @@ Union, cast, ) -import weakref import attr from sortedcontainers import SortedList, SortedSet @@ -221,7 +220,7 @@ def __init__( writers: List[str], positive: bool = True, ) -> None: - self._db = weakref.proxy(db) + self._db = db self._notifier = notifier self._stream_name = stream_name self._instance_name = instance_name diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 2e0b7803e41..f858167e030 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -31,7 +31,6 @@ Tuple, TypeVar, ) -import weakref from prometheus_client import Gauge @@ -108,7 +107,7 @@ def __init__( self._next_values: Dict[Hashable, List[Tuple[V, defer.Deferred]]] = {} # The function to call with batches of values. - self._process_batch_callback = weakref.proxy(process_batch_callback) + self._process_batch_callback = process_batch_callback number_queued.labels(self._name).set_function( lambda: sum(len(q) for q in self._next_values.values()) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 608a4d48489..6b89ba20633 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -307,3 +307,6 @@ def register_hook(self, metric_name: str, hook: Callable[[], None]) -> None: """ self._pre_update_hooks[metric_name] = hook + + def clear(self) -> None: + self._pre_update_hooks.clear() diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 4d7a3749242..07294836e80 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -20,7 +20,6 @@ # import logging -import weakref from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional, Set, Tuple from twisted.python.failure import Failure @@ -101,8 +100,8 @@ class TaskScheduler: OCCASIONAL_REPORT_INTERVAL_MS = 5 * 60 * 1000 # 5 minutes def __init__(self, hs: "HomeServer"): - self._hs = weakref.proxy(hs) - self._store = weakref.proxy(hs.get_datastores().main) + self._hs = hs + self._store = hs.get_datastores().main self._clock = hs.get_clock() self._running_tasks: Set[str] = set() # A map between action names and their registered function diff --git a/tests/federation/test_federation_catch_up.py b/tests/federation/test_federation_catch_up.py index cab769cf789..1e1ed8e6426 100644 --- a/tests/federation/test_federation_catch_up.py +++ b/tests/federation/test_federation_catch_up.py @@ -1,7 +1,6 @@ from typing import Callable, Collection, List, Optional, Tuple from unittest import mock from unittest.mock import AsyncMock, Mock -import weakref from twisted.test.proto_helpers import MemoryReactor diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index e0ae2c7a5f3..25cf5269b8e 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -19,7 +19,6 @@ # # -import weakref from typing import Dict, Iterable, List, Optional from unittest.mock import AsyncMock, Mock @@ -415,7 +414,7 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = weakref.proxy(hs) + self.hs = hs # Mock the ApplicationServiceScheduler's _TransactionController's send method so that # we can track any outgoing ephemeral events self.send_mock = AsyncMock() diff --git a/tests/handlers/test_room_policy.py b/tests/handlers/test_room_policy.py index 8070dcdf22c..26642c18eac 100644 --- a/tests/handlers/test_room_policy.py +++ b/tests/handlers/test_room_policy.py @@ -12,7 +12,6 @@ # . # # -import weakref from typing import Optional from unittest import mock @@ -52,7 +51,7 @@ def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: ) def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = weakref.proxy(hs) + self.hs = hs self.handler = hs.get_room_policy_handler() main_store = self.hs.get_datastores().main diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index 5baa71fab8e..bf18c1e72a5 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -18,7 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import weakref from typing import Any, Dict, Iterable, List, Optional, Set, Tuple from unittest import mock @@ -128,7 +127,7 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = weakref.proxy(hs) + self.hs = hs self.handler = self.hs.get_room_summary_handler() # Create a user. @@ -1146,7 +1145,7 @@ class RoomSummaryTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = weakref.proxy(hs) + self.hs = hs self.handler = self.hs.get_room_summary_handler() # Create a user. diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index 498212075bb..18039a07e20 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -19,7 +19,6 @@ # # import json -import weakref from contextlib import contextmanager from typing import Generator, List, Set, Tuple from unittest import mock @@ -54,7 +53,7 @@ class HaveSeenEventsTestCase(unittest.HomeserverTestCase): ] def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: - self.hs = weakref.proxy(hs) + self.hs = hs self.store: EventsWorkerStore = hs.get_datastores().main self.user = self.register_user("user", "pass") From 31a607abee0c0bfd7949ff42f19c0178d37773e4 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 15 Aug 2025 16:08:18 -0600 Subject: [PATCH 006/181] Make clock capable of cleaning up it's own outstanding calls --- synapse/api/ratelimiting.py | 2 +- synapse/app/_base.py | 26 +++++---- synapse/app/homeserver.py | 17 ++++++ synapse/app/phone_stats_home.py | 26 ++++----- synapse/appservice/scheduler.py | 9 ++- synapse/config/logger.py | 2 +- synapse/federation/federation_client.py | 2 +- synapse/federation/federation_server.py | 4 +- synapse/federation/send_queue.py | 2 +- synapse/federation/sender/__init__.py | 4 +- synapse/handlers/account_validity.py | 2 +- synapse/handlers/appservice.py | 2 +- synapse/handlers/auth.py | 4 +- synapse/handlers/device.py | 12 ++-- synapse/handlers/e2e_keys.py | 3 +- synapse/handlers/federation.py | 4 +- synapse/handlers/federation_event.py | 2 +- synapse/handlers/message.py | 7 ++- synapse/handlers/pagination.py | 4 +- synapse/handlers/presence.py | 12 ++-- synapse/handlers/read_marker.py | 2 +- synapse/handlers/room_member.py | 4 +- synapse/handlers/sync.py | 3 +- synapse/handlers/typing.py | 4 +- synapse/handlers/worker_lock.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/media/media_repository.py | 10 ++-- synapse/media/url_previewer.py | 4 +- synapse/metrics/common_usage_metrics.py | 4 +- synapse/notifier.py | 4 +- synapse/replication/tcp/client.py | 2 +- synapse/replication/tcp/redis.py | 2 +- synapse/replication/tcp/resource.py | 2 +- synapse/rest/client/push_rule.py | 2 +- synapse/rest/client/transactions.py | 2 +- synapse/rest/synapse/mas/_base.py | 2 +- synapse/server.py | 57 +++++++++++-------- synapse/state/__init__.py | 4 +- synapse/storage/controllers/purge_events.py | 4 +- synapse/storage/controllers/state.py | 2 +- synapse/storage/database.py | 2 +- synapse/storage/databases/main/cache.py | 1 - .../storage/databases/main/censor_events.py | 2 +- synapse/storage/databases/main/client_ips.py | 8 +-- synapse/storage/databases/main/deviceinbox.py | 4 +- synapse/storage/databases/main/devices.py | 4 +- .../databases/main/event_federation.py | 6 +- .../databases/main/event_push_actions.py | 14 ++--- .../storage/databases/main/events_worker.py | 4 +- synapse/storage/databases/main/lock.py | 6 +- synapse/storage/databases/main/metrics.py | 2 +- .../storage/databases/main/registration.py | 8 +-- synapse/storage/databases/main/roommember.py | 4 +- synapse/storage/databases/main/session.py | 2 +- .../storage/databases/main/transactions.py | 2 +- synapse/util/__init__.py | 33 ++++++++++- synapse/util/caches/dictionary_cache.py | 1 + synapse/util/caches/expiringcache.py | 2 +- synapse/util/caches/lrucache.py | 4 +- synapse/util/task_scheduler.py | 9 ++- tests/appservice/test_scheduler.py | 2 + 61 files changed, 222 insertions(+), 162 deletions(-) diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index 79d1d1cce62..ce2a29dd7cd 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -96,7 +96,7 @@ def __init__( # * The rate_hz (leak rate) of this particular bucket. self.actions: Dict[Hashable, Tuple[float, float, float]] = {} - hs.register_looping_call(self.clock.looping_call(self._prune_message_counts, 60 * 1000)) + self.clock.looping_call(self._prune_message_counts, 60 * 1000) def _get_key( self, requester: Optional[Requester], key: Optional[Hashable] diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 6472c426e16..679c3455fb7 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -100,13 +100,13 @@ logger = logging.getLogger(__name__) # list of tuples of function, args list, kwargs dict -_sighup_callbacks: List[ - Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]] -] = [] +_sighup_callbacks: Dict[str, + List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] +] = {} P = ParamSpec("P") -def register_sighup(func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: +def register_sighup(server_name: str, func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: """ Register a function to be called when a SIGHUP occurs. @@ -114,7 +114,12 @@ def register_sighup(func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ - _sighup_callbacks.append((func, args, kwargs)) + + _sighup_callbacks.setdefault(server_name, []).append((func, args, kwargs)) + + +def unregister_sighups(server_name: str) -> None: + _sighup_callbacks.pop(server_name, []) def start_worker_reactor( @@ -537,8 +542,9 @@ async def handle_sighup(*args: Any, **kwargs: Any) -> None: # we're not using systemd. sdnotify(b"RELOADING=1") - for i, args, kwargs in _sighup_callbacks: - i(*args, **kwargs) + for _, v in _sighup_callbacks.items(): + for i, args, kwargs in v: + i(*args, **kwargs) sdnotify(b"READY=1") @@ -552,9 +558,8 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: signal.signal(signal.SIGHUP, run_sighup) - # TODO: FIXME, weakref needed here - register_sighup(refresh_certificate, weakref.proxy(hs)) - register_sighup(reload_cache_config, hs.config) + register_sighup(hs.config.server.server_name, refresh_certificate, hs) + register_sighup(hs.config.server.server_name, reload_cache_config, hs.config) # Apply the cache config. hs.config.caches.resize_all_caches() @@ -591,6 +596,7 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: hs.get_pusherpool().start() # Log when we start the shut down process. + # TODO: fixme hs.get_reactor().addSystemEventTrigger( "before", "shutdown", logger.info, "Shutting down..." ) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 57a15ef2a65..bd35fcccfa3 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -86,8 +86,25 @@ class SynapseHomeServer(HomeServer): def shutdown(self) -> None: super().shutdown() + + logger.info("Shutting down listening services") for listener in self._listening_services: + logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) + #listener.socket.shutdown(0) + #listener.stopListening() + #listener.stopConsuming() + #listener.loseConnection() + + # Preferred over connectionLost since it allows buffers to flush + listener.unregisterProducer() listener.loseConnection() + + # NOTE: not guaranteed to immediately shutdown + # Sometimes takes a second for some deferred to fire that cancels the socket + # But seems to always do so within a minute + # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. + + #listener.connectionLost(None) self._listening_services.clear() #self._reactor.stop() diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 4dd3c4a6914..1b170c3dca1 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -198,18 +198,16 @@ def performance_stats_init() -> None: # Rather than update on per session basis, batch up the requests. # If you increase the loop period, the accuracy of user_daily_visits # table will decrease - hs.register_looping_call( - clock.looping_call( - hs.get_datastores().main.generate_user_daily_visits, - 5 * MILLISECONDS_PER_SECOND, - ) + clock.looping_call( + hs.get_datastores().main.generate_user_daily_visits, + 5 * MILLISECONDS_PER_SECOND, ) # monthly active user limiting functionality - hs.register_looping_call(clock.looping_call( + clock.looping_call( hs.get_datastores().main.reap_monthly_active_users, ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, - )) + ) # TODO: (devon) how does this hold onto a DB pool reference? #hs.register_background_process(hs.get_datastores().main.reap_monthly_active_users()) @@ -235,18 +233,16 @@ async def generate_monthly_active_users() -> None: if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only: generate_monthly_active_users() - hs.register_looping_call(clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000)) + clock.looping_call(generate_monthly_active_users, 5 * 60 * 1000) # End of monthly active user settings if hs.config.metrics.report_stats: logger.info("Scheduling stats reporting for 3 hour intervals") - hs.register_looping_call( - clock.looping_call( - phone_stats_home, - PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, - hs, - stats, - ) + clock.looping_call( + phone_stats_home, + PHONE_HOME_INTERVAL_SECONDS * MILLISECONDS_PER_SECOND, + hs, + stats, ) # We need to defer this init for the cases that we daemonize diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index d90dc2d3e1a..df499817eed 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -113,7 +113,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.as_api = hs.get_application_service_api() - self.txn_ctrl = _TransactionController(self.clock, self.store, self.as_api) + self.txn_ctrl = _TransactionController(hs, self.clock, self.store, self.as_api) self.queuer = _ServiceQueuer(self.txn_ctrl, self.clock, hs) async def start(self) -> None: @@ -359,10 +359,11 @@ class _TransactionController: (Note we have only have one of these in the homeserver.) """ - def __init__(self, clock: Clock, store: DataStore, as_api: ApplicationServiceApi): + def __init__(self, hs: "HomeServer", clock: Clock, store: DataStore, as_api: ApplicationServiceApi): self.clock = clock self.store = store self.as_api = as_api + self.hs = hs # map from service id to recoverer instance self.recoverers: Dict[str, "_Recoverer"] = {} @@ -446,7 +447,7 @@ def start_recoverer(self, service: ApplicationService) -> None: logger.info("Starting recoverer for AS ID %s", service.id) assert service.id not in self.recoverers recoverer = self.RECOVERER_CLASS( - self.clock, self.store, self.as_api, service, self.on_recovered + self.hs, self.clock, self.store, self.as_api, service, self.on_recovered ) self.recoverers[service.id] = recoverer recoverer.recover() @@ -486,12 +487,14 @@ class _Recoverer: def __init__( self, + hs: "HomeServer", clock: Clock, store: DataStore, as_api: ApplicationServiceApi, service: ApplicationService, callback: Callable[["_Recoverer"], Awaitable[None]], ): + self.hs = hs self.clock = clock self.store = store self.as_api = as_api diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 3f86ec11691..1faed0b49a5 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -348,7 +348,7 @@ def setup_logging( # Add a SIGHUP handler to reload the logging configuration, if one is available. from synapse.app import _base as appbase - appbase.register_sighup(_reload_logging_config, log_config_path) + appbase.register_sighup(config.server.server_name, _reload_logging_config, log_config_path) # Log immediately so we can grep backwards. logger.warning("***** STARTING SERVER *****") diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 4acc8cd8b6d..a27d41d408b 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -133,7 +133,7 @@ def __init__(self, hs: "HomeServer"): super().__init__(hs) self.pdu_destination_tried: Dict[str, Dict[str, int]] = {} - hs.register_looping_call(self._clock.looping_call(self._clear_tried_cache, 60 * 1000)) + self._clock.looping_call(self._clear_tried_cache, 60 * 1000) self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 87d237f2add..cabb9236202 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -151,7 +151,7 @@ def __init__(self, hs: "HomeServer"): # with FederationHandlerRegistry. hs.get_directory_handler() - self._server_linearizer = Linearizer("fed_server") + self._server_linearizer = Linearizer("fed_server", clock=hs.get_clock()) # origins that we are currently processing a transaction from. # a dict from origin to txn id. @@ -300,7 +300,7 @@ async def on_incoming_transaction( # Start a periodic check for old staged events. This is to handle # the case where locks time out, e.g. if another process gets killed # without dropping its locks. - self.hs.register_looping_call(self._clock.looping_call(self._handle_old_staged_events, 60 * 1000)) + self._clock.looping_call(self._handle_old_staged_events, 60 * 1000) # keep this as early as possible to make the calculated origin ts as # accurate as possible. diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index ae94a328cf2..b7f0b0de30a 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -128,7 +128,7 @@ def register(name: str, queue: Sized) -> None: ]: register(queue_name, getattr(self, queue_name)) - hs.register_looping_call(self.clock.looping_call(self._clear_queue, 30 * 1000)) + self.clock.looping_call(self._clear_queue, 30 * 1000) def _next_pos(self) -> int: pos = self.pos diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index b38f9680208..ab54422da1b 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -429,12 +429,12 @@ def __init__(self, hs: "HomeServer"): ) # Regularly wake up destinations that have outstanding PDUs to be caught up - hs.register_looping_call(self.clock.looping_call_now( + self.clock.looping_call_now( run_as_background_process, WAKEUP_RETRY_PERIOD_SEC * 1000.0, "wake_destinations_needing_catchup", self._wake_destinations_needing_catchup, - )) + ) def _get_per_destination_queue( self, destination: str diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 64f9c314337..7004d95a0f3 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -72,7 +72,7 @@ def __init__(self, hs: "HomeServer"): # Check the renewal emails to send and send them every 30min. if hs.config.worker.run_background_tasks: - hs.register_looping_call(self.clock.looping_call(self._send_renewal_emails, 30 * 60 * 1000)) + self.clock.looping_call(self._send_renewal_emails, 30 * 60 * 1000) async def is_user_expired(self, user_id: str) -> bool: """Checks if a user has expired against third-party modules. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 28a6158f9e6..6f788a2e483 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -93,7 +93,7 @@ def __init__(self, hs: "HomeServer"): self.is_processing = False self._ephemeral_events_linearizer = Linearizer( - name="appservice_ephemeral_events" + name="appservice_ephemeral_events", clock=hs.get_clock() ) def notify_interested_services(self, max_token: RoomStreamToken) -> None: diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index dd7a001475e..194defe1f58 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -245,12 +245,12 @@ def __init__(self, hs: "HomeServer"): # Expire old UI auth sessions after a period of time. if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( run_as_background_process, 5 * 60 * 1000, "expire_old_sessions", self._expire_old_sessions, - )) + ) # Load the SSO HTML templates. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index f64dfe6e60d..bc417ed7ef0 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -190,12 +190,12 @@ def __init__(self, hs: "HomeServer"): hs.config.worker.run_background_tasks and self._delete_stale_devices_after is not None ): - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( run_as_background_process, DELETE_STALE_DEVICES_INTERVAL_MS, "delete_stale_devices", self._delete_stale_devices, - )) + ) async def _delete_stale_devices(self) -> None: """Background task that deletes devices which haven't been accessed for more than @@ -1445,8 +1445,8 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler - self._remote_edu_linearizer = Linearizer(name="remote_device_list") - self._resync_linearizer = Linearizer(name="remote_device_resync") + self._remote_edu_linearizer = Linearizer(name="remote_device_list", clock=hs.get_clock()) + self._resync_linearizer = Linearizer(name="remote_device_resync", clock=hs.get_clock()) # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[ @@ -1468,12 +1468,12 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): # Attempt to resync out of sync device lists every 30s. self._resync_retry_lock = Lock() - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( run_as_background_process, 30 * 1000, func=self._maybe_retry_device_resync, desc="_maybe_retry_device_resync", - )) + ) @trace async def incoming_device_list_update( diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 00708c7a4c6..96f53bc40ed 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -114,6 +114,7 @@ def __init__(self, hs: "HomeServer"): self._query_devices_linearizer = Linearizer( name="query_devices", max_count=10, + clock=hs.get_clock(), ) self._query_appservices_for_otks = ( @@ -1765,7 +1766,7 @@ def __init__(self, hs: "HomeServer"): assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler - self._remote_edu_linearizer = Linearizer(name="remote_signing_key") + self._remote_edu_linearizer = Linearizer(name="remote_signing_key", clock=hs.get_clock()) # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[str, List[Tuple[JsonDict, JsonDict]]] = {} diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 87afb0a056d..4a64092f7d4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -157,7 +157,7 @@ def __init__(self, hs: "HomeServer"): self._notifier = hs.get_notifier() self._worker_locks = hs.get_worker_locks_handler() - self._room_backfill = Linearizer("room_backfill") + self._room_backfill = Linearizer("room_backfill", clock=hs.get_clock()) self._third_party_event_rules = ( hs.get_module_api_callbacks().third_party_event_rules @@ -177,7 +177,7 @@ def __init__(self, hs: "HomeServer"): # When the lock is held for a given room, no other concurrent code may # partial state or un-partial state the room. self._is_partial_state_room_linearizer = Linearizer( - name="_is_partial_state_room_linearizer" + name="_is_partial_state_room_linearizer", clock=hs.get_clock() ) # if this is the main process, fire off a background process to resume diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 93ef8c7dfef..a7bdf28370b 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -182,7 +182,7 @@ def __init__(self, hs: "HomeServer"): # federation event staging area. self.room_queues: Dict[str, List[Tuple[EventBase, str]]] = {} - self._room_pdu_linearizer = Linearizer("fed_room_pdu") + self._room_pdu_linearizer = Linearizer("fed_room_pdu", clock=hs.get_clock()) async def on_receive_pdu(self, origin: str, pdu: EventBase) -> None: """Process a PDU received via a federation /send/ transaction diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index a4d94eccd9d..a5d53b74363 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -92,6 +92,7 @@ class MessageHandler: """Contains some read only APIs to get state about a room""" def __init__(self, hs: "HomeServer"): + self.hs = hs self.auth = hs.get_auth() self.clock = hs.get_clock() self.state = hs.get_state_handler() @@ -506,7 +507,7 @@ def __init__(self, hs: "HomeServer"): # We limit concurrent event creation for a room to 1. This prevents state resolution # from occurring when sending bursts of events to a local room - self.limiter = Linearizer(max_count=1, name="room_event_creation_limit") + self.limiter = Linearizer(max_count=1, name="room_event_creation_limit", clock=hs.get_clock()) self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() @@ -538,13 +539,13 @@ def __init__(self, hs: "HomeServer"): self.config.worker.run_background_tasks and self.config.server.cleanup_extremities_with_dummy_events ): - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( lambda: run_as_background_process( "send_dummy_events_to_fill_extremities", self._send_dummy_events_to_fill_extremities, ), 5 * 60 * 1000, - )) + ) self._message_handler = hs.get_message_handler() diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 8eb2cf31ce0..4070b74b7af 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -115,14 +115,14 @@ def __init__(self, hs: "HomeServer"): for job in hs.config.retention.retention_purge_jobs: logger.info("Setting up purge job with config: %s", job) - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( run_as_background_process, job.interval, "purge_history_for_rooms_in_range", self.purge_history_for_rooms_in_range, job.shortest_max_lifetime, job.longest_max_lifetime, - )) + ) self._task_scheduler.register_action( self._purge_history, PURGE_HISTORY_ACTION_NAME diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 1853f917eb1..c5ebfcd25ca 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -507,11 +507,11 @@ def __init__(self, hs: "HomeServer"): self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs) self._set_state_client = ReplicationPresenceSetState.make_client(hs) - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( self.send_stop_syncing, UPDATE_SYNCING_USERS_MS - )) + ) - hs.register_shutdown_handler("generic_presence.on_shutdown", self._on_shutdown) + hs.register_async_shutdown_handler("generic_presence.on_shutdown", self._on_shutdown) @wrap_as_background_process("WorkerPresenceHandler._on_shutdown") async def _on_shutdown(self) -> None: @@ -804,7 +804,7 @@ def __init__(self, hs: "HomeServer"): # have not yet been persisted self.unpersisted_users_changes: Set[str] = set() - hs.register_shutdown_handler("presence.on_shutdown", self._on_shutdown) + hs.register_async_shutdown_handler("presence.on_shutdown", self._on_shutdown) # Keeps track of the number of *ongoing* syncs on this process. While # this is non zero a user will never go offline. @@ -827,7 +827,7 @@ def __init__(self, hs: "HomeServer"): ] = {} self.external_process_last_updated_ms: Dict[str, int] = {} - self.external_sync_linearizer = Linearizer(name="external_sync_linearizer") + self.external_sync_linearizer = Linearizer(name="external_sync_linearizer", clock=hs.get_clock()) if self._track_presence: # Start a LoopingCall in 30s that fires every 5s. @@ -2399,7 +2399,7 @@ def __init__(self, hs: "HomeServer", presence_handler: BasePresenceHandler): self._current_tokens: Dict[str, int] = {} if self._queue_presence_updates: - hs.register_looping_call(self._clock.looping_call(self._clear_queue, self._CLEAR_ITEMS_EVERY_MS)) + self._clock.looping_call(self._clear_queue, self._CLEAR_ITEMS_EVERY_MS) def _clear_queue(self) -> None: """Clear out older entries from the queue.""" diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index fb39c8e04b5..d04412aea2b 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -36,7 +36,7 @@ class ReadMarkerHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.account_data_handler = hs.get_account_data_handler() - self.read_marker_linearizer = Linearizer(name="read_marker") + self.read_marker_linearizer = Linearizer(name="read_marker", clock=hs.get_clock()) async def received_client_read_marker( self, room_id: str, user_id: str, event_id: str diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index d7e9a85c029..7e5ed6d33c4 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -114,8 +114,8 @@ def __init__(self, hs: "HomeServer"): if self.hs.config.server.include_profile_data_on_invite: self._membership_types_to_include_profile_data_in.add(Membership.INVITE) - self.member_linearizer: Linearizer = Linearizer(name="member") - self.member_as_limiter = Linearizer(max_count=10, name="member_as_limiter") + self.member_linearizer: Linearizer = Linearizer(name="member", clock=hs.get_clock()) + self.member_as_limiter = Linearizer(max_count=10, name="member_as_limiter", clock=hs.get_clock()) self.clock = hs.get_clock() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 44068941732..5fbf6c07a4d 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1137,7 +1137,8 @@ def get_lazy_loaded_members_cache( ) if cache is None: logger.debug("creating LruCache for %r", cache_key) - cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) + cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE, + ) self.lazy_loaded_members_cache[cache_key] = cache else: logger.debug("found LruCache for %r", cache_key) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index dec0f47311a..c59f1f39c97 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -105,8 +105,8 @@ def __init__(self, hs: "HomeServer"): self._rooms_updated: Set[str] = set() - hs.register_looping_call(self.clock.looping_call(self._handle_timeouts, 5000)) - hs.register_looping_call(self.clock.looping_call(self._prune_old_typing, FORGET_TIMEOUT)) + self.clock.looping_call(self._handle_timeouts, 5000) + self.clock.looping_call(self._prune_old_typing, FORGET_TIMEOUT) def _reset(self) -> None: """Reset the typing handler's data caches.""" diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index d9e61962e51..3077e9e463d 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -78,7 +78,7 @@ def __init__(self, hs: "HomeServer") -> None: Tuple[str, str], WeakSet[Union[WaitingLock, WaitingMultiLock]] ] = {} - hs.register_looping_call(self._clock.looping_call(self._cleanup_locks, 30_000)) + self._clock.looping_call(self._cleanup_locks, 30_000) self._notifier.add_lock_released_callback(self._on_lock_released) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 31bde39c71a..677cae02bb3 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -476,7 +476,7 @@ def __init__( use_proxy=True, ) - self.remote_download_linearizer = Linearizer("remote_download_linearizer", 6) + self.remote_download_linearizer = Linearizer("remote_download_linearizer", 6, clock=hs.get_clock()) def wake_destination(self, destination: str) -> None: """Called when the remote server may have come back online.""" diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index bc6d6c574b0..f135ad46aae 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -108,7 +108,7 @@ def __init__(self, hs: "HomeServer"): self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails self.thumbnail_requirements = hs.config.media.thumbnail_requirements - self.remote_media_linearizer = Linearizer(name="media_remote") + self.remote_media_linearizer = Linearizer(name="media_remote", clock=hs.get_clock()) self.recently_accessed_remotes: Set[Tuple[str, str]] = set() self.recently_accessed_locals: Set[str] = set() @@ -147,9 +147,9 @@ def __init__(self, hs: "HomeServer"): hs, self.primary_base_path, self.filepaths, storage_providers ) - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( self._start_update_recently_accessed, UPDATE_RECENTLY_ACCESSED_TS - )) + ) # Media retention configuration options self._media_retention_local_media_lifetime_ms = ( @@ -166,10 +166,10 @@ def __init__(self, hs: "HomeServer"): ): # Run the background job to apply media retention rules routinely, # with the duration between runs dictated by the homeserver config. - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( self._start_apply_media_retention_rules, MEDIA_RETENTION_CHECK_PERIOD_MS, - )) + ) if hs.config.media.url_preview_enabled: self.url_previewer: Optional[UrlPreviewer] = UrlPreviewer( diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index f60899eaf59..7e5ba43b814 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -208,9 +208,9 @@ def __init__( ) if self._worker_run_media_background_jobs: - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( self._start_expire_url_cache_data, 10 * 1000 - )) + ) async def preview(self, url: str, user: UserID, ts: int) -> bytes: # the in-memory cache: diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index 8d7229d9aae..eeb38de8138 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -65,12 +65,12 @@ async def setup(self) -> None: run_as_background_process( desc="common_usage_metrics_update_gauges", func=self._update_gauges ) - self.hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( run_as_background_process, 5 * 60 * 1000, desc="common_usage_metrics_update_gauges", func=self._update_gauges, - )) + ) async def _collect(self) -> CommonUsageMetrics: """Collect the common metrics and either create the CommonUsageMetrics object to diff --git a/synapse/notifier.py b/synapse/notifier.py index 07e5ea47132..5b65218a479 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -251,9 +251,9 @@ def __init__(self, hs: "HomeServer"): self.state_handler = hs.get_state_handler() - hs.register_looping_call(self.clock.looping_call( + self.clock.looping_call( self.remove_expired_streams, self.UNUSED_STREAM_EXPIRY_MS - )) + ) # This is not a very cheap test to perform, but it's only executed # when rendering the metrics page, which is likely once per minute at diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 177b183aaa9..f3b4164a520 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -427,7 +427,7 @@ def __init__(self, hs: "HomeServer"): # to. This is always set before we use it. self.federation_position: Optional[int] = None - self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer") + self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer", clock=hs.get_clock()) async def process_replication_rows( self, stream_name: str, token: int, rows: list diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 180daee9196..78220671357 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -278,7 +278,7 @@ def __init__( convertNumbers=convertNumbers, ) - hs.register_looping_call(hs.get_clock().looping_call(self._send_ping, 30 * 1000)) + hs.get_clock().looping_call(self._send_ping, 30 * 1000) @wrap_as_background_process("redis_ping") async def _send_ping(self) -> None: diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index 559b785a2d3..0080a76f6f1 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -112,7 +112,7 @@ def __init__(self, hs: "HomeServer"): # # Note that if the position hasn't advanced then we won't send anything. if any(EventsStream.NAME == s.NAME for s in self.streams): - hs.register_looping_call(self.clock.looping_call(self.on_notifier_poke, 1000)) + self.clock.looping_call(self.on_notifier_poke, 1000) def on_notifier_poke(self) -> None: """Checks if there is actually any new data and sends it to the diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index af042504c9b..58cbb8a5869 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -63,7 +63,7 @@ def __init__(self, hs: "HomeServer"): hs.get_instance_name() in hs.config.worker.writers.push_rules ) self._push_rules_handler = hs.get_push_rules_handler() - self._push_rule_linearizer = Linearizer(name="push_rules") + self._push_rule_linearizer = Linearizer(name="push_rules", clock=hs.get_clock()) async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: if not self._is_push_worker: diff --git a/synapse/rest/client/transactions.py b/synapse/rest/client/transactions.py index 49de9976f6f..571ba2fa623 100644 --- a/synapse/rest/client/transactions.py +++ b/synapse/rest/client/transactions.py @@ -56,7 +56,7 @@ def __init__(self, hs: "HomeServer"): ] = {} # Try to clean entries every 30 mins. This means entries will exist # for at *LEAST* 30 mins, and at *MOST* 60 mins. - hs.register_looping_call(self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS)) + self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS) def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hashable: """A helper function which returns a transaction key that can be used diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py index caf392fc3ae..ed2d3dea946 100644 --- a/synapse/rest/synapse/mas/_base.py +++ b/synapse/rest/synapse/mas/_base.py @@ -31,7 +31,7 @@ def __init__(self, hs: "HomeServer"): # dependency but required if msc3861 is enabled from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth - DirectServeJsonResource.__init__(self, extract_context=True) + DirectServeJsonResource.__init__(self, extract_context=True, clock=hs.get_clock()) auth = hs.get_auth() assert isinstance(auth, MSC3861DelegatedAuth) self.msc3861_auth = auth diff --git a/synapse/server.py b/synapse/server.py index 85d69e71cb5..f078241470c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -32,7 +32,7 @@ from twisted.internet import defer from typing_extensions import TypeAlias -from twisted.internet.interfaces import IOpenSSLContextFactory +from twisted.internet.interfaces import IDelayedCall, IOpenSSLContextFactory from twisted.internet.task import LoopingCall from twisted.internet.tcp import Port from twisted.python.threadpool import ThreadPool @@ -323,10 +323,10 @@ def __init__( # This attribute is set by the free function `refresh_certificate`. self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None - self._looping_calls: List[LoopingCall] = [] self._later_gauges: List[LaterGauge] = [] self._background_processes: List[defer.Deferred] = [] - self._shutdown_handlers: List[ShutdownInfo] = [] + self._async_shutdown_handlers: List[ShutdownInfo] = [] + self._sync_shutdown_handlers: List[ShutdownInfo] = [] def shutdown(self) -> None: logger.info("Received shutdown request") @@ -351,10 +351,10 @@ def shutdown(self) -> None: # self.get_replication_command_handler()._connection.disconnect() # self.get_replication_command_handler()._factory.synapse_outbound_redis_connection.disconnect() # self.get_replication_command_handler()._factory = None - self.get_replication_command_handler()._streams.clear() - self.get_replication_command_handler()._streams_to_replicate.clear() - # XXX: Does this trample on other tenants? - STREAMS_MAP.clear() + # self.get_replication_command_handler()._streams.clear() + # self.get_replication_command_handler()._streams_to_replicate.clear() + # # XXX: Does this trample on other tenants? + # STREAMS_MAP.clear() from synapse.util.batching_queue import number_of_keys number_of_keys.clear() @@ -371,13 +371,11 @@ def shutdown(self) -> None: for db in self.get_datastores().databases: db.stop_background_updates() - logger.info("Stopping looping calls: %d", len(self._looping_calls)) - for looping_call in self._looping_calls: - looping_call.stop() - self._looping_calls.clear() - - self._looping_calls.clear() - + logger.info("Stopping looping calls") + self.get_clock().cancel_all_looping_calls() + logger.info("Stopping call_later calls") + self.get_clock().cancel_all_delayed_calls() + logger.info("Stopping background processes: %d", len(self._background_processes)) for process in self._background_processes: process.cancel() @@ -389,17 +387,26 @@ def shutdown(self) -> None: # only clear them for this server CACHE_METRIC_REGISTRY.clear() - for shutdown_handler in self._shutdown_handlers: + for shutdown_handler in self._async_shutdown_handlers: logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) # TODO: we should probably run these #yield defer.ensureDeferred(shutdown_handler.func()) - self._shutdown_handlers.clear() + self._async_shutdown_handlers.clear() + + for shutdown_handler in self._sync_shutdown_handlers: + logger.info("Shutting down %s", shutdown_handler.desc) + self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) + shutdown_handler.func() + self._sync_shutdown_handlers.clear() + + from synapse.app._base import unregister_sighups + unregister_sighups(self.config.server.server_name) for call in self.get_reactor().getDelayedCalls(): call.cancel() - def register_shutdown_handler(self, desc: "LiteralString", shutdown_func: Callable[..., Any]) -> None: + def register_async_shutdown_handler(self, desc: "LiteralString", shutdown_func: Callable[..., Any]) -> None: id = self.get_reactor().addSystemEventTrigger( "before", "shutdown", @@ -407,10 +414,15 @@ def register_shutdown_handler(self, desc: "LiteralString", shutdown_func: Callab desc, shutdown_func, ) - self._shutdown_handlers.append(ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id)) + self._async_shutdown_handlers.append(ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id)) - def register_looping_call(self, looping_call: LoopingCall) -> None: - self._looping_calls.append(looping_call) + def register_sync_shutdown_handler(self, phase: str, eventType: str, desc: "LiteralString", shutdown_func: Callable[..., Any]) -> None: + id = self.get_reactor().addSystemEventTrigger( + phase, + eventType, + shutdown_func, + ) + self._sync_shutdown_handlers.append(ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id)) def register_later_gauge(self, later_gauge: LaterGauge) -> None: self._later_gauges.append(later_gauge) @@ -1084,10 +1096,7 @@ def get_media_sender_thread_pool(self) -> ThreadPool: ) media_threadpool.start() - # TODO: what do. it's different since it uses during instead of before - self.get_reactor().addSystemEventTrigger( - "during", "shutdown", media_threadpool.stop - ) + hs.register_sync_shutdown_handler("during", "shutdown", "Homeserver media_threadpool.stop", media_threadpool.stop) # Register the threadpool with our metrics. register_threadpool("media", media_threadpool) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 677c8659eb7..270b39263ca 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -635,7 +635,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() - self.resolve_linearizer = Linearizer(name="state_resolve_lock") + self.resolve_linearizer = Linearizer(name="state_resolve_lock", clock=hs.get_clock()) # dict of set of event_ids -> _StateCacheEntry. self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( @@ -660,7 +660,7 @@ def __init__(self, hs: "HomeServer"): _StateResMetrics ) - hs.register_looping_call(self.clock.looping_call(self._report_metrics, 120 * 1000)) + self.clock.looping_call(self._report_metrics, 120 * 1000) async def resolve_state_groups( self, diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 8acc98f0b63..08e6412112b 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -49,9 +49,9 @@ def __init__(self, hs: "HomeServer", stores: Databases): self.stores = stores if hs.config.worker.run_background_tasks: - hs.register_looping_call(hs.get_clock().looping_call( + hs.get_clock().looping_call( self._delete_state_groups_loop, 60 * 1000 - )) + ) self.stores.state.db_pool.updates.register_background_update_handler( _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE, diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 95b4adf9ed4..58a02d10b6f 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -78,7 +78,7 @@ def __init__(self, hs: "HomeServer", stores: "Databases"): # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. - self._joined_host_linearizer = Linearizer("_JoinedHostsCache") + self._joined_host_linearizer = Linearizer("_JoinedHostsCache", clock=hs.get_clock()) def notify_event_un_partial_stated(self, event_id: str) -> None: self._partial_state_events_tracker.notify_un_partial_stated(event_id) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 9267edc724c..40fb5ee6c90 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -672,7 +672,7 @@ def loop() -> None: "Total database time: %.3f%% {%s}", ratio * 100, top_three_counters ) - self.hs.register_looping_call(self._clock.looping_call(loop, 10000)) + self._clock.looping_call(loop, 10000) def new_transaction( self, diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 7bee8a1a42c..dc37f671101 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -794,7 +794,6 @@ async def _clean_up_cache_invalidation_wrapper(self) -> None: else: next_interval = REGULAR_CLEANUP_INTERVAL_MS - # TODO: (devon) what is the best way to track these manually looping calls? self.hs.get_clock().call_later( next_interval / 1000, self._clean_up_cache_invalidation_wrapper ) diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py index 1c03874ab41..5b15fd707d2 100644 --- a/synapse/storage/databases/main/censor_events.py +++ b/synapse/storage/databases/main/censor_events.py @@ -54,7 +54,7 @@ def __init__( hs.config.worker.run_background_tasks and self.hs.config.server.redaction_retention_period is not None ): - hs.register_looping_call(hs.get_clock().looping_call(self._censor_redactions, 5 * 60 * 1000)) + hs.get_clock().looping_call(self._censor_redactions, 5 * 60 * 1000) @wrap_as_background_process("_censor_redactions") async def _censor_redactions(self) -> None: diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index e87d75c8850..12d372935dd 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -441,7 +441,7 @@ def __init__( ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: - hs.register_looping_call(self._clock.looping_call(self._prune_old_user_ips, 5 * 1000)) + self._clock.looping_call(self._prune_old_user_ips, 5 * 1000) if self._update_on_this_worker: # This is the designated worker that can write to the client IP @@ -452,10 +452,10 @@ def __init__( Tuple[str, str, str], Tuple[str, Optional[str], int] ] = {} - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._update_client_ips_batch, 5 * 1000 - )) - hs.register_shutdown_handler("ClientIpWorkerStore _update_client_ips_batch", self._on_shutdown) + ) + hs.register_async_shutdown_handler("ClientIpWorkerStore _update_client_ips_batch", self._on_shutdown) @wrap_as_background_process("prune_old_user_ips") async def _prune_old_user_ips(self) -> None: diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 18484c10d8f..5c57b4d58bf 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -153,12 +153,12 @@ def __init__( ) if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( run_as_background_process, DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS, "_delete_old_federation_inbox_rows", self._delete_old_federation_inbox_rows, - )) + ) def process_replication_rows( self, diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 81a06e44dc0..6ed9f858003 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -194,9 +194,9 @@ def __init__( ) if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._prune_old_outbound_device_pokes, 60 * 60 * 1000 - )) + ) def process_replication_rows( self, stream_name: str, instance_name: str, token: int, rows: Iterable[Any] diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 697985215b7..8e623bf0617 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -142,9 +142,9 @@ def __init__( self.hs = hs if hs.config.worker.run_background_tasks: - hs.register_looping_call(hs.get_clock().looping_call( + hs.get_clock().looping_call( self._delete_old_forward_extrem_cache, 60 * 60 * 1000 - )) + ) # Cache of event ID to list of auth event IDs and their depths. self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache( @@ -158,7 +158,7 @@ def __init__( # index. self.tests_allow_no_chain_cover_index = True - hs.register_looping_call(self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000)) + self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000) if isinstance(self.database_engine, PostgresEngine): self.db_pool.updates.register_background_validate_constraint_and_delete_rows( diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 32a7d51692f..5ab41f64111 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -276,22 +276,20 @@ def __init__( self._find_stream_orderings_for_times_txn(cur) cur.close() - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._find_stream_orderings_for_times, 10 * 60 * 1000 - )) + ) self._rotate_count = 10000 self._doing_notif_rotation = False if hs.config.worker.run_background_tasks: - hs.register_looping_call( - self._clock.looping_call( - self._rotate_notifs, 30 * 1000 - ) + self._clock.looping_call( + self._rotate_notifs, 30 * 1000 ) - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._clear_old_push_actions_staging, 30 * 60 * 1000 - )) + ) self.db_pool.updates.register_background_index_update( "event_push_summary_unique_index2", diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index b4cc57a28cc..7927842d0f0 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -277,10 +277,10 @@ def __init__( if hs.config.worker.run_background_tasks: # We periodically clean out old transaction ID mappings - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._cleanup_old_transaction_ids, 5 * 60 * 1000, - )) + ) self._get_event_cache: AsyncLruCache[Tuple[str], EventCacheEntry] = ( AsyncLruCache( diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index 42cd7eeeef8..d1ee7cb3b02 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -95,13 +95,13 @@ def __init__( # lead to a race, as we may drop the lock while we are still processing. # However, a) it should be a small window, b) the lock is best effort # anyway and c) we want to really avoid leaking locks when we restart. - hs.register_shutdown_handler("LockStore _on_shutdown", self._on_shutdown) + hs.register_async_shutdown_handler("LockStore _on_shutdown", self._on_shutdown) self._acquiring_locks: Set[Tuple[str, str]] = set() - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._reap_stale_read_write_locks, _LOCK_TIMEOUT_MS / 10.0 - )) + ) @wrap_as_background_process("LockStore._on_shutdown") async def _on_shutdown(self) -> None: diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index 73be6dc573d..163c56e9e2c 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -76,7 +76,7 @@ def __init__( # Read the extrems every 60 minutes if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000)) + self._clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000) # Used in _generate_user_daily_visits to keep track of progress self._last_user_visit_update = self._get_start_of_day() diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 8140832aee4..320d29e4748 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -226,9 +226,9 @@ def __init__( # Create a background job for culling expired 3PID validity tokens if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self.cull_expired_threepid_validation_tokens, THIRTY_MINUTES_IN_MS - )) + ) async def register_user( self, @@ -2727,9 +2727,9 @@ def __init__( # Create a background job for removing expired login tokens if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._delete_expired_login_tokens, THIRTY_MINUTES_IN_MS - )) + ) async def add_access_token_to_user( self, diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 2c4f1379d2d..b7fbc43d025 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -111,10 +111,10 @@ def __init__( ): self._known_servers_count = 1 self._count_known_servers_task: Optional[LoopingCall] = ( - hs.register_looping_call(self.hs.get_clock().looping_call( + self.hs.get_clock().looping_call( self._count_known_servers, 6 * 1000, - )) + ) ) self.hs.get_clock().call_later( 1, diff --git a/synapse/storage/databases/main/session.py b/synapse/storage/databases/main/session.py index 419b58c8f67..8a1331d4c8e 100644 --- a/synapse/storage/databases/main/session.py +++ b/synapse/storage/databases/main/session.py @@ -55,7 +55,7 @@ def __init__( # Create a background job for culling expired sessions. if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000)) + self._clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000) async def create_session( self, session_type: str, value: JsonDict, expiry_ms: int diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index fd9e9001bce..bfc324b80d2 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -81,7 +81,7 @@ def __init__( super().__init__(database, db_conn, hs) if hs.config.worker.run_background_tasks: - hs.register_looping_call(self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000)) + self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000) @wrap_as_background_process("cleanup_transactions") async def _cleanup_transactions(self) -> None: diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index bd4d20accbd..d7937e31f94 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -29,6 +29,7 @@ Dict, Generator, Iterator, + List, Mapping, Optional, Sequence, @@ -110,7 +111,6 @@ def unwrapFirstError(failure: Failure) -> Failure: P = ParamSpec("P") -@attr.s(slots=True) class Clock: """ A Clock wraps a Twisted reactor and provides utilities on top of it. @@ -119,7 +119,14 @@ class Clock: reactor: The Twisted reactor to use. """ - _reactor: IReactorTime = attr.ib() + _reactor: IReactorTime + _call_id: int = 0 + + _looping_calls: List[LoopingCall] = [] + _delayed_calls: Dict[int, IDelayedCall] = {} + + def __init__(self, reactor: IReactorTime) -> None: + self._reactor = reactor @defer.inlineCallbacks def sleep(self, seconds: float) -> "Generator[Deferred[float], Any, Any]": @@ -200,8 +207,14 @@ def _looping_call_common( call.clock = self._reactor d = call.start(msec / 1000.0, now=now) d.addErrback(log_failure, "Looping call died", consumeErrors=False) + self._looping_calls.append(call) return call + def cancel_all_looping_calls(self) -> None: + for call in self._looping_calls: + call.stop() + self._looping_calls.clear() + def call_later( self, delay: float, callback: Callable, *args: Any, **kwargs: Any ) -> IDelayedCall: @@ -217,12 +230,17 @@ def call_later( **kwargs: Key arguments to pass to function. """ + id = self._call_id + self._call_id += 1 def wrapped_callback(*args: Any, **kwargs: Any) -> None: with context.PreserveLoggingContext(): callback(*args, **kwargs) + self._delayed_calls.pop(id) with context.PreserveLoggingContext(): - return self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) + call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) + self._delayed_calls[id] = call + return call def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: try: @@ -231,6 +249,15 @@ def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> N if not ignore_errs: raise + def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: + for call in self._delayed_calls.values(): + try: + call.cancel() + except Exception: + if not ignore_errs: + raise + self._delayed_calls.clear() + def log_failure( failure: Failure, msg: str, consumeErrors: bool = True diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 168ddc51cd5..9f49c5f8a97 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -34,6 +34,7 @@ ) import attr +from synapse.util import Clock from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index c29781cd3d4..a3ac4f5626f 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -105,7 +105,7 @@ def __init__( def f() -> "defer.Deferred[None]": return run_as_background_process("prune_cache", self._prune_cache) - hs.register_looping_call(self._clock.looping_call(f, self._expiry_ms / 2)) + self._clock.looping_call(f, self._expiry_ms / 2) def __setitem__(self, key: KT, value: VT) -> None: now = self._clock.time_msec() diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 1ba1c5e650a..466362e79ce 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -235,13 +235,13 @@ def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: USE_GLOBAL_LIST = True clock = hs.get_clock() - hs.register_looping_call(clock.looping_call( + clock.looping_call( _expire_old_entries, 30 * 1000, clock, expiry_time, hs.config.caches.cache_autotuning, - )) + ) class _Node(Generic[KT, VT]): diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 07294836e80..fcd703bea58 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -118,14 +118,14 @@ def __init__(self, hs: "HomeServer"): self._launching_new_tasks = False if self._run_background_tasks: - hs.register_looping_call(self._clock.looping_call( + self._clock.looping_call( self._launch_scheduled_tasks, TaskScheduler.SCHEDULE_INTERVAL_MS, - )) - hs.register_looping_call(self._clock.looping_call( + ) + self._clock.looping_call( self._clean_scheduled_tasks, TaskScheduler.SCHEDULE_INTERVAL_MS, - )) + ) hs.register_later_gauge(LaterGauge( "synapse_scheduler_running_tasks", @@ -435,7 +435,6 @@ async def wrapper() -> None: log_context, start_time, ) - self._hs.register_looping_call(occasional_status_call) try: (status, result, error) = await function(task) except Exception: diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index a5bf7e06357..7d9b2a3bce0 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -158,11 +158,13 @@ def test_single_service_up_txn_not_sent(self) -> None: class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): def setUp(self) -> None: self.clock = MockClock() + self.hs = Mock() self.as_api = Mock() self.store = Mock() self.service = Mock() self.callback = AsyncMock() self.recoverer = _Recoverer( + hs=self.hs, clock=cast(Clock, self.clock), as_api=self.as_api, store=self.store, From 144bff02dcd319b62d2177e98cfc82244628fa97 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 15 Aug 2025 16:21:19 -0600 Subject: [PATCH 007/181] Remove unused code --- synapse/server.py | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index f078241470c..d3d470d2f12 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -331,30 +331,7 @@ def __init__( def shutdown(self) -> None: logger.info("Received shutdown request") - logger.info("Unsubscribing replication callbacks") - self.get_replication_notifier()._replication_callbacks.clear() - - logger.info("Stopping replication") - for connection in self.get_replication_command_handler()._connections: - connection.close() - self.get_replication_command_handler()._connections.clear() - # NOTE: Dont trample on other tenants - # for connection in connected_connections: - # logger.info("Closing replication connection...") - # connection.close() - # connected_connections.clear() - - logger.info("Stopping replication streams") - # self.get_replication_command_handler()._factory.continueTrying = False - # self.get_replication_command_handler()._factory.stopFactory() - # self.get_replication_command_handler()._factory.stopTrying() - # self.get_replication_command_handler()._connection.disconnect() - # self.get_replication_command_handler()._factory.synapse_outbound_redis_connection.disconnect() - # self.get_replication_command_handler()._factory = None - # self.get_replication_command_handler()._streams.clear() - # self.get_replication_command_handler()._streams_to_replicate.clear() - # # XXX: Does this trample on other tenants? - # STREAMS_MAP.clear() + # TODO: Cleanup replication pieces from synapse.util.batching_queue import number_of_keys number_of_keys.clear() From 94a1bbb0e40acf83613f910c6788ad1191dfd3e3 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 15 Aug 2025 17:51:46 -0600 Subject: [PATCH 008/181] Cleanup unnecessary teardown attempts --- synapse/api/auth_blocking.py | 4 +- synapse/app/_base.py | 1 - synapse/app/homeserver.py | 9 ---- synapse/app/phone_stats_home.py | 2 +- synapse/appservice/scheduler.py | 2 +- synapse/crypto/keyring.py | 4 +- synapse/federation/federation_base.py | 1 + synapse/federation/federation_client.py | 6 +-- synapse/federation/federation_server.py | 2 +- synapse/federation/send_queue.py | 5 +- synapse/federation/sender/__init__.py | 16 +++--- synapse/federation/transport/client.py | 4 +- synapse/federation/transport/server/_base.py | 4 +- synapse/handlers/account.py | 4 +- synapse/handlers/appservice.py | 4 +- synapse/handlers/device.py | 1 - synapse/handlers/devicemessage.py | 6 +-- synapse/handlers/e2e_keys.py | 12 ++--- synapse/handlers/event_auth.py | 4 +- synapse/handlers/federation.py | 8 +-- synapse/handlers/federation_event.py | 8 +-- synapse/handlers/message.py | 2 - synapse/handlers/presence.py | 19 +++---- synapse/handlers/profile.py | 4 +- synapse/handlers/sliding_sync/__init__.py | 1 + synapse/handlers/sliding_sync/room_lists.py | 10 ++-- synapse/handlers/sso.py | 6 +-- synapse/handlers/stats.py | 3 +- synapse/handlers/sync.py | 1 - synapse/handlers/typing.py | 31 +++++++----- synapse/handlers/user_directory.py | 13 ++--- synapse/media/media_repository.py | 2 +- synapse/media/url_previewer.py | 1 - synapse/replication/tcp/client.py | 3 +- synapse/replication/tcp/protocol.py | 3 -- synapse/replication/tcp/redis.py | 3 -- synapse/rest/admin/devices.py | 18 +++---- synapse/rest/admin/experimental_features.py | 6 +-- synapse/rest/admin/media.py | 10 ++-- synapse/rest/admin/rooms.py | 10 ++-- synapse/rest/admin/server_notice_servlet.py | 4 +- synapse/rest/admin/users.py | 50 +++++++++---------- synapse/rest/client/media.py | 8 +-- synapse/rest/client/thread_subscriptions.py | 1 + synapse/rest/media/download_resource.py | 4 +- synapse/rest/media/thumbnail_resource.py | 4 +- synapse/server.py | 8 +-- .../server_notices/server_notices_manager.py | 4 +- synapse/state/__init__.py | 1 - synapse/storage/controllers/persist_events.py | 9 ++-- synapse/storage/controllers/state.py | 8 +-- synapse/storage/databases/main/deviceinbox.py | 1 - .../databases/main/event_push_actions.py | 1 + synapse/storage/databases/main/events.py | 15 +++--- synapse/storage/databases/main/metrics.py | 2 - synapse/storage/databases/main/roommember.py | 1 - synapse/util/caches/expiringcache.py | 1 - tests/util/test_expiring_cache.py | 4 -- 58 files changed, 185 insertions(+), 194 deletions(-) diff --git a/synapse/api/auth_blocking.py b/synapse/api/auth_blocking.py index 19a46028997..303c9ba03e3 100644 --- a/synapse/api/auth_blocking.py +++ b/synapse/api/auth_blocking.py @@ -46,7 +46,7 @@ def __init__(self, hs: "HomeServer"): self._mau_limits_reserved_threepids = ( hs.config.server.mau_limits_reserved_threepids ) - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips async def check_auth_blocking( @@ -84,7 +84,7 @@ async def check_auth_blocking( if requester: if requester.authenticated_entity.startswith("@"): user_id = requester.authenticated_entity - elif self.hs._is_mine_server_name(requester.authenticated_entity): + elif self._is_mine_server_name(requester.authenticated_entity): # We never block the server from doing actions on behalf of # users. return diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 77a4972634b..abbf322c04a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -40,7 +40,6 @@ Tuple, cast, ) -import weakref from cryptography.utils import CryptographyDeprecationWarning from typing_extensions import ParamSpec diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index bd35fcccfa3..02b05b46011 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -90,11 +90,6 @@ def shutdown(self) -> None: logger.info("Shutting down listening services") for listener in self._listening_services: logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) - #listener.socket.shutdown(0) - #listener.stopListening() - #listener.stopConsuming() - #listener.loseConnection() - # Preferred over connectionLost since it allows buffers to flush listener.unregisterProducer() listener.loseConnection() @@ -103,12 +98,8 @@ def shutdown(self) -> None: # Sometimes takes a second for some deferred to fire that cancels the socket # But seems to always do so within a minute # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. - - #listener.connectionLost(None) self._listening_services.clear() - #self._reactor.stop() - def _listener_http( self, config: HomeServerConfig, diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index da105519265..c118a6f4505 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -225,7 +225,7 @@ def performance_stats_init() -> None: # table will decrease clock.looping_call( hs.get_datastores().main.generate_user_daily_visits, - 5 * MILLISECONDS_PER_SECOND, + 5 * ONE_MINUTE_SECONDS * MILLISECONDS_PER_SECOND, ) # monthly active user limiting functionality diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index d1e1822b382..638ef3789aa 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -539,7 +539,7 @@ def force_retry(self) -> None: if self.scheduled_recovery: self.clock.cancel_call_later(self.scheduled_recovery) # Run a retry, which will resechedule a recovery if it fails. - deferred = run_as_background_process( + run_as_background_process( "retry", self.server_name, self.retry, diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 94fd5a688c6..8c59772e56c 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -178,7 +178,7 @@ def __init__( process_batch_callback=self._inner_fetch_key_requests, ) - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name # build a FetchKeyResult for each of our own keys, to shortcircuit the # fetcher. @@ -282,7 +282,7 @@ async def process_request(self, verify_request: VerifyJsonRequest) -> None: # If we are the originating server, short-circuit the key-fetch for any keys # we already have - if self.hs._is_mine_server_name(verify_request.server_name): + if self._is_mine_server_name(verify_request.server_name): for key_id in verify_request.key_ids: if key_id in self._local_verify_keys: found_keys[key_id] = self._local_verify_keys[key_id] diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 3e664c2d4fe..a1c9c286ac7 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -58,6 +58,7 @@ class FederationBase: def __init__(self, hs: "HomeServer"): self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name self.keyring = hs.get_keyring() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self.store = hs.get_datastores().main diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 1d6e56fe789..542d9650d47 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -146,7 +146,6 @@ def __init__(self, hs: "HomeServer"): # Cache mapping `event_id` to a tuple of the event itself and the `pull_origin` # (which server we pulled the event from) self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache( - hs=hs, cache_name="get_pdu_cache", server_name=self.server_name, clock=self._clock, @@ -166,7 +165,6 @@ def __init__(self, hs: "HomeServer"): Tuple[str, bool], Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]], ] = ExpiringCache( - hs=hs, cache_name="get_room_hierarchy_cache", server_name=self.server_name, clock=self._clock, @@ -943,7 +941,7 @@ async def _try_destination_list( for destination in destinations: # We don't want to ask our own server for information we don't have - if self.hs._is_mine_server_name(destination): + if self._is_mine_server_name(destination): continue try: @@ -1623,7 +1621,7 @@ async def forward_third_party_invite( self, destinations: Iterable[str], room_id: str, event_dict: JsonDict ) -> None: for destination in destinations: - if self.hs._is_mine_server_name(destination): + if self._is_mine_server_name(destination): continue try: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 099b43cba68..ff51b613ce9 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -995,7 +995,7 @@ async def _on_send_membership_event( authorising_server = get_domain_from_id( event.content[EventContentFields.AUTHORISING_USER] ) - if not self.hs._is_mine_server_name(authorising_server): + if not self._is_mine_server_name(authorising_server): raise SynapseError( 400, f"Cannot authorise membership event for {authorising_server}. We can only authorise requests from our own homeserver", diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 8a5e05c1fa9..745e6fa18a3 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -74,7 +74,8 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.notifier = hs.get_notifier() - self.hs = hs + self.is_mine_id = hs.is_mine_id + self.is_mine_server_name = hs.is_mine_server_name # We may have multiple federation sender instances, so we need to track # their positions separately. @@ -207,7 +208,7 @@ def build_and_send_edu( key: Optional[Hashable] = None, ) -> None: """As per FederationSender""" - if self.hs.is_mine_server_name(destination): + if self.is_mine_server_name(destination): logger.info("Not sending EDU to ourselves") return diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index a38b48d603a..dd7c36b7925 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -386,6 +386,8 @@ def __init__(self, hs: "HomeServer"): self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() + self.is_mine_id = hs.is_mine_id + self.is_mine_server_name = hs.is_mine_server_name self._presence_router: Optional["PresenceRouter"] = None self._transaction_manager = TransactionManager(hs) @@ -527,7 +529,7 @@ async def _process_event_queue_loop(self) -> None: async def handle_event(event: EventBase) -> None: # Only send events for this server. send_on_behalf_of = event.internal_metadata.get_send_on_behalf_of() - is_mine = self.hs.is_mine_id(event.sender) + is_mine = self.is_mine_id(event.sender) if not is_mine and send_on_behalf_of is None: logger.debug("Not sending remote-origin event %s", event) return @@ -852,7 +854,7 @@ async def send_read_receipt(self, receipt: ReadReceipt) -> None: domains: StrCollection = [ d for d in domains_set - if not self.hs.is_mine_server_name(d) + if not self.is_mine_server_name(d) and self._federation_shard_config.should_handle(self._instance_name, d) ] @@ -929,7 +931,7 @@ async def send_presence_to_destinations( # Ensure we only send out presence states for local users. for state in states: - assert self.hs.is_mine_id(state.user_id) + assert self.is_mine_id(state.user_id) destinations = await filter_destinations_by_retry_limiter( [ @@ -943,7 +945,7 @@ async def send_presence_to_destinations( ) for destination in destinations: - if self.hs.is_mine_server_name(destination): + if self.is_mine_server_name(destination): continue queue = self._get_per_destination_queue(destination) @@ -968,7 +970,7 @@ def build_and_send_edu( content: content of EDU key: clobbering key for this edu """ - if self.hs.is_mine_server_name(destination): + if self.is_mine_server_name(destination): logger.info("Not sending EDU to ourselves") return @@ -1016,7 +1018,7 @@ async def send_device_messages( if self._federation_shard_config.should_handle( self._instance_name, destination ) - and not self.hs.is_mine_server_name(destination) + and not self.is_mine_server_name(destination) ], clock=self.clock, store=self.store, @@ -1043,7 +1045,7 @@ def wake_destination(self, destination: str) -> None: might have come back. """ - if self.hs.is_mine_server_name(destination): + if self.is_mine_server_name(destination): logger.warning("Not waking up ourselves") return diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 12bc2ed8d81..62bf96ce913 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -68,7 +68,7 @@ class TransportLayerClient: def __init__(self, hs: "HomeServer"): self.client = hs.get_federation_http_client() - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name async def get_room_state_ids( self, destination: str, room_id: str, event_id: str @@ -270,7 +270,7 @@ async def send_transaction( transaction.transaction_id, ) - if self.hs._is_mine_server_name(transaction.destination): + if self._is_mine_server_name(transaction.destination): raise RuntimeError("Transport layer cannot send to itself!") # FIXME: This is only used by the tests. The actual json sent is diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index c215e346246..cba309635b7 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -64,7 +64,7 @@ def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self.keyring = hs.get_keyring() self.server_name = hs.hostname - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name self.store = hs.get_datastores().main self.federation_domain_whitelist = ( hs.config.federation.federation_domain_whitelist @@ -108,7 +108,7 @@ async def authenticate_request( json_request["signatures"].setdefault(origin, {})[key] = sig # if the origin_server sent a destination along it needs to match our own server_name - if destination is not None and not self.hs._is_mine_server_name( + if destination is not None and not self._is_mine_server_name( destination ): raise AuthenticationError( diff --git a/synapse/handlers/account.py b/synapse/handlers/account.py index 58d2988c3f5..37cc3d3ff56 100644 --- a/synapse/handlers/account.py +++ b/synapse/handlers/account.py @@ -31,7 +31,7 @@ class AccountHandler: def __init__(self, hs: "HomeServer"): self._main_store = hs.get_datastores().main - self.hs = hs + self._is_mine = hs.is_mine self._federation_client = hs.get_federation_client() self._use_account_validity_in_account_status = ( hs.config.server.use_account_validity_in_account_status @@ -75,7 +75,7 @@ async def get_account_statuses( Codes.INVALID_PARAM, ) - if self.hs._is_mine(user_id): + if self._is_mine(user_id): status = await self._get_local_account_status(user_id) statuses[user_id.to_string()] = status else: diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f7809fd011a..bf36cf39a19 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -80,7 +80,7 @@ def __init__(self, hs: "HomeServer"): hs.hostname ) # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main - self.hs = hs + self.is_mine_id = hs.is_mine_id self.appservice_api = hs.get_application_service_api() self.scheduler = hs.get_application_service_scheduler() self.started_scheduler = False @@ -853,7 +853,7 @@ def _get_services_for_3pn(self, protocol: str) -> List[ApplicationService]: return [s for s in services if s.is_interested_in_protocol(protocol)] async def _is_unknown_user(self, user_id: str) -> bool: - if not self.hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): # we don't know if they are unknown or not since it isn't one of our # users. We can't poke ASes. return False diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 448d81bbdd2..d957b04d952 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1462,7 +1462,6 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): # but they're useful to have them about to reduce the number of spurious # resyncs. self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache( - hs=hs, cache_name="device_update_edu", server_name=self.server_name, clock=self.clock, diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index 06584a32409..e3c8ed827b4 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -52,7 +52,7 @@ def __init__(self, hs: "HomeServer"): """ self.store = hs.get_datastores().main self.notifier = hs.get_notifier() - self.hs = hs + self.is_mine = hs.is_mine self.device_handler = hs.get_device_handler() if hs.config.experimental.msc3814_enabled: self.event_sources = hs.get_event_sources() @@ -109,7 +109,7 @@ async def on_direct_to_device_edu(self, origin: str, content: JsonDict) -> None: message_id = content["message_id"] for user_id, by_device in content["messages"].items(): # we use UserID.from_string to catch invalid user ids - if not self.hs.is_mine(UserID.from_string(user_id)): + if not self.is_mine(UserID.from_string(user_id)): logger.warning("To-device message to non-local user %s", user_id) raise SynapseError(400, "Not a user here") @@ -263,7 +263,7 @@ async def send_device_message( continue # we use UserID.from_string to catch invalid user ids - if self.hs.is_mine(UserID.from_string(user_id)): + if self.is_mine(UserID.from_string(user_id)): messages_by_device = { device_id: { "content": message_content, diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 96f53bc40ed..e9740ea7560 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -68,7 +68,7 @@ def __init__(self, hs: "HomeServer"): self.federation = hs.get_federation_client() self.device_handler = hs.get_device_handler() self._appservice_handler = hs.get_application_service_handler() - self.hs = hs + self.is_mine = hs.is_mine self.clock = hs.get_clock() self._worker_lock_handler = hs.get_worker_locks_handler() self._task_scheduler = hs.get_task_scheduler() @@ -202,7 +202,7 @@ async def filter_device_key_query( for user_id, device_ids in device_keys_query.items(): # we use UserID.from_string to catch invalid user ids - if self.hs.is_mine(UserID.from_string(user_id)): + if self.is_mine(UserID.from_string(user_id)): local_query[user_id] = device_ids else: remote_queries[user_id] = device_ids @@ -550,7 +550,7 @@ async def query_local_devices( result_dict: Dict[str, Dict[str, dict]] = {} for user_id, device_ids in query.items(): # we use UserID.from_string to catch invalid user ids - if not self.hs.is_mine(UserID.from_string(user_id)): + if not self.is_mine(UserID.from_string(user_id)): logger.warning("Request for keys for non-local user %s", user_id) log_kv( { @@ -621,7 +621,7 @@ async def on_federation_query_client_keys( "device_keys", {} ) if any( - not self.hs.is_mine(UserID.from_string(user_id)) + not self.is_mine(UserID.from_string(user_id)) for user_id in device_keys_query ): raise SynapseError(400, "User is not hosted on this homeserver") @@ -762,7 +762,7 @@ async def claim_one_time_keys( for user_id, one_time_keys in query.items(): # we use UserID.from_string to catch invalid user ids - if self.hs.is_mine(UserID.from_string(user_id)): + if self.is_mine(UserID.from_string(user_id)): for device_id, algorithms in one_time_keys.items(): for algorithm, count in algorithms.items(): local_query.append((user_id, device_id, algorithm, count)) @@ -1432,7 +1432,7 @@ async def _get_e2e_cross_signing_verify_key( # cross-sign a remote user, but does not share any rooms with them yet. # Thus, we would not have their key list yet. We instead fetch the key, # store it and notify clients of new, associated device IDs. - if self.hs.is_mine(user) or key_type not in ["master", "self_signing"]: + if self.is_mine(user) or key_type not in ["master", "self_signing"]: # Note that master and self_signing keys are the only cross-signing keys we # can request over federation raise NotFoundError("No %s key found for %s" % (key_type, user_id)) diff --git a/synapse/handlers/event_auth.py b/synapse/handlers/event_auth.py index 72ec866a856..1f1f67dc0df 100644 --- a/synapse/handlers/event_auth.py +++ b/synapse/handlers/event_auth.py @@ -56,7 +56,7 @@ def __init__(self, hs: "HomeServer"): self._store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state self._server_name = hs.hostname - self.hs = hs + self._is_mine_id = hs.is_mine_id async def check_auth_rules_from_context( self, @@ -272,7 +272,7 @@ async def check_restricted_join_rules( if not await self.is_user_in_rooms(allowed_rooms, user_id): # If this is a remote request, the user might be in an allowed room # that we do not know about. - if not self.hs._is_mine_id(user_id): + if not self._is_mine_id(user_id): for room_id in allowed_rooms: if not await self._store.is_host_joined(room_id, self._server_name): raise SynapseError( diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 4346a4a2c44..6f9d6a18545 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -144,6 +144,8 @@ def __init__(self, hs: "HomeServer"): self.state_handler = hs.get_state_handler() self.server_name = hs.hostname self.keyring = hs.get_keyring() + self.is_mine_id = hs.is_mine_id + self.is_mine_server_name = hs.is_mine_server_name self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker self.event_creation_handler = hs.get_event_creation_handler() self.event_builder_factory = hs.get_event_builder_factory() @@ -462,7 +464,7 @@ async def try_backfill(domains: StrCollection) -> bool: for dom in domains: # We don't want to ask our own server for information we don't have - if self.hs.is_mine_server_name(dom): + if self.is_mine_server_name(dom): continue try: @@ -782,7 +784,7 @@ async def do_invite_join( ) # We retrieve the room member handler here as to not cause a cyclic dependency - member_handler = self.hs.get_room_member_handler() + member_handler = self.get_room_member_handler() await member_handler.transfer_room_state_on_room_upgrade( old_room_id, room_id ) @@ -1080,7 +1082,7 @@ async def on_invite_request( 400, "The invite event was not from the server sending it" ) - if not self.hs.is_mine_id(event.state_key): + if not self.is_mine_id(event.state_key): raise SynapseError(400, "The invite event must be for this server") # block any attempts to invite the server notices mxid diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index f4e2052e92c..540f8438700 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -162,6 +162,8 @@ def __init__(self, hs: "HomeServer"): self._message_handler = hs.get_message_handler() self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() self._state_resolution_handler = hs.get_state_resolution_handler() + # avoid a circular dependency by deferring execution here + self._get_room_member_handler = hs.get_room_member_handler self._federation_client = hs.get_federation_client() self._third_party_event_rules = ( @@ -682,7 +684,7 @@ async def backfill( server from invalid events (there is probably no point in trying to re-fetch invalid events from every other HS in the room.) """ - if self.hs._is_mine_server_name(dest): + if self._is_mine_server_name(dest): raise SynapseError(400, "Can't backfill from self.") events = await self._federation_client.backfill( @@ -1948,7 +1950,7 @@ async def _maybe_kick_guest_users(self, event: EventBase) -> None: event.room_id ) current_state_list = list(current_state.values()) - await self.hs._get_room_member_handler().kick_guest_users(current_state_list) + await self._get_room_member_handler().kick_guest_users(current_state_list) async def _check_for_soft_fail( self, @@ -2324,7 +2326,7 @@ async def _notify_persisted_event( # users if event.internal_metadata.is_outlier(): if event.membership != Membership.INVITE: - if not self.hs._is_mine_id(target_user_id): + if not self._is_mine_id(target_user_id): return target_user = UserID.from_string(target_user_id) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index b9d11b78d34..2eddcdf2041 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -96,7 +96,6 @@ class MessageHandler: """Contains some read only APIs to get state about a room""" def __init__(self, hs: "HomeServer"): - self.hs = hs self.server_name = hs.hostname self.auth = hs.get_auth() self.clock = hs.get_clock() @@ -565,7 +564,6 @@ def __init__(self, hs: "HomeServer"): self._external_cache_joined_hosts_updates: Optional[ExpiringCache] = None if self._external_cache.is_enabled(): self._external_cache_joined_hosts_updates = ExpiringCache( - hs=hs, cache_name="_external_cache_joined_hosts_updates", server_name=self.server_name, clock=self.clock, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index b2c7460aa00..451bf4f091a 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -214,6 +214,7 @@ def __init__(self, hs: "HomeServer"): self._storage_controllers = hs.get_storage_controllers() self.presence_router = hs.get_presence_router() self.state = hs.get_state_handler() + self.is_mine_id = hs.is_mine_id self._presence_enabled = hs.config.server.presence_enabled self._track_presence = hs.config.server.track_presence @@ -415,7 +416,7 @@ async def maybe_send_presence_to_interested_destinations( if not self._federation: return - states = [s for s in states if self.hs.is_mine_id(s.user_id)] + states = [s for s in states if self.is_mine_id(s.user_id)] if not states: return @@ -808,7 +809,7 @@ def __init__(self, hs: "HomeServer"): obj=state.user_id, then=state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT, ) - if self.hs.is_mine_id(state.user_id): + if self.is_mine_id(state.user_id): self.wheel_timer.insert( now=now, obj=state.user_id, @@ -1095,7 +1096,7 @@ async def _handle_timeouts(self) -> None: changes = handle_timeouts( states, - is_mine_fn=self.hs.is_mine_id, + is_mine_fn=self.is_mine_id, syncing_user_devices=syncing_user_devices, user_to_devices=self._user_to_device_to_current_state, now=now, @@ -1625,7 +1626,7 @@ async def _handle_state_delta(self, room_id: str, deltas: List[StateDelta]) -> N prev_local_users = [] prev_remote_hosts = set() for user_id in prev_users: - if self.hs.is_mine_id(user_id): + if self.is_mine_id(user_id): prev_local_users.append(user_id) else: prev_remote_hosts.add(get_domain_from_id(user_id)) @@ -1637,7 +1638,7 @@ async def _handle_state_delta(self, room_id: str, deltas: List[StateDelta]) -> N newly_joined_local_users = [] newly_joined_remote_hosts = set() for user_id in newly_joined_users: - if self.hs.is_mine_id(user_id): + if self.is_mine_id(user_id): newly_joined_local_users.append(user_id) else: host = get_domain_from_id(user_id) @@ -1888,7 +1889,7 @@ async def get_new_events( # Figure out which other users this user should explicitly receive # updates for additional_users_interested_in = ( - await self.hs.get_presence_router().get_interested_users( + await self.get_presence_router().get_interested_users( user.to_string() ) ) @@ -1971,7 +1972,7 @@ async def get_new_events( # Retrieve the current presence state for each user users_to_state = ( - await self.hs.get_presence_handler().current_state_for_users( + await self.get_presence_handler().current_state_for_users( interested_and_updated_users ) ) @@ -2014,7 +2015,7 @@ async def _filter_all_presence_updates_for_user( if updated_users is not None: # Get the actual presence update for each change users_to_state = ( - await self.hs.get_presence_handler().current_state_for_users( + await self.get_presence_handler().current_state_for_users( updated_users ) ) @@ -2035,7 +2036,7 @@ async def _filter_all_presence_updates_for_user( # for a single user # Filter through the presence router - users_to_state_set = await self.hs.get_presence_router().get_users_for_states( + users_to_state_set = await self.get_presence_router().get_users_for_states( presence_updates ) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index d0a2143a300..dbff28e7fb5 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -73,6 +73,8 @@ def __init__(self, hs: "HomeServer"): hs.config.server.allowed_avatar_mimetypes ) + self._is_mine_server_name = hs.is_mine_server_name + self._third_party_rules = hs.get_module_api_callbacks().third_party_event_rules async def get_profile(self, user_id: str, ignore_backoff: bool = True) -> JsonDict: @@ -364,7 +366,7 @@ async def check_avatar_size_and_mime_type(self, mxc: str) -> bool: else: server_name = host - if self.hs._is_mine_server_name(server_name): + if self._is_mine_server_name(server_name): media_info: Optional[ Union[LocalMedia, RemoteMedia] ] = await self.store.get_local_media(media_id) diff --git a/synapse/handlers/sliding_sync/__init__.py b/synapse/handlers/sliding_sync/__init__.py index de0bcedf0eb..a9573ba0f15 100644 --- a/synapse/handlers/sliding_sync/__init__.py +++ b/synapse/handlers/sliding_sync/__init__.py @@ -104,6 +104,7 @@ def __init__(self, hs: "HomeServer"): self.event_sources = hs.get_event_sources() self.relations_handler = hs.get_relations_handler() self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync + self.is_mine_id = hs.is_mine_id self.connection_store = SlidingSyncConnectionStore(self.store) self.extensions = SlidingSyncExtensionHandler(hs) diff --git a/synapse/handlers/sliding_sync/room_lists.py b/synapse/handlers/sliding_sync/room_lists.py index 200111a4015..e196199f8ad 100644 --- a/synapse/handlers/sliding_sync/room_lists.py +++ b/synapse/handlers/sliding_sync/room_lists.py @@ -187,7 +187,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.storage_controllers = hs.get_storage_controllers() self.rooms_to_exclude_globally = hs.config.server.rooms_to_exclude_from_sync - self.hs = hs + self.is_mine_id = hs.is_mine_id async def compute_interested_rooms( self, @@ -469,7 +469,7 @@ async def _compute_interested_rooms_new_tables( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.hs.is_mine_id): + if room_sync_config.must_await_full_state(self.is_mine_id): filtered_sync_room_map = { room_id: room for room_id, room in filtered_sync_room_map.items() @@ -592,7 +592,7 @@ async def _compute_interested_rooms_new_tables( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.hs.is_mine_id): + if room_sync_config.must_await_full_state(self.is_mine_id): if room_id in partial_state_rooms: continue @@ -683,7 +683,7 @@ async def _compute_interested_rooms_fallback( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.hs.is_mine_id): + if room_sync_config.must_await_full_state(self.is_mine_id): filtered_sync_room_map = { room_id: room for room_id, room in filtered_sync_room_map.items() @@ -778,7 +778,7 @@ async def _compute_interested_rooms_fallback( # Exclude partially-stated rooms if we must wait for the room to be # fully-stated - if room_sync_config.must_await_full_state(self.hs.is_mine_id): + if room_sync_config.must_await_full_state(self.is_mine_id): if room_id in partial_state_rooms: continue diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 70ab166bad7..eec420cbb17 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -202,9 +202,7 @@ class SsoHandler: def __init__(self, hs: "HomeServer"): self._clock = hs.get_clock() self._store = hs.get_datastores().main - self._server_name = hs.hostname - self.hs = hs - self._server_name = hs.hostname + self.server_name = hs.hostname self._is_mine_server_name = hs.is_mine_server_name self._registration_handler = hs.get_registration_handler() self._auth_handler = hs.get_auth_handler() @@ -820,7 +818,7 @@ def is_allowed_mime_type(content_type: str) -> bool: avatar_url_parts = profile[ProfileFields.AVATAR_URL].split("/") server_name = avatar_url_parts[-2] media_id = avatar_url_parts[-1] - if self.hs._is_mine_server_name(server_name): + if self._is_mine_server_name(server_name): media = await self._media_repo.store.get_local_media(media_id) if media is not None and upload_name == media.upload_name: logger.info("skipping saving the user avatar") diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 51001c3eb92..a2602ea818e 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -60,6 +60,7 @@ def __init__(self, hs: "HomeServer"): self.state = hs.get_state_handler() self.clock = hs.get_clock() self.notifier = hs.get_notifier() + self.is_mine_id = hs.is_mine_id self.stats_enabled = hs.config.stats.stats_enabled @@ -267,7 +268,7 @@ async def _handle_deltas( raise ValueError("%r is not a valid membership" % (membership,)) user_id = delta.state_key - if self.hs.is_mine_id(user_id): + if self.is_mine_id(user_id): # this accounts for transitions like leave → ban and so on. has_changed_joinedness = (prev_membership == Membership.JOIN) != ( membership == Membership.JOIN diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index a98dddc8cb8..94365d0ed82 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -364,7 +364,6 @@ def __init__(self, hs: "HomeServer"): self.lazy_loaded_members_cache: ExpiringCache[ Tuple[str, Optional[str]], LruCache[str, str] ] = ExpiringCache( - hs=hs, cache_name="lazy_loaded_members_cache", server_name=self.server_name, clock=self.clock, diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 2f82a2b4133..6a7b36ea0c8 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -84,7 +84,8 @@ def __init__(self, hs: "HomeServer"): hs.hostname ) # nb must be called this for @wrap_as_background_process self.clock = hs.get_clock() - self.hs = hs + self.is_mine_id = hs.is_mine_id + self.is_mine_server_name = hs.is_mine_server_name self.federation = None if hs.should_send_federation(): @@ -140,7 +141,7 @@ def _handle_timeout_for_member(self, now: int, member: RoomMember) -> None: # Check if we need to resend a keep alive over federation for this # user. - if self.federation and self.hs.is_mine_id(member.user_id): + if self.federation and self.is_mine_id(member.user_id): last_fed_poke = self._member_last_federation_poke.get(member, None) if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now: run_as_background_process( @@ -181,7 +182,7 @@ async def _push_remote(self, member: RoomMember, typing: bool) -> None: store=self.store, ) for domain in hosts: - if not self.hs.is_mine_server_name(domain): + if not self.is_mine_server_name(domain): logger.debug("sending typing update to %s", domain) self.federation.build_and_send_edu( destination=domain, @@ -239,11 +240,11 @@ async def _send_changes_in_typing_to_remotes( return for user_id in now_typing - prev_typing: - if self.hs.is_mine_id(user_id): + if self.is_mine_id(user_id): await self._push_remote(RoomMember(room_id, user_id), True) for user_id in prev_typing - now_typing: - if self.hs.is_mine_id(user_id): + if self.is_mine_id(user_id): await self._push_remote(RoomMember(room_id, user_id), False) def get_current_token(self) -> int: @@ -310,7 +311,7 @@ async def started_typing( ) -> None: target_user_id = target_user.to_string() - if not self.hs.is_mine_id(target_user_id): + if not self.is_mine_id(target_user_id): raise SynapseError(400, "User is not hosted on this homeserver") if target_user != requester.user: @@ -345,7 +346,7 @@ async def stopped_typing( ) -> None: target_user_id = target_user.to_string() - if not self.hs.is_mine_id(target_user_id): + if not self.is_mine_id(target_user_id): raise SynapseError(400, "User is not hosted on this homeserver") if target_user != requester.user: @@ -366,7 +367,7 @@ async def stopped_typing( def user_left_room(self, user: UserID, room_id: str) -> None: user_id = user.to_string() - if self.hs.is_mine_id(user_id): + if self.is_mine_id(user_id): member = RoomMember(room_id=room_id, user_id=user_id) self._stopped_typing(member) @@ -519,10 +520,14 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self._main_store = hs.get_datastores().main self.clock = hs.get_clock() - self.hs = hs + # We can't call get_typing_handler here because there's a cycle: + # + # Typing -> Notifier -> TypingNotificationEventSource -> Typing + # + self.get_typing_handler = hs.get_typing_handler def _make_event_for(self, room_id: str) -> JsonMapping: - typing = self.hs.get_typing_handler()._room_typing[room_id] + typing = self.get_typing_handler()._room_typing[room_id] return { "type": EduTypes.TYPING, "room_id": room_id, @@ -548,7 +553,7 @@ async def get_new_events_as( with Measure( self.clock, name="typing.get_new_events_as", server_name=self.server_name ): - handler = self.hs.get_typing_handler() + handler = self.get_typing_handler() events = [] @@ -587,7 +592,7 @@ async def get_new_events( self.clock, name="typing.get_new_events", server_name=self.server_name ): from_key = int(from_key) - handler = self.hs.get_typing_handler() + handler = self.get_typing_handler() events = [] for room_id in room_ids: @@ -603,4 +608,4 @@ async def get_new_events( return events, handler._latest_room_serial def get_current_key(self) -> int: - return self.hs.get_typing_handler()._latest_room_serial + return self.get_typing_handler()._latest_room_serial diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 13cc829efaa..130099a2390 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -106,6 +106,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() self.notifier = hs.get_notifier() + self.is_mine_id = hs.is_mine_id self.update_user_directory = hs.config.worker.should_update_user_directory self.search_all_users = hs.config.userdirectory.user_directory_search_all_users self.exclude_remote_users = ( @@ -362,7 +363,7 @@ async def _handle_room_publicity_change( # being added multiple times. The batching upserts shouldn't make this # too bad, though. for user_id in users_in_room: - if not self._hs.is_mine_id( + if not self.is_mine_id( user_id ) or await self.store.should_include_local_user_in_dir(user_id): await self._track_user_joined_room(room_id, user_id) @@ -395,7 +396,7 @@ async def _handle_room_membership_event( ) # Both cases ignore excluded local users, so start by discarding them. - is_remote = not self._hs.is_mine_id(state_key) + is_remote = not self.is_mine_id(state_key) if not is_remote and not await self.store.should_include_local_user_in_dir( state_key ): @@ -458,7 +459,7 @@ async def _track_user_joined_room(self, room_id: str, joining_user_id: str) -> N and ( # We can't apply any special rules to remote users so # they're always included - not self._hs.is_mine_id(other) + not self.is_mine_id(other) # Check the special rules whether the local user should be # included in the user directory or await self.store.should_include_local_user_in_dir(other) @@ -468,7 +469,7 @@ async def _track_user_joined_room(self, room_id: str, joining_user_id: str) -> N # First, if the joining user is our local user then we need an # update for every other user in the room. - if self._hs.is_mine_id(joining_user_id): + if self.is_mine_id(joining_user_id): for other_user_id in other_users_in_room: updates_to_users_who_share_rooms.add( (joining_user_id, other_user_id) @@ -477,7 +478,7 @@ async def _track_user_joined_room(self, room_id: str, joining_user_id: str) -> N # Next, we need an update for every other local user in the room # that they now share a room with the joining user. for other_user_id in other_users_in_room: - if self._hs.is_mine_id(other_user_id): + if self.is_mine_id(other_user_id): updates_to_users_who_share_rooms.add( (other_user_id, joining_user_id) ) @@ -506,7 +507,7 @@ async def _handle_remove_user(self, room_id: str, user_id: str) -> None: # Additionally, if they're a remote user and we're no longer joined # to any rooms they're in, remove them from the user directory. - if not self._hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): rooms_user_is_in = await self.store.get_user_dir_rooms_user_is_in(user_id) if len(rooms_user_is_in) == 0: diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index daee490c561..16bf9d9c4ad 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -144,7 +144,7 @@ def __init__(self, hs: "HomeServer"): storage_providers.append(provider) self.media_storage: MediaStorage = MediaStorage( - hs, self.primary_base_path, self.filepaths, storage_providers + self.hs, self.primary_base_path, self.filepaths, storage_providers ) self.clock.looping_call( diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 2d4de78e35a..60cdf0fed52 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -199,7 +199,6 @@ def __init__( # memory cache mapping urls to an ObservableDeferred returning # JSON-encoded OG metadata self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache( - hs=hs, cache_name="url_previews", server_name=self.server_name, clock=self.clock, diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 16b6efc8189..908b87cba14 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -415,6 +415,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.store = hs.get_datastores().main + self._is_mine_id = hs.is_mine_id self._hs = hs # We need to make a temporary value to ensure that mypy picks up the @@ -470,7 +471,7 @@ async def _on_new_receipts( """ for receipt in rows: # we only want to send on receipts for our own users - if not self._hs._is_mine_id(receipt.user_id): + if not self._is_mine_id(receipt.user_id): continue # Private read receipts never get sent over federation. if receipt.receipt_type == ReceiptTypes.READ_PRIVATE: diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index f9aa4bb2c9f..969f0303e05 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -107,9 +107,6 @@ class IReplicationConnection(Interface): def send_command(cmd: Command) -> None: """Send the command down the connection""" - def close(self) -> None: - """Close the connection""" - @implementer(IReplicationConnection) class BaseReplicationStreamProtocol(LineOnlyReceiver): diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index abb4975810e..aba79b23781 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -150,9 +150,6 @@ def connectionMade(self) -> None: "subscribe-replication", self.server_name, self._send_subscribe ) - def close(self) -> None: - pass - async def _send_subscribe(self) -> None: # it's important to make sure that we only send the REPLICATE command once we # have successfully subscribed to the stream - otherwise we might miss the diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index 37364a587a3..c488bce58e1 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -52,7 +52,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.hs = hs + self.is_mine = hs.is_mine async def on_GET( self, request: SynapseRequest, user_id: str, device_id: str @@ -60,7 +60,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -80,7 +80,7 @@ async def on_DELETE( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -96,7 +96,7 @@ async def on_PUT( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -124,7 +124,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_worker_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.hs = hs + self.is_mine = hs.is_mine async def on_GET( self, request: SynapseRequest, user_id: str @@ -132,7 +132,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) @@ -162,7 +162,7 @@ async def on_POST( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Can only create devices for local users" ) @@ -197,7 +197,7 @@ def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main - self.hs = hs + self.is_mine = hs.is_mine async def on_POST( self, request: SynapseRequest, user_id: str @@ -205,7 +205,7 @@ async def on_POST( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only lookup local users") u = await self.store.get_user_by_id(target_user.to_string()) diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index d6dd85fb698..afb71f4a0fc 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -68,7 +68,7 @@ def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() self.store = hs.get_datastores().main - self.hs = hs + self.is_mine = hs.is_mine async def on_GET( self, @@ -81,7 +81,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "User must be local to check what experimental features are enabled.", @@ -108,7 +108,7 @@ async def on_PUT( body = parse_json_object_from_request(request) target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "User must be local to enable experimental features.", diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index d872fa69358..195f22a4c25 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -266,7 +266,7 @@ class DeleteMediaByID(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name self.media_repository = hs.get_media_repository() async def on_DELETE( @@ -274,7 +274,7 @@ async def on_DELETE( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.hs._is_mine_server_name(server_name): + if not self._is_mine_server_name(server_name): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only delete local media") if await self.store.get_local_media(media_id) is None: @@ -359,7 +359,7 @@ class UserMediaRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/media$") def __init__(self, hs: "HomeServer"): - self.hs = hs + self.is_mine = hs.is_mine self.auth = hs.get_auth() self.store = hs.get_datastores().main self.media_repository = hs.get_media_repository() @@ -372,7 +372,7 @@ async def on_GET( await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine(UserID.from_string(user_id)): + if not self.is_mine(UserID.from_string(user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") user = await self.store.get_user_by_id(user_id) @@ -416,7 +416,7 @@ async def on_DELETE( await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine(UserID.from_string(user_id)): + if not self.is_mine(UserID.from_string(user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") user = await self.store.get_user_by_id(user_id) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index caf62e4cc2f..5bed89c2c46 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -494,7 +494,7 @@ def __init__(self, hs: "HomeServer"): self.admin_handler = hs.get_admin_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.hs = hs + self.is_mine = hs.is_mine async def on_POST( self, request: SynapseRequest, room_identifier: str @@ -510,7 +510,7 @@ async def on_POST( assert_params_in_dict(content, ["user_id"]) target_user = UserID.from_string(content["user_id"]) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "This endpoint can only be used with local users", @@ -587,7 +587,7 @@ def __init__(self, hs: "HomeServer"): self._state_storage_controller = hs.get_storage_controllers().state self.event_creation_handler = hs.get_event_creation_handler() self.state_handler = hs.get_state_handler() - self.hs = hs + self.is_mine_id = hs.is_mine_id async def on_POST( self, request: SynapseRequest, room_identifier: str @@ -623,7 +623,7 @@ async def on_POST( # We pick the local user with the highest power. user_power = power_levels.content.get("users", {}) admin_users = [ - user_id for user_id in user_power if self.hs.is_mine_id(user_id) + user_id for user_id in user_power if self.is_mine_id(user_id) ] admin_users.sort(key=lambda user: user_power[user]) @@ -665,7 +665,7 @@ async def on_POST( # If there is no power level events then the creator has rights. pl_content = {} admin_user_id = create_event.sender - if not self.hs.is_mine_id(admin_user_id): + if not self.is_mine_id(admin_user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "No local admin user in room", diff --git a/synapse/rest/admin/server_notice_servlet.py b/synapse/rest/admin/server_notice_servlet.py index 9f51a5a873f..f3150e88d71 100644 --- a/synapse/rest/admin/server_notice_servlet.py +++ b/synapse/rest/admin/server_notice_servlet.py @@ -62,7 +62,7 @@ def __init__(self, hs: "HomeServer"): self.server_notices_manager = hs.get_server_notices_manager() self.admin_handler = hs.get_admin_handler() self.txns = HttpTransactionCache(hs) - self.hs = hs + self.is_mine = hs.is_mine def register(self, json_resource: HttpServer) -> None: PATTERN = "/send_server_notice" @@ -97,7 +97,7 @@ async def _do( ) target_user = UserID.from_string(body["user_id"]) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Server notices can only be sent to local users" ) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 0f69e956306..25a38dc4acb 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -726,7 +726,7 @@ class WhoisRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() self.admin_handler = hs.get_admin_handler() - self.hs = hs + self.is_mine = hs.is_mine async def on_GET( self, request: SynapseRequest, user_id: str @@ -737,7 +737,7 @@ async def on_GET( if target_user != requester.user: await assert_user_is_admin(self.auth, requester) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only whois a local user") ret = await self.admin_handler.get_whois(target_user) @@ -751,7 +751,7 @@ class DeactivateAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self._deactivate_account_handler = hs.get_deactivate_account_handler() self.auth = hs.get_auth() - self.hs = hs + self.is_mine = hs.is_mine self.store = hs.get_datastores().main async def on_POST( @@ -760,7 +760,7 @@ async def on_POST( requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - if not self.hs.is_mine(UserID.from_string(target_user_id)): + if not self.is_mine(UserID.from_string(target_user_id)): raise SynapseError( HTTPStatus.BAD_REQUEST, "Can only deactivate local users" ) @@ -793,7 +793,7 @@ class SuspendAccountRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - self.hs = hs + self.is_mine = hs.is_mine self.store = hs.get_datastores().main class PutBody(RequestBodyModel): @@ -805,7 +805,7 @@ async def on_PUT( requester = await self.auth.get_user_by_req(request) await assert_user_is_admin(self.auth, requester) - if not self.hs.is_mine(UserID.from_string(target_user_id)): + if not self.is_mine(UserID.from_string(target_user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only suspend local users") if not await self.store.get_user_by_id(target_user_id): @@ -916,7 +916,7 @@ class SearchUsersRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = hs + self.is_mine = hs.is_mine async def on_GET( self, request: SynapseRequest, target_user_id: str @@ -933,7 +933,7 @@ async def on_GET( # if not is_admin and target_user != auth_user: # raise AuthError(HTTPStatus.FORBIDDEN, "You are not a server admin") - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only users a local user") term = parse_string(request, "term", required=True) @@ -985,7 +985,7 @@ class UserAdminServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = hs + self.is_mine = hs.is_mine async def on_GET( self, request: SynapseRequest, user_id: str @@ -994,7 +994,7 @@ async def on_GET( target_user = UserID.from_string(user_id) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be admins of this homeserver", @@ -1017,7 +1017,7 @@ async def on_PUT( assert_params_in_dict(body, ["admin"]) - if not self.hs.is_mine(target_user): + if not self.is_mine(target_user): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be admins of this homeserver", @@ -1041,7 +1041,7 @@ class UserMembershipRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/joined_rooms$") def __init__(self, hs: "HomeServer"): - self.hs = hs + self.is_mine = hs.is_mine self.auth = hs.get_auth() self.store = hs.get_datastores().main @@ -1073,7 +1073,7 @@ class PushersRestServlet(RestServlet): PATTERNS = admin_patterns("/users/(?P[^/]*)/pushers$") def __init__(self, hs: "HomeServer"): - self.hs = hs + self.is_mine = hs.is_mine self.store = hs.get_datastores().main self.auth = hs.get_auth() @@ -1082,7 +1082,7 @@ async def on_GET( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine(UserID.from_string(user_id)): + if not self.is_mine(UserID.from_string(user_id)): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") if not await self.store.get_user_by_id(user_id): @@ -1118,7 +1118,7 @@ def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() self.auth_handler = hs.get_auth_handler() - self.hs = hs + self.is_mine_id = hs.is_mine_id async def on_POST( self, request: SynapseRequest, user_id: str @@ -1127,7 +1127,7 @@ async def on_POST( await assert_user_is_admin(self.auth, requester) auth_user = requester.user - if not self.hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be logged in as" ) @@ -1186,14 +1186,14 @@ class ShadowBanRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = hs + self.is_mine_id = hs.is_mine_id async def on_POST( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be shadow-banned" ) @@ -1207,7 +1207,7 @@ async def on_DELETE( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be shadow-banned" ) @@ -1238,14 +1238,14 @@ class RateLimitRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.auth = hs.get_auth() - self.hs = hs + self.is_mine_id = hs.is_mine_id async def on_GET( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") if not await self.store.get_user_by_id(user_id): @@ -1276,7 +1276,7 @@ async def on_POST( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be ratelimited" ) @@ -1324,7 +1324,7 @@ async def on_DELETE( ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self.auth, request) - if not self.hs.is_mine_id(user_id): + if not self.is_mine_id(user_id): raise SynapseError( HTTPStatus.BAD_REQUEST, "Only local users can be ratelimited" ) @@ -1345,14 +1345,14 @@ class AccountDataRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self._auth = hs.get_auth() self._store = hs.get_datastores().main - self.hs = hs + self._is_mine_id = hs.is_mine_id async def on_GET( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self._auth, request) - if not self.hs._is_mine_id(user_id): + if not self._is_mine_id(user_id): raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") if not await self._store.get_user_by_id(user_id): diff --git a/synapse/rest/client/media.py b/synapse/rest/client/media.py index 80f184ea7af..4c044ae900e 100644 --- a/synapse/rest/client/media.py +++ b/synapse/rest/client/media.py @@ -134,7 +134,7 @@ def __init__( self.media_repo = media_repo self.media_storage = media_storage self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name self._server_name = hs.hostname self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.thumbnailer = ThumbnailProvider(hs, media_repo, media_storage) @@ -159,7 +159,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self.hs._is_mine_server_name(server_name): + if self._is_mine_server_name(server_name): if self.dynamic_thumbnails: await self.thumbnailer.select_or_generate_local_thumbnail( request, @@ -223,7 +223,7 @@ class DownloadResource(RestServlet): def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name self.auth = hs.get_auth() async def on_GET( @@ -258,7 +258,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self.hs._is_mine_server_name(server_name): + if self._is_mine_server_name(server_name): await self.media_repo.get_local_media( request, media_id, file_name, max_timeout_ms ) diff --git a/synapse/rest/client/thread_subscriptions.py b/synapse/rest/client/thread_subscriptions.py index 76f3914720d..4e7b5d06dbe 100644 --- a/synapse/rest/client/thread_subscriptions.py +++ b/synapse/rest/client/thread_subscriptions.py @@ -27,6 +27,7 @@ class ThreadSubscriptionsRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() + self.is_mine = hs.is_mine self.store = hs.get_datastores().main self.handler = hs.get_thread_subscriptions_handler() diff --git a/synapse/rest/media/download_resource.py b/synapse/rest/media/download_resource.py index 12b1e6701a4..3c3f703667a 100644 --- a/synapse/rest/media/download_resource.py +++ b/synapse/rest/media/download_resource.py @@ -50,7 +50,7 @@ class DownloadResource(RestServlet): def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): super().__init__() self.media_repo = media_repo - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name async def on_GET( self, @@ -82,7 +82,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self.hs._is_mine_server_name(server_name): + if self._is_mine_server_name(server_name): await self.media_repo.get_local_media( request, media_id, file_name, max_timeout_ms, allow_authenticated=False ) diff --git a/synapse/rest/media/thumbnail_resource.py b/synapse/rest/media/thumbnail_resource.py index b24533677fc..536fea4c32f 100644 --- a/synapse/rest/media/thumbnail_resource.py +++ b/synapse/rest/media/thumbnail_resource.py @@ -61,7 +61,7 @@ def __init__( self.store = hs.get_datastores().main self.media_repo = media_repo self.media_storage = media_storage - self.hs = hs + self._is_mine_server_name = hs.is_mine_server_name self._server_name = hs.hostname self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails @@ -85,7 +85,7 @@ async def on_GET( ) max_timeout_ms = min(max_timeout_ms, MAXIMUM_ALLOWED_MAX_TIMEOUT_MS) - if self.hs._is_mine_server_name(server_name): + if self._is_mine_server_name(server_name): if self.dynamic_thumbnails: await self.thumbnail_provider.select_or_generate_local_thumbnail( request, diff --git a/synapse/server.py b/synapse/server.py index 64182ff25d9..4eb26d8e11c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -1,4 +1,6 @@ # +# This file is licensed under the Affero General Public License (AGPL) version 3. +# # Copyright 2021 The Matrix.org Foundation C.I.C. # Copyright (C) 2023-2024 New Vector, Ltd # @@ -234,7 +236,7 @@ def _get(self: "HomeServer") -> T: setattr(self, depname, dep) finally: building[0] = False - + return dep return cast(F, _get) @@ -331,7 +333,7 @@ def __init__( def shutdown(self) -> None: logger.info("Received shutdown request") - + # TODO: Cleanup replication pieces from synapse.util.batching_queue import number_of_keys @@ -353,7 +355,7 @@ def shutdown(self) -> None: self.get_clock().cancel_all_looping_calls() logger.info("Stopping call_later calls") self.get_clock().cancel_all_delayed_calls() - + logger.info("Stopping background processes: %d", len(self._background_processes)) for process in self._background_processes: process.cancel() diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index d10274eddd8..0ddc24e9495 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -44,7 +44,7 @@ def __init__(self, hs: "HomeServer"): self._event_creation_handler = hs.get_event_creation_handler() self._message_handler = hs.get_message_handler() self._storage_controllers = hs.get_storage_controllers() - self.hs = hs + self._is_mine_id = hs.is_mine_id self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.servernotices.server_notices_mxid @@ -148,7 +148,7 @@ async def get_or_create_notice_room_for_user(self, user_id: str) -> str: if self.server_notices_mxid is None: raise Exception("Server notices not enabled") - assert self.hs._is_mine_id(user_id), ( + assert self._is_mine_id(user_id), ( "Cannot send server notices to remote users" ) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 03514014c2e..ce2d4676c02 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -647,7 +647,6 @@ def __init__(self, hs: "HomeServer"): # dict of set of event_ids -> _StateCacheEntry. self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( ExpiringCache( - hs=hs, cache_name="state_cache", server_name=self.server_name, clock=self.clock, diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index f875daf5301..95a34f7be1d 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -352,6 +352,7 @@ def __init__( self.server_name = hs.hostname self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() + self.is_mine_id = hs.is_mine_id self._event_persist_queue = _EventPeristenceQueue( self.server_name, self._process_event_persist_queue_task ) @@ -1108,7 +1109,7 @@ async def _prune_extremities( while events_to_check: new_events: Set[str] = set() for event_to_check in events_to_check: - if self.hs.is_mine_id(event_to_check.sender): + if self.is_mine_id(event_to_check.sender): if event_to_check.type != EventTypes.Dummy: logger.debug("Not dropping own event") return new_latest_event_ids @@ -1189,7 +1190,7 @@ async def _is_server_still_joined( """ if not any( - self.hs.is_mine_id(state_key) + self.is_mine_id(state_key) for typ, state_key in itertools.chain(delta.to_delete, delta.to_insert) if typ == EventTypes.Member ): @@ -1201,7 +1202,7 @@ async def _is_server_still_joined( # current state events_to_check = [] # Event IDs that aren't an event we're persisting for (typ, state_key), event_id in delta.to_insert.items(): - if typ != EventTypes.Member or not self.hs.is_mine_id(state_key): + if typ != EventTypes.Member or not self.is_mine_id(state_key): continue for event, _ in ev_ctx_rm: @@ -1231,7 +1232,7 @@ async def _is_server_still_joined( users_to_ignore = [ state_key for typ, state_key in itertools.chain(delta.to_insert, delta.to_delete) - if typ == EventTypes.Member and self.hs.is_mine_id(state_key) + if typ == EventTypes.Member and self.is_mine_id(state_key) ] if await self.main_store.is_local_host_in_room_ignoring_users( diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 58a02d10b6f..96159b3fbe8 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -70,7 +70,7 @@ class StateStorageController: def __init__(self, hs: "HomeServer", stores: "Databases"): self.server_name = hs.hostname # nb must be called this for @cached - self.hs = hs + self._is_mine_id = hs.is_mine_id self._clock = hs.get_clock() self.stores = stores self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) @@ -240,7 +240,7 @@ async def get_state_for_events( state_filter = StateFilter.all() await_full_state = True - if not state_filter.must_await_full_state(self.hs._is_mine_id): + if not state_filter.must_await_full_state(self._is_mine_id): await_full_state = False event_to_groups = await self.get_state_group_for_events( @@ -301,7 +301,7 @@ async def get_state_ids_for_events( state_filter = StateFilter.all() if await_full_state and not state_filter.must_await_full_state( - self.hs._is_mine_id + self._is_mine_id ): # Full state is not required if the state filter is restrictive enough. await_full_state = False @@ -603,7 +603,7 @@ async def get_current_state_ids( if state_filter is None: state_filter = StateFilter.all() - if await_full_state and state_filter.must_await_full_state(self.hs._is_mine_id): + if await_full_state and state_filter.must_await_full_state(self._is_mine_id): await self._partial_state_room_tracker.await_full_state(room_id) if state_filter is not None and not state_filter.is_full(): diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 708ebf50de5..c10e2d26113 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -93,7 +93,6 @@ def __init__( self._last_device_delete_cache: ExpiringCache[ Tuple[str, Optional[str]], int ] = ExpiringCache( - hs=hs, cache_name="last_device_delete_cache", server_name=self.server_name, clock=self._clock, diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 5ab41f64111..a451ba1013e 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -1920,6 +1920,7 @@ def __init__( where_clause="highlight=0", ) + def _action_has_highlight(actions: Collection[Union[Mapping, str]]) -> bool: for action in actions: if not isinstance(action, dict): diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 3a6abfa13fb..420591ae1af 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -253,6 +253,7 @@ def __init__( self._instance_name = hs.get_instance_name() self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages + self.is_mine_id = hs.is_mine_id # This should only exist on instances that are configured to write assert hs.get_instance_name() in hs.config.worker.writers.events, ( @@ -380,7 +381,7 @@ async def _persist_events_and_state_updates( if context.app_service: origin_type = "local" origin_entity = context.app_service.id - elif self.hs.is_mine_id(event.sender): + elif self.is_mine_id(event.sender): origin_type = "local" origin_entity = "*client*" else: @@ -580,7 +581,7 @@ async def _calculate_sliding_sync_table_changes( user_ids_to_delete_membership_snapshots = [ state_key for event_type, state_key in to_delete - if event_type == EventTypes.Member and self.hs.is_mine_id(state_key) + if event_type == EventTypes.Member and self.is_mine_id(state_key) ] membership_snapshot_shared_insert_values: SlidingSyncMembershipSnapshotSharedInsertValues = {} @@ -590,7 +591,7 @@ async def _calculate_sliding_sync_table_changes( if to_insert: membership_event_id_to_user_id_map: Dict[str, str] = {} for state_key, event_id in to_insert.items(): - if state_key[0] == EventTypes.Member and self.hs.is_mine_id( + if state_key[0] == EventTypes.Member and self.is_mine_id( state_key[1] ): membership_event_id_to_user_id_map[event_id] = state_key[1] @@ -1957,7 +1958,7 @@ def _update_current_state_txn( [ (room_id, state_key) for etype, state_key in itertools.chain(to_delete, to_insert) - if etype == EventTypes.Member and self.hs.is_mine_id(state_key) + if etype == EventTypes.Member and self.is_mine_id(state_key) ], ) @@ -1974,7 +1975,7 @@ def _update_current_state_txn( [ (room_id, key[1], ev_id, ev_id, ev_id) for key, ev_id in to_insert.items() - if key[0] == EventTypes.Member and self.hs.is_mine_id(key[1]) + if key[0] == EventTypes.Member and self.is_mine_id(key[1]) ], ) @@ -2078,7 +2079,7 @@ def _update_current_state_txn( # Check if any of the remote membership changes requires us to # unsubscribe from their device lists. self.store.handle_potentially_left_users_txn( - txn, {m for m in members_to_cache_bust if not self.hs.is_mine_id(m)} + txn, {m for m in members_to_cache_bust if not self.is_mine_id(m)} ) @classmethod @@ -3026,7 +3027,7 @@ def _store_room_members_txn( # unless its an outlier, and an outlier is only "current" if it's an "out of # band membership", like a remote invite or a rejection of a remote invite. if ( - self.hs.is_mine_id(event.state_key) + self.is_mine_id(event.state_key) and not inhibit_local_membership_updates and event.internal_metadata.is_outlier() and event.internal_metadata.is_out_of_band_membership() diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index db48273f252..a3467bff3dc 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -399,8 +399,6 @@ async def generate_user_daily_visits(self) -> None: Generates daily visit data for use in cohort/ retention analysis """ - logger.info("generate user daily visits called") - def _generate_user_daily_visits(txn: LoggingTransaction) -> None: logger.info("Calling _generate_user_daily_visits") today_start = self._get_start_of_day() diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 6cbaf029d8a..e1202b0f8c9 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -161,7 +161,6 @@ def _transact(txn: LoggingTransaction) -> int: # We always know about ourselves, even if we have nothing in # room_memberships (for example, the server is new). self._known_servers_count = max([count, 1]) - logger.info("Known servers: %s", self._known_servers_count) return self._known_servers_count @cached(max_entries=100000, iterable=True) diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 6aa54513e64..73f622cf3f4 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -50,7 +50,6 @@ class ExpiringCache(Generic[KT, VT]): def __init__( self, *, - hs: "HomeServer", cache_name: str, server_name: str, clock: Clock, diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index 7a862bfec56..75bf50e644b 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -33,7 +33,6 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_get_set(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( - hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), @@ -47,7 +46,6 @@ def test_get_set(self) -> None: def test_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( - hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), @@ -67,7 +65,6 @@ def test_eviction(self) -> None: def test_iterable_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, List[int]] = ExpiringCache( - hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), @@ -92,7 +89,6 @@ def test_iterable_eviction(self) -> None: def test_time_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, int] = ExpiringCache( - hs=self.hs, cache_name="test", server_name="testserver", clock=cast(Clock, clock), From fe7548102b16f14efd734a69e94f8435235a7793 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 15 Aug 2025 17:54:29 -0600 Subject: [PATCH 009/181] Add changelog entry --- changelog.d/18828.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/18828.feature diff --git a/changelog.d/18828.feature b/changelog.d/18828.feature new file mode 100644 index 00000000000..8a61a97ea74 --- /dev/null +++ b/changelog.d/18828.feature @@ -0,0 +1 @@ +Cleanly shutdown SynapseHomeServer object. From aa62ad866fd46d9f5730e52de91706c224895388 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 10:49:55 -0600 Subject: [PATCH 010/181] Address linter errors --- synapse/api/ratelimiting.py | 5 -- synapse/app/_base.py | 9 ++- synapse/app/homeserver.py | 7 +- synapse/app/phone_stats_home.py | 2 +- synapse/config/logger.py | 4 +- synapse/federation/send_queue.py | 14 ++-- synapse/federation/sender/__init__.py | 74 ++++++++++--------- synapse/handlers/auth.py | 4 +- synapse/handlers/delayed_events.py | 1 - synapse/handlers/device.py | 8 +- synapse/handlers/devicemessage.py | 1 - synapse/handlers/e2e_keys.py | 4 +- synapse/handlers/federation.py | 2 +- synapse/handlers/identity.py | 2 - synapse/handlers/message.py | 4 +- synapse/handlers/presence.py | 56 +++++++------- synapse/handlers/read_marker.py | 4 +- synapse/handlers/register.py | 27 ++++++- synapse/handlers/reports.py | 1 - synapse/handlers/room.py | 1 - synapse/handlers/room_member.py | 15 ++-- synapse/handlers/room_summary.py | 1 - synapse/handlers/sync.py | 5 +- synapse/http/matrixfederationclient.py | 4 +- synapse/media/media_repository.py | 5 +- synapse/media/url_previewer.py | 4 +- synapse/notifier.py | 52 +++++++------ synapse/replication/tcp/client.py | 4 +- synapse/replication/tcp/handler.py | 34 +++++---- synapse/rest/client/login.py | 2 - synapse/rest/client/login_token_request.py | 1 - synapse/rest/client/presence.py | 1 - synapse/rest/client/register.py | 1 - synapse/rest/client/sync.py | 1 - synapse/rest/media/create_resource.py | 1 - synapse/rest/synapse/mas/_base.py | 4 +- synapse/server.py | 65 ++++++++++++---- .../server_notices/server_notices_manager.py | 6 +- synapse/state/__init__.py | 4 +- synapse/storage/background_updates.py | 14 ++-- synapse/storage/controllers/purge_events.py | 2 +- synapse/storage/controllers/state.py | 5 +- synapse/storage/database.py | 14 ++-- synapse/storage/databases/main/client_ips.py | 7 +- .../databases/main/event_push_actions.py | 9 +-- synapse/storage/databases/main/events.py | 4 +- synapse/storage/databases/main/roommember.py | 14 ++-- synapse/util/__init__.py | 1 + synapse/util/batching_queue.py | 1 - synapse/util/caches/dictionary_cache.py | 1 - synapse/util/caches/expiringcache.py | 14 +++- synapse/util/task_scheduler.py | 14 ++-- 52 files changed, 312 insertions(+), 228 deletions(-) diff --git a/synapse/api/ratelimiting.py b/synapse/api/ratelimiting.py index ce2a29dd7cd..509ef6b2c18 100644 --- a/synapse/api/ratelimiting.py +++ b/synapse/api/ratelimiting.py @@ -33,7 +33,6 @@ from synapse.module_api.callbacks.ratelimit_callbacks import ( RatelimitModuleApiCallbacks, ) - from synapse.server import HomeServer class Ratelimiter: @@ -76,7 +75,6 @@ class Ratelimiter: def __init__( self, - hs: "HomeServer", store: DataStore, clock: Clock, cfg: RatelimitSettings, @@ -350,7 +348,6 @@ async def ratelimit( class RequestRatelimiter: def __init__( self, - hs: "HomeServer", store: DataStore, clock: Clock, rc_message: RatelimitSettings, @@ -361,7 +358,6 @@ def __init__( # The rate_hz and burst_count are overridden on a per-user basis self.request_ratelimiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=RatelimitSettings(key=rc_message.key, per_second=0, burst_count=0), @@ -372,7 +368,6 @@ def __init__( # by the presence of rate limits in the config if rc_admin_redaction: self.admin_redaction_ratelimiter: Optional[Ratelimiter] = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=rc_admin_redaction, diff --git a/synapse/app/_base.py b/synapse/app/_base.py index abbf322c04a..e6564f36857 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -18,7 +18,6 @@ # [This file includes modifications made by New Vector Limited] # # -import atexit import gc import logging import os @@ -99,13 +98,15 @@ logger = logging.getLogger(__name__) # list of tuples of function, args list, kwargs dict -_sighup_callbacks: Dict[str, - List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] +_sighup_callbacks: Dict[ + str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] ] = {} P = ParamSpec("P") -def register_sighup(server_name: str, func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None: +def register_sighup( + server_name: str, func: Callable[P, None], *args: P.args, **kwargs: P.kwargs +) -> None: """ Register a function to be called when a SIGHUP occurs. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 02b05b46011..4918198771d 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -22,7 +22,7 @@ import logging import os import sys -from typing import Dict, Iterable, List +from typing import Dict, Iterable, List, Optional from twisted.internet.tcp import Port from twisted.web.resource import EncodingResourceWrapper, Resource @@ -325,8 +325,9 @@ def start_listening(self) -> None: logger.warning("Unrecognized listener type: %s", listener.type) -from typing import Optional -def setup(config_options: List[str], reactor: Optional[ISynapseReactor]=None) -> SynapseHomeServer: +def setup( + config_options: List[str], reactor: Optional[ISynapseReactor] = None +) -> SynapseHomeServer: """ Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index c118a6f4505..4d24b1cfea9 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -234,7 +234,7 @@ def performance_stats_init() -> None: ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, ) # TODO: (devon) how does this hold onto a DB pool reference? - #hs.register_background_process(hs.get_datastores().main.reap_monthly_active_users()) + # hs.register_background_process(hs.get_datastores().main.reap_monthly_active_users()) def generate_monthly_active_users() -> "defer.Deferred[None]": async def _generate_monthly_active_users() -> None: diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 1faed0b49a5..4bd46aa01f2 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -348,7 +348,9 @@ def setup_logging( # Add a SIGHUP handler to reload the logging configuration, if one is available. from synapse.app import _base as appbase - appbase.register_sighup(config.server.server_name, _reload_logging_config, log_config_path) + appbase.register_sighup( + config.server.server_name, _reload_logging_config, log_config_path + ) # Log immediately so we can grep backwards. logger.warning("***** STARTING SERVER *****") diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 745e6fa18a3..21296795d23 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -112,12 +112,14 @@ def __init__(self, hs: "HomeServer"): # lambda binds to the queue rather than to the name of the queue which # changes. ARGH. def register(name: str, queue: Sized) -> None: - hs.register_later_gauge(LaterGauge( - name="synapse_federation_send_queue_%s_size" % (queue_name,), - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): len(queue)}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_federation_send_queue_%s_size" % (queue_name,), + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): len(queue)}, + ) + ) for queue_name in [ "presence_map", diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index dd7c36b7925..a204e80cd86 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -398,39 +398,47 @@ def __init__(self, hs: "HomeServer"): # map from destination to PerDestinationQueue self._per_destination_queues: Dict[str, PerDestinationQueue] = {} - hs.register_later_gauge(LaterGauge( - name="synapse_federation_transaction_queue_pending_destinations", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: { - (self.server_name,): sum( - 1 - for d in self._per_destination_queues.values() - if d.transmission_loop_running - ) - }, - )) - - hs.register_later_gauge(LaterGauge( - name="synapse_federation_transaction_queue_pending_pdus", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: { - (self.server_name,): sum( - d.pending_pdu_count() for d in self._per_destination_queues.values() - ) - }, - )) - hs.register_later_gauge(LaterGauge( - name="synapse_federation_transaction_queue_pending_edus", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: { - (self.server_name,): sum( - d.pending_edu_count() for d in self._per_destination_queues.values() - ) - }, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_federation_transaction_queue_pending_destinations", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: { + (self.server_name,): sum( + 1 + for d in self._per_destination_queues.values() + if d.transmission_loop_running + ) + }, + ) + ) + + hs.register_later_gauge( + LaterGauge( + name="synapse_federation_transaction_queue_pending_pdus", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: { + (self.server_name,): sum( + d.pending_pdu_count() + for d in self._per_destination_queues.values() + ) + }, + ) + ) + hs.register_later_gauge( + LaterGauge( + name="synapse_federation_transaction_queue_pending_edus", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: { + (self.server_name,): sum( + d.pending_edu_count() + for d in self._per_destination_queues.values() + ) + }, + ) + ) self._is_processing = False self._last_poked_id = -1 diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index c6602a353f8..2d1990cce5b 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -215,7 +215,7 @@ def __init__(self, hs: "HomeServer"): self.password_auth_provider = hs.get_password_auth_provider() - self.hs = hs # FIXME better possibility to access registrationHandler later? + self.hs = hs # FIXME better possibility to access registrationHandler later? self.macaroon_gen = hs.get_macaroon_generator() self._password_enabled_for_login = hs.config.auth.password_enabled_for_login self._password_enabled_for_reauth = hs.config.auth.password_enabled_for_reauth @@ -227,7 +227,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for failed auth during UIA. Uses same ratelimit config # as per `rc_login.failed_attempts`. self._failed_uia_attempts_ratelimiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=self.hs.config.ratelimiting.rc_login_failed_attempts, @@ -238,7 +237,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for failed /login attempts self._failed_login_attempts_ratelimiter = Ratelimiter( - hs=hs, store=self.store, clock=hs.get_clock(), cfg=self.hs.config.ratelimiting.rc_login_failed_attempts, diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index d25a7e4e2c3..ce13dcc737c 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -67,7 +67,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for management of existing delayed events, # keyed by the sending user ID & device ID. self._delayed_event_mgmt_ratelimiter = Ratelimiter( - hs=hs, store=self._store, clock=self._clock, cfg=self._config.ratelimiting.rc_delayed_event_mgmt, diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index d957b04d952..90a6c17cc5b 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1450,8 +1450,12 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler - self._remote_edu_linearizer = Linearizer(name="remote_device_list", clock=hs.get_clock()) - self._resync_linearizer = Linearizer(name="remote_device_resync", clock=hs.get_clock()) + self._remote_edu_linearizer = Linearizer( + name="remote_device_list", clock=hs.get_clock() + ) + self._resync_linearizer = Linearizer( + name="remote_device_resync", clock=hs.get_clock() + ) # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[ diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index e3c8ed827b4..b43cbd9c154 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -80,7 +80,6 @@ def __init__(self, hs: "HomeServer"): # a rate limiter for room key requests. The keys are # (sending_user_id, sending_device_id). self._ratelimiter = Ratelimiter( - hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_key_requests, diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index e9740ea7560..ac3348dbc1e 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -1766,7 +1766,9 @@ def __init__(self, hs: "HomeServer"): assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler - self._remote_edu_linearizer = Linearizer(name="remote_signing_key", clock=hs.get_clock()) + self._remote_edu_linearizer = Linearizer( + name="remote_signing_key", clock=hs.get_clock() + ) # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[str, List[Tuple[JsonDict, JsonDict]]] = {} diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 6f9d6a18545..18f078df6f4 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -784,7 +784,7 @@ async def do_invite_join( ) # We retrieve the room member handler here as to not cause a cyclic dependency - member_handler = self.get_room_member_handler() + member_handler = self.hs.get_room_member_handler() await member_handler.transfer_room_state_on_room_upgrade( old_room_id, room_id ) diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index ccb265d2bee..d96b585308b 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -73,13 +73,11 @@ def __init__(self, hs: "HomeServer"): # Ratelimiters for `/requestToken` endpoints. self._3pid_validation_ratelimiter_ip = Ratelimiter( - hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_3pid_validation, ) self._3pid_validation_ratelimiter_address = Ratelimiter( - hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_3pid_validation, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 2eddcdf2041..b84c133b6e8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -511,7 +511,9 @@ def __init__(self, hs: "HomeServer"): # We limit concurrent event creation for a room to 1. This prevents state resolution # from occurring when sending bursts of events to a local room - self.limiter = Linearizer(max_count=1, name="room_event_creation_limit", clock=hs.get_clock()) + self.limiter = Linearizer( + max_count=1, name="room_event_creation_limit", clock=hs.get_clock() + ) self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 451bf4f091a..bb49969900c 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -525,11 +525,11 @@ def __init__(self, hs: "HomeServer"): self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs) self._set_state_client = ReplicationPresenceSetState.make_client(hs) - self.clock.looping_call( - self.send_stop_syncing, UPDATE_SYNCING_USERS_MS - ) + self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS_MS) - hs.register_async_shutdown_handler("generic_presence.on_shutdown", self._on_shutdown) + hs.register_async_shutdown_handler( + "generic_presence.on_shutdown", self._on_shutdown + ) @wrap_as_background_process("WorkerPresenceHandler._on_shutdown") async def _on_shutdown(self) -> None: @@ -773,12 +773,14 @@ def __init__(self, hs: "HomeServer"): EduTypes.PRESENCE, self.incoming_presence ) - hs.register_later_gauge(LaterGauge( - name="synapse_handlers_presence_user_to_current_state_size", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): len(self.user_to_current_state)}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_handlers_presence_user_to_current_state_size", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): len(self.user_to_current_state)}, + ) + ) # The per-device presence state, maps user to devices to per-device presence state. self._user_to_device_to_current_state: Dict[ @@ -849,7 +851,9 @@ def __init__(self, hs: "HomeServer"): ] = {} self.external_process_last_updated_ms: Dict[str, int] = {} - self.external_sync_linearizer = Linearizer(name="external_sync_linearizer", clock=hs.get_clock()) + self.external_sync_linearizer = Linearizer( + name="external_sync_linearizer", clock=hs.get_clock() + ) if self._track_presence: # Start a LoopingCall in 30s that fires every 5s. @@ -869,12 +873,14 @@ def __init__(self, hs: "HomeServer"): 60 * 1000, ) - hs.register_later_gauge(LaterGauge( - name="synapse_handlers_presence_wheel_timer_size", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): len(self.wheel_timer)}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_handlers_presence_wheel_timer_size", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): len(self.wheel_timer)}, + ) + ) # Used to handle sending of presence to newly joined users/servers if self._track_presence: @@ -1889,9 +1895,7 @@ async def get_new_events( # Figure out which other users this user should explicitly receive # updates for additional_users_interested_in = ( - await self.get_presence_router().get_interested_users( - user.to_string() - ) + await self.get_presence_router().get_interested_users(user.to_string()) ) # We have a set of users that we're interested in the presence of. We want to @@ -1971,10 +1975,8 @@ async def get_new_events( interested_and_updated_users.update(additional_users_interested_in) # Retrieve the current presence state for each user - users_to_state = ( - await self.get_presence_handler().current_state_for_users( - interested_and_updated_users - ) + users_to_state = await self.get_presence_handler().current_state_for_users( + interested_and_updated_users ) presence_updates = list(users_to_state.values()) @@ -2014,10 +2016,8 @@ async def _filter_all_presence_updates_for_user( if updated_users is not None: # Get the actual presence update for each change - users_to_state = ( - await self.get_presence_handler().current_state_for_users( - updated_users - ) + users_to_state = await self.get_presence_handler().current_state_for_users( + updated_users ) presence_updates = list(users_to_state.values()) diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index d04412aea2b..85d2dd62bb7 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -36,7 +36,9 @@ class ReadMarkerHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.account_data_handler = hs.get_account_data_handler() - self.read_marker_linearizer = Linearizer(name="read_marker", clock=hs.get_clock()) + self.read_marker_linearizer = Linearizer( + name="read_marker", clock=hs.get_clock() + ) async def received_client_read_marker( self, room_id: str, user_id: str, event_id: str diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 300b4007348..06739951642 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -23,7 +23,16 @@ """Contains functions for registering clients.""" import logging -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, TypedDict +from typing import ( + TYPE_CHECKING, + Any, + Coroutine, + Iterable, + List, + Optional, + Tuple, + TypedDict, +) from prometheus_client import Counter @@ -136,6 +145,7 @@ def __init__(self, hs: "HomeServer"): ) else: self.device_handler = hs.get_device_handler() + async def do_it( user_id: str, device_id: Optional[str], @@ -145,8 +155,19 @@ async def do_it( should_issue_refresh_token: bool = False, auth_provider_id: Optional[str] = None, auth_provider_session_id: Optional[str] = None, - ): - return RegistrationHandler.register_device_inner(self, user_id, device_id,initial_display_name, is_guest,is_appservice_ghost,should_issue_refresh_token, auth_provider_id,auth_provider_session_id) + ) -> Coroutine[Any, Any, LoginDict]: + return RegistrationHandler.register_device_inner( + self, + user_id, + device_id, + initial_display_name, + is_guest, + is_appservice_ghost, + should_issue_refresh_token, + auth_provider_id, + auth_provider_session_id, + ) + self._register_device_client = do_it self.pusher_pool = hs.get_pusherpool() diff --git a/synapse/handlers/reports.py b/synapse/handlers/reports.py index 116685e6de5..a7b8a4bed74 100644 --- a/synapse/handlers/reports.py +++ b/synapse/handlers/reports.py @@ -38,7 +38,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for management of existing delayed events, # keyed by the requesting user ID. self._reports_ratelimiter = Ratelimiter( - hs=hs, store=self._store, clock=self._clock, cfg=hs.config.ratelimiting.rc_reports, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8df4213d48b..47bd139ca71 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -137,7 +137,6 @@ def __init__(self, hs: "HomeServer"): self.config = hs.config self.common_request_ratelimiter = hs.get_request_ratelimiter() self.creation_ratelimiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=self.config.ratelimiting.rc_room_creation, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 3fd79dce847..623823acb02 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -114,8 +114,12 @@ def __init__(self, hs: "HomeServer"): if self.hs.config.server.include_profile_data_on_invite: self._membership_types_to_include_profile_data_in.add(Membership.INVITE) - self.member_linearizer: Linearizer = Linearizer(name="member", clock=hs.get_clock()) - self.member_as_limiter = Linearizer(max_count=10, name="member_as_limiter", clock=hs.get_clock()) + self.member_linearizer: Linearizer = Linearizer( + name="member", clock=hs.get_clock() + ) + self.member_as_limiter = Linearizer( + max_count=10, name="member_as_limiter", clock=hs.get_clock() + ) self.clock = hs.get_clock() self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker @@ -127,7 +131,6 @@ def __init__(self, hs: "HomeServer"): self.allow_per_room_profiles = self.config.server.allow_per_room_profiles self._join_rate_limiter_local = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_joins_local, @@ -136,7 +139,6 @@ def __init__(self, hs: "HomeServer"): # I.e. joins this server makes by requesting /make_join /send_join from # another server. self._join_rate_limiter_remote = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_joins_remote, @@ -150,7 +152,6 @@ def __init__(self, hs: "HomeServer"): # other homeservers # I wonder if a homeserver-wide collection of rate limiters might be cleaner? self._join_rate_per_room_limiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_joins_per_room, @@ -159,7 +160,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for invites, keyed by room (across all issuers, all # recipients). self._invites_per_room_limiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_room, @@ -169,7 +169,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for invites, keyed by recipient (across all rooms, all # issuers). self._invites_per_recipient_limiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_user, @@ -179,7 +178,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for invites, keyed by issuer (across all rooms, all # recipients). self._invites_per_issuer_limiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_invites_per_issuer, @@ -187,7 +185,6 @@ def __init__(self, hs: "HomeServer"): ) self._third_party_invite_limiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_third_party_invite, diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 9eb6bd75a4b..838fee6a303 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -104,7 +104,6 @@ def __init__(self, hs: "HomeServer"): self._server_name = hs.hostname self._federation_client = hs.get_federation_client() self._ratelimiter = Ratelimiter( - hs=hs, store=self._store, clock=hs.get_clock(), cfg=RatelimitSettings("", per_second=5, burst_count=10), diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 94365d0ed82..01d091e7f02 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1141,8 +1141,9 @@ def get_lazy_loaded_members_cache( ) if cache is None: logger.debug("creating LruCache for %r", cache_key) - cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE, - ) + cache = LruCache( + max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE, + ) self.lazy_loaded_members_cache[cache_key] = cache else: logger.debug("found LruCache for %r", cache_key) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 11eee599e68..afe305ab41d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -481,7 +481,9 @@ def __init__( use_proxy=True, ) - self.remote_download_linearizer = Linearizer("remote_download_linearizer", 6, clock=hs.get_clock()) + self.remote_download_linearizer = Linearizer( + "remote_download_linearizer", 6, clock=hs.get_clock() + ) def wake_destination(self, destination: str) -> None: """Called when the remote server may have come back online.""" diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 16bf9d9c4ad..6ccf8ccb284 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -108,7 +108,9 @@ def __init__(self, hs: "HomeServer"): self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails self.thumbnail_requirements = hs.config.media.thumbnail_requirements - self.remote_media_linearizer = Linearizer(name="media_remote", clock=hs.get_clock()) + self.remote_media_linearizer = Linearizer( + name="media_remote", clock=hs.get_clock() + ) self.recently_accessed_remotes: Set[Tuple[str, str]] = set() self.recently_accessed_locals: Set[str] = set() @@ -119,7 +121,6 @@ def __init__(self, hs: "HomeServer"): self.prevent_media_downloads_from = hs.config.media.prevent_media_downloads_from self.download_ratelimiter = Ratelimiter( - hs=hs, store=hs.get_storage_controllers().main, clock=hs.get_clock(), cfg=hs.config.ratelimiting.remote_media_downloads, diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 60cdf0fed52..6f6914ab0d5 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -207,9 +207,7 @@ def __init__( ) if self._worker_run_media_background_jobs: - self.clock.looping_call( - self._start_expire_url_cache_data, 10 * 1000 - ) + self.clock.looping_call(self._start_expire_url_cache_data, 10 * 1000) async def preview(self, url: str, user: UserID, ts: int) -> bytes: # the in-memory cache: diff --git a/synapse/notifier.py b/synapse/notifier.py index 0e79b69a1dc..77209d28259 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -282,29 +282,35 @@ def count_listeners() -> Mapping[Tuple[str, ...], int]: ) } - hs.register_later_gauge(LaterGauge( - name="synapse_notifier_listeners", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=count_listeners, - )) - - hs.register_later_gauge(LaterGauge( - name="synapse_notifier_rooms", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: { - (self.server_name,): count( - bool, list(self.room_to_user_streams.values()) - ) - }, - )) - hs.register_later_gauge(LaterGauge( - name="synapse_notifier_users", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): len(self.user_to_user_stream)}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_notifier_listeners", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=count_listeners, + ) + ) + + hs.register_later_gauge( + LaterGauge( + name="synapse_notifier_rooms", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: { + (self.server_name,): count( + bool, list(self.room_to_user_streams.values()) + ) + }, + ) + ) + hs.register_later_gauge( + LaterGauge( + name="synapse_notifier_users", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): len(self.user_to_user_stream)}, + ) + ) def add_replication_callback(self, cb: Callable[[], None]) -> None: """Add a callback that will be called when some new data is available. diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 908b87cba14..9583a76779a 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -429,7 +429,9 @@ def __init__(self, hs: "HomeServer"): # to. This is always set before we use it. self.federation_position: Optional[int] = None - self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer", clock=hs.get_clock()) + self._fed_position_linearizer = Linearizer( + name="_fed_position_linearizer", clock=hs.get_clock() + ) async def process_replication_rows( self, stream_name: str, token: int, rows: list diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 6fd9831ebc0..99805335ffc 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -243,12 +243,14 @@ def __init__(self, hs: "HomeServer"): # outgoing replication commands to.) self._connections: List[IReplicationConnection] = [] - hs.register_later_gauge(LaterGauge( - name="synapse_replication_tcp_resource_total_connections", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): len(self._connections)}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_replication_tcp_resource_total_connections", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): len(self._connections)}, + ) + ) # When POSITION or RDATA commands arrive, we stick them in a queue and process # them in order in a separate background process. @@ -266,15 +268,17 @@ def __init__(self, hs: "HomeServer"): # from that connection. self._streams_by_connection: Dict[IReplicationConnection, Set[str]] = {} - hs.register_later_gauge(LaterGauge( - name="synapse_replication_tcp_command_queue", - desc="Number of inbound RDATA/POSITION commands queued for processing", - labelnames=["stream_name", SERVER_NAME_LABEL], - caller=lambda: { - (stream_name, self.server_name): len(queue) - for stream_name, queue in self._command_queues_by_stream.items() - }, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_replication_tcp_command_queue", + desc="Number of inbound RDATA/POSITION commands queued for processing", + labelnames=["stream_name", SERVER_NAME_LABEL], + caller=lambda: { + (stream_name, self.server_name): len(queue) + for stream_name, queue in self._command_queues_by_stream.items() + }, + ) + ) self._is_master = hs.config.worker.worker_app is None diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 987df3a3d1f..acb9111ad28 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -126,13 +126,11 @@ def __init__(self, hs: "HomeServer"): self._well_known_builder = WellKnownBuilder(hs) self._address_ratelimiter = Ratelimiter( - hs=hs, store=self._main_store, clock=hs.get_clock(), cfg=self.hs.config.ratelimiting.rc_login_address, ) self._account_ratelimiter = Ratelimiter( - hs=hs, store=self._main_store, clock=hs.get_clock(), cfg=self.hs.config.ratelimiting.rc_login_account, diff --git a/synapse/rest/client/login_token_request.py b/synapse/rest/client/login_token_request.py index d86b87e190a..a053db8e551 100644 --- a/synapse/rest/client/login_token_request.py +++ b/synapse/rest/client/login_token_request.py @@ -79,7 +79,6 @@ def __init__(self, hs: "HomeServer"): # This endpoint can be used to spawn additional sessions and could be # abused by a malicious client to create many sessions. self._ratelimiter = Ratelimiter( - hs=hs, store=self._main_store, clock=hs.get_clock(), cfg=RatelimitSettings( diff --git a/synapse/rest/client/presence.py b/synapse/rest/client/presence.py index f439da8e52b..104d54cd890 100644 --- a/synapse/rest/client/presence.py +++ b/synapse/rest/client/presence.py @@ -53,7 +53,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for presence updates, keyed by requester. self._presence_per_user_limiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_presence_per_user, diff --git a/synapse/rest/client/register.py b/synapse/rest/client/register.py index 0d4d20fb0b7..102c04bb67d 100644 --- a/synapse/rest/client/register.py +++ b/synapse/rest/client/register.py @@ -397,7 +397,6 @@ def __init__(self, hs: "HomeServer"): self.hs = hs self.store = hs.get_datastores().main self.ratelimiter = Ratelimiter( - hs=hs, store=self.store, clock=hs.get_clock(), cfg=hs.config.ratelimiting.rc_registration_token_validity, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 11bb93947c9..d7f1fa81695 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -131,7 +131,6 @@ def __init__(self, hs: "HomeServer"): # Ratelimiter for presence updates, keyed by requester. self._presence_per_user_limiter = Ratelimiter( - hs=hs, store=self.store, clock=self.clock, cfg=hs.config.ratelimiting.rc_presence_per_user, diff --git a/synapse/rest/media/create_resource.py b/synapse/rest/media/create_resource.py index b832e4bc0e4..e45df11c9fa 100644 --- a/synapse/rest/media/create_resource.py +++ b/synapse/rest/media/create_resource.py @@ -49,7 +49,6 @@ def __init__(self, hs: "HomeServer", media_repo: "MediaRepository"): # A rate limiter for creating new media IDs. self._create_media_rate_limiter = Ratelimiter( - hs=hs, store=hs.get_datastores().main, clock=self.clock, cfg=hs.config.ratelimiting.rc_media_create, diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py index bea321e4166..07332af1773 100644 --- a/synapse/rest/synapse/mas/_base.py +++ b/synapse/rest/synapse/mas/_base.py @@ -42,7 +42,9 @@ def __init__(self, hs: "HomeServer"): self._is_request_from_mas = auth.is_request_using_the_admin_token - DirectServeJsonResource.__init__(self, extract_context=True, clock=hs.get_clock()) + DirectServeJsonResource.__init__( + self, extract_context=True, clock=hs.get_clock() + ) self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.hostname = hs.hostname diff --git a/synapse/server.py b/synapse/server.py index 4eb26d8e11c..aee4b59e055 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -28,14 +28,23 @@ import abc import functools import logging -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Type, TypeVar, cast -from attr import dataclass +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Type, + TypeVar, + cast, +) -from twisted.internet import defer +from attr import dataclass from typing_extensions import TypeAlias -from twisted.internet.interfaces import IDelayedCall, IOpenSSLContextFactory -from twisted.internet.task import LoopingCall +from twisted.internet import defer +from twisted.internet.interfaces import IOpenSSLContextFactory from twisted.internet.tcp import Port from twisted.python.threadpool import ThreadPool from twisted.web.iweb import IPolicyForHTTPS @@ -143,7 +152,6 @@ from synapse.replication.tcp.client import ReplicationDataHandler from synapse.replication.tcp.external_cache import ExternalCache from synapse.replication.tcp.handler import ReplicationCommandHandler -from synapse.replication.tcp.protocol import connected_connections from synapse.replication.tcp.resource import ReplicationStreamer from synapse.replication.tcp.streams import STREAMS_MAP, Stream from synapse.rest.media.media_repository_resource import MediaRepositoryResource @@ -168,7 +176,9 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: + # Old versions don't have `LiteralString` from txredisapi import ConnectionHandler + from typing_extensions import LiteralString from synapse.handlers.jwt import JwtHandler from synapse.handlers.oidc import OidcHandler @@ -337,8 +347,10 @@ def shutdown(self) -> None: # TODO: Cleanup replication pieces from synapse.util.batching_queue import number_of_keys + number_of_keys.clear() from synapse.util.batching_queue import number_queued + number_queued.clear() # TODO: unregister all metrics that bind to HS resources! @@ -356,13 +368,18 @@ def shutdown(self) -> None: logger.info("Stopping call_later calls") self.get_clock().cancel_all_delayed_calls() - logger.info("Stopping background processes: %d", len(self._background_processes)) + logger.info( + "Stopping background processes: %d", len(self._background_processes) + ) for process in self._background_processes: process.cancel() self._background_processes.clear() from synapse.util.caches import CACHE_METRIC_REGISTRY - logger.info("Clearing cache metrics: %d", len(CACHE_METRIC_REGISTRY._pre_update_hooks)) + + logger.info( + "Clearing cache metrics: %d", len(CACHE_METRIC_REGISTRY._pre_update_hooks) + ) # TODO: Do this better, ie. don't clear metrics for other tenants # only clear them for this server CACHE_METRIC_REGISTRY.clear() @@ -371,7 +388,7 @@ def shutdown(self) -> None: logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) # TODO: we should probably run these - #yield defer.ensureDeferred(shutdown_handler.func()) + # yield defer.ensureDeferred(shutdown_handler.func()) self._async_shutdown_handlers.clear() for shutdown_handler in self._sync_shutdown_handlers: @@ -381,12 +398,15 @@ def shutdown(self) -> None: self._sync_shutdown_handlers.clear() from synapse.app._base import unregister_sighups + unregister_sighups(self.config.server.server_name) for call in self.get_reactor().getDelayedCalls(): call.cancel() - def register_async_shutdown_handler(self, desc: "LiteralString", shutdown_func: Callable[..., Any]) -> None: + def register_async_shutdown_handler( + self, desc: "LiteralString", shutdown_func: Callable[..., Any] + ) -> None: id = self.get_reactor().addSystemEventTrigger( "before", "shutdown", @@ -395,15 +415,25 @@ def register_async_shutdown_handler(self, desc: "LiteralString", shutdown_func: self.config.server.server_name, shutdown_func, ) - self._async_shutdown_handlers.append(ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id)) + self._async_shutdown_handlers.append( + ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id) + ) - def register_sync_shutdown_handler(self, phase: str, eventType: str, desc: "LiteralString", shutdown_func: Callable[..., Any]) -> None: + def register_sync_shutdown_handler( + self, + phase: str, + eventType: str, + desc: "LiteralString", + shutdown_func: Callable[..., Any], + ) -> None: id = self.get_reactor().addSystemEventTrigger( phase, eventType, shutdown_func, ) - self._sync_shutdown_handlers.append(ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id)) + self._sync_shutdown_handlers.append( + ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id) + ) def register_later_gauge(self, later_gauge: LaterGauge) -> None: self._later_gauges.append(later_gauge) @@ -528,7 +558,6 @@ def get_distributor(self) -> Distributor: @cache_in_self def get_registration_ratelimiter(self) -> Ratelimiter: return Ratelimiter( - hs=self, store=self.get_datastores().main, clock=self.get_clock(), cfg=self.config.ratelimiting.rc_registration, @@ -1048,7 +1077,6 @@ def should_send_federation(self) -> bool: @cache_in_self def get_request_ratelimiter(self) -> RequestRatelimiter: return RequestRatelimiter( - self, self.get_datastores().main, self.get_clock(), self.config.ratelimiting.rc_message, @@ -1080,7 +1108,12 @@ def get_media_sender_thread_pool(self) -> ThreadPool: ) media_threadpool.start() - hs.register_sync_shutdown_handler("during", "shutdown", "Homeserver media_threadpool.stop", media_threadpool.stop) + self.register_sync_shutdown_handler( + "during", + "shutdown", + "Homeserver media_threadpool.stop", + media_threadpool.stop, + ) # Register the threadpool with our metrics. server_name = self.hostname diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 0ddc24e9495..19f86b5a563 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -44,7 +44,7 @@ def __init__(self, hs: "HomeServer"): self._event_creation_handler = hs.get_event_creation_handler() self._message_handler = hs.get_message_handler() self._storage_controllers = hs.get_storage_controllers() - self._is_mine_id = hs.is_mine_id + self._is_mine_id = hs.is_mine_id self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.servernotices.server_notices_mxid @@ -148,9 +148,7 @@ async def get_or_create_notice_room_for_user(self, user_id: str) -> str: if self.server_notices_mxid is None: raise Exception("Server notices not enabled") - assert self._is_mine_id(user_id), ( - "Cannot send server notices to remote users" - ) + assert self._is_mine_id(user_id), "Cannot send server notices to remote users" requester = create_requester( self.server_notices_mxid, authenticated_entity=self.server_name diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index ce2d4676c02..8e7af360b80 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -642,7 +642,9 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() - self.resolve_linearizer = Linearizer(name="state_resolve_lock", clock=hs.get_clock()) + self.resolve_linearizer = Linearizer( + name="state_resolve_lock", clock=hs.get_clock() + ) # dict of set of event_ids -> _StateCacheEntry. self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 6dd3714983e..12ad973f982 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -395,12 +395,14 @@ def start_doing_background_updates(self) -> None: # if we start a new background update, not all updates are done. self._all_done = False sleep = self.sleep_enabled - self.hs.register_background_process(run_as_background_process( - "background_updates", - self.server_name, - self.run_background_updates, - sleep, - )) + self.hs.register_background_process( + run_as_background_process( + "background_updates", + self.server_name, + self.run_background_updates, + sleep, + ) + ) async def run_background_updates(self, sleep: bool) -> None: if self._running or not self.enabled: diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 9c96b293b11..14b37ac543b 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -52,7 +52,7 @@ def __init__(self, hs: "HomeServer", stores: Databases): self.stores = stores if hs.config.worker.run_background_tasks: - hs.get_clock().looping_call( + self._delete_state_loop_call = hs.get_clock().looping_call( self._delete_state_groups_loop, 60 * 1000 ) diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 96159b3fbe8..7528b3d2a68 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -37,7 +37,6 @@ from synapse.api.constants import EventTypes, Membership from synapse.events import EventBase -from synapse.http import proxy from synapse.logging.opentracing import tag_args, trace from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.roommember import ProfileInfo @@ -78,7 +77,9 @@ def __init__(self, hs: "HomeServer", stores: "Databases"): # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. - self._joined_host_linearizer = Linearizer("_JoinedHostsCache", clock=hs.get_clock()) + self._joined_host_linearizer = Linearizer( + "_JoinedHostsCache", clock=hs.get_clock() + ) def notify_event_un_partial_stated(self, event_id: str) -> None: self._partial_state_events_tracker.notify_un_partial_stated(event_id) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 50a46fcdf74..d53ecfdba41 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -611,12 +611,14 @@ def __init__( ) self.updates = BackgroundUpdater(hs, self) - hs.register_later_gauge(LaterGauge( - name="synapse_background_update_status", - desc="Background update status", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): self.updates.get_status()}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_background_update_status", + desc="Background update status", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): self.updates.get_status()}, + ) + ) self._previous_txn_total_time = 0.0 self._current_txn_total_time = 0.0 diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 12d372935dd..4a9a57ff455 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -452,10 +452,11 @@ def __init__( Tuple[str, str, str], Tuple[str, Optional[str], int] ] = {} - self._clock.looping_call( - self._update_client_ips_batch, 5 * 1000 + self._clock.looping_call(self._update_client_ips_batch, 5 * 1000) + hs.register_async_shutdown_handler( + "ClientIpWorkerStore _update_client_ips_batch", + self._update_client_ips_batch, ) - hs.register_async_shutdown_handler("ClientIpWorkerStore _update_client_ips_batch", self._on_shutdown) @wrap_as_background_process("prune_old_user_ips") async def _prune_old_user_ips(self) -> None: diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index a451ba1013e..0b1eaf6d591 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -94,6 +94,7 @@ ) import attr + from twisted.internet.task import LoopingCall from synapse.api.constants import MAIN_TIMELINE, ReceiptTypes @@ -276,16 +277,12 @@ def __init__( self._find_stream_orderings_for_times_txn(cur) cur.close() - self._clock.looping_call( - self._find_stream_orderings_for_times, 10 * 60 * 1000 - ) + self._clock.looping_call(self._find_stream_orderings_for_times, 10 * 60 * 1000) self._rotate_count = 10000 self._doing_notif_rotation = False if hs.config.worker.run_background_tasks: - self._clock.looping_call( - self._rotate_notifs, 30 * 1000 - ) + self._clock.looping_call(self._rotate_notifs, 30 * 1000) self._clock.looping_call( self._clear_old_push_actions_staging, 30 * 60 * 1000 diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 420591ae1af..c6fe8a6901b 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -591,9 +591,7 @@ async def _calculate_sliding_sync_table_changes( if to_insert: membership_event_id_to_user_id_map: Dict[str, str] = {} for state_key, event_id in to_insert.items(): - if state_key[0] == EventTypes.Member and self.is_mine_id( - state_key[1] - ): + if state_key[0] == EventTypes.Member and self.is_mine_id(state_key[1]): membership_event_id_to_user_id_map[event_id] = state_key[1] membership_event_map: Dict[str, EventBase] = {} diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index e1202b0f8c9..fce6edad652 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -120,12 +120,14 @@ def __init__( 1, self._count_known_servers, ) - hs.register_later_gauge(LaterGauge( - name="synapse_federation_known_servers", - desc="", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): self._known_servers_count}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_federation_known_servers", + desc="", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): self._known_servers_count}, + ) + ) @wrap_as_background_process("_count_known_servers") async def _count_known_servers(self) -> int: diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 5b1a3a284ca..8ee7b3dc871 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -228,6 +228,7 @@ def call_later( id = self._call_id self._call_id += 1 + def wrapped_callback(*args: Any, **kwargs: Any) -> None: with context.PreserveLoggingContext(): callback(*args, **kwargs) diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 43804aeeb0b..470906484ba 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -128,7 +128,6 @@ def __init__( name=self._name, **{SERVER_NAME_LABEL: self.server_name} ) - async def add_to_queue(self, value: V, key: Hashable = ()) -> R: """Adds the value to the queue with the given key, returning the result of the processing function for the batch that included the given value. diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 9f49c5f8a97..168ddc51cd5 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -34,7 +34,6 @@ ) import attr -from synapse.util import Clock from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 73f622cf3f4..c7476a10c52 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -21,7 +21,17 @@ import logging from collections import OrderedDict -from typing import TYPE_CHECKING, Any, Generic, Iterable, Literal, Optional, TypeVar, Union, overload +from typing import ( + TYPE_CHECKING, + Any, + Generic, + Iterable, + Literal, + Optional, + TypeVar, + Union, + overload, +) import attr @@ -33,7 +43,7 @@ from synapse.util.caches import EvictionReason, register_cache if TYPE_CHECKING: - from synapse.server import HomeServer + pass logger = logging.getLogger(__name__) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 108dd06dddf..fc688795272 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -130,12 +130,14 @@ def __init__(self, hs: "HomeServer"): TaskScheduler.SCHEDULE_INTERVAL_MS, ) - hs.register_later_gauge(LaterGauge( - name="synapse_scheduler_running_tasks", - desc="The number of concurrent running tasks handled by the TaskScheduler", - labelnames=[SERVER_NAME_LABEL], - caller=lambda: {(self.server_name,): len(self._running_tasks)}, - )) + hs.register_later_gauge( + LaterGauge( + name="synapse_scheduler_running_tasks", + desc="The number of concurrent running tasks handled by the TaskScheduler", + labelnames=[SERVER_NAME_LABEL], + caller=lambda: {(self.server_name,): len(self._running_tasks)}, + ) + ) def register_action( self, From adeba659405e8b9c69f8d188e45fc01092252998 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 11:15:23 -0600 Subject: [PATCH 011/181] Remove unnecessary redirect --- synapse/handlers/register.py | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 06739951642..c3ff0cfaf81 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -25,8 +25,6 @@ import logging from typing import ( TYPE_CHECKING, - Any, - Coroutine, Iterable, List, Optional, @@ -145,30 +143,7 @@ def __init__(self, hs: "HomeServer"): ) else: self.device_handler = hs.get_device_handler() - - async def do_it( - user_id: str, - device_id: Optional[str], - initial_display_name: Optional[str], - is_guest: bool = False, - is_appservice_ghost: bool = False, - should_issue_refresh_token: bool = False, - auth_provider_id: Optional[str] = None, - auth_provider_session_id: Optional[str] = None, - ) -> Coroutine[Any, Any, LoginDict]: - return RegistrationHandler.register_device_inner( - self, - user_id, - device_id, - initial_display_name, - is_guest, - is_appservice_ghost, - should_issue_refresh_token, - auth_provider_id, - auth_provider_session_id, - ) - - self._register_device_client = do_it + self._register_device_client = self.register_device_inner self.pusher_pool = hs.get_pusherpool() self.session_lifetime = hs.config.registration.session_lifetime From 35fc370d1c00135c753a4fd0089d9d40544110b2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 11:23:36 -0600 Subject: [PATCH 012/181] Add new func to MockHomeserver --- synapse/_scripts/synapse_port_db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 0f54cfc64af..5a9dcb34403 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -339,6 +339,9 @@ def should_send_federation(self) -> bool: def get_replication_notifier(self) -> ReplicationNotifier: return ReplicationNotifier() + def register_later_gauge(self, later_gauge: LaterGauge) -> None: + pass + class Porter: def __init__( From f26088b1c425ca08a5ab34e6b928457ee91a2e8c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 11:38:18 -0600 Subject: [PATCH 013/181] Add import --- synapse/_scripts/synapse_port_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 5a9dcb34403..7a76b4fb2b8 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -58,6 +58,7 @@ make_deferred_yieldable, run_in_background, ) +from synapse.metrics import LaterGauge from synapse.notifier import ReplicationNotifier from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn from synapse.storage.databases.main import FilteringWorkerStore From 287fc3c18c25d29f94172b469af7b760b9febc12 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 12:35:07 -0600 Subject: [PATCH 014/181] Modify tests to shutdown homeserver on teardown --- synapse/app/_base.py | 2 ++ synapse/app/homeserver.py | 22 +++++++++++++--------- synapse/server.py | 20 +++++++++++++------- synapse/util/__init__.py | 8 ++++++-- tests/unittest.py | 1 + 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e6564f36857..43f82cd28c0 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -307,6 +307,7 @@ def listen_metrics(bind_addresses: StrCollection, port: int) -> None: from synapse.metrics import RegistryProxy + # TODO: track for shutdown for host in bind_addresses: logger.info("Starting metrics listener on %s:%d", host, port) start_http_server_prometheus(port, addr=host, registry=RegistryProxy) @@ -329,6 +330,7 @@ def listen_manhole( from synapse.util.manhole import manhole + # TODO: return port listen_tcp( bind_addresses, port, diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 4918198771d..d512f4ea274 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -89,15 +89,19 @@ def shutdown(self) -> None: logger.info("Shutting down listening services") for listener in self._listening_services: - logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) - # Preferred over connectionLost since it allows buffers to flush - listener.unregisterProducer() - listener.loseConnection() - - # NOTE: not guaranteed to immediately shutdown - # Sometimes takes a second for some deferred to fire that cancels the socket - # But seems to always do so within a minute - # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. + # During unit tests, an incomplete `_FakePort` is used for listeners so + # check listener type here to ensure shutdown procedure is only applied to + # actual `Port` instances. + if listener is Port: + #logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) + # Preferred over connectionLost since it allows buffers to flush + listener.unregisterProducer() + listener.loseConnection() + + # NOTE: not guaranteed to immediately shutdown + # Sometimes takes a second for some deferred to fire that cancels the socket + # But seems to always do so within a minute + # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. self._listening_services.clear() def _listener_http( diff --git a/synapse/server.py b/synapse/server.py index aee4b59e055..b272e02a8d4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -385,16 +385,22 @@ def shutdown(self) -> None: CACHE_METRIC_REGISTRY.clear() for shutdown_handler in self._async_shutdown_handlers: - logger.info("Shutting down %s", shutdown_handler.desc) - self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - # TODO: we should probably run these - # yield defer.ensureDeferred(shutdown_handler.func()) + try: + logger.info("Shutting down %s", shutdown_handler.desc) + self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) + # TODO: we should probably run these + # yield defer.ensureDeferred(shutdown_handler.func()) + except Exception: + pass self._async_shutdown_handlers.clear() for shutdown_handler in self._sync_shutdown_handlers: - logger.info("Shutting down %s", shutdown_handler.desc) - self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - shutdown_handler.func() + try: + logger.info("Shutting down %s", shutdown_handler.desc) + self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) + shutdown_handler.func() + except Exception: + pass self._sync_shutdown_handlers.clear() from synapse.app._base import unregister_sighups diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 8ee7b3dc871..659f4405e86 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -206,9 +206,13 @@ def _looping_call_common( self._looping_calls.append(call) return call - def cancel_all_looping_calls(self) -> None: + def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: for call in self._looping_calls: - call.stop() + try: + call.stop() + except Exception: + if not consumeErrors: + raise self._looping_calls.clear() def call_later( diff --git a/tests/unittest.py b/tests/unittest.py index 5e6957dc6d2..a5f1aea3aa9 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -473,6 +473,7 @@ async def get_requester(*args: Any, **kwargs: Any) -> Requester: def tearDown(self) -> None: # Reset to not use frozen dicts. events.USE_FROZEN_DICTS = False + self.hs.shutdown() def wait_on_thread(self, deferred: Deferred, timeout: int = 10) -> None: """ From a94c483c27561e83f9d78e2381ea44ad62399bbf Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 12:39:29 -0600 Subject: [PATCH 015/181] Fix type comparison --- synapse/app/homeserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index d512f4ea274..ca0dc8d5707 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -92,8 +92,8 @@ def shutdown(self) -> None: # During unit tests, an incomplete `_FakePort` is used for listeners so # check listener type here to ensure shutdown procedure is only applied to # actual `Port` instances. - if listener is Port: - #logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) + if type(listener) is Port: + # logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) # Preferred over connectionLost since it allows buffers to flush listener.unregisterProducer() listener.loseConnection() From 2b38ed02c35d5fe875d649471b704787b4f50ef7 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 14:47:36 -0600 Subject: [PATCH 016/181] Clean up batching queue metrics shutdown --- synapse/crypto/keyring.py | 8 ++++++++ synapse/server.py | 8 +------- synapse/util/batching_queue.py | 9 ++++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 8c59772e56c..81cf93989bc 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -194,6 +194,11 @@ def __init__( valid_until_ts=2**63, # fake future timestamp ) + def shutdown(self) -> None: + self._fetch_keys_queue.shutdown() + for key_fetcher in self._key_fetchers: + key_fetcher.shutdown() + async def verify_json_for_server( self, server_name: str, @@ -484,6 +489,9 @@ def __init__(self, hs: "HomeServer"): process_batch_callback=self._fetch_keys, ) + def shutdown(self) -> None: + self._queue.shutdown() + async def get_keys( self, server_name: str, key_ids: List[str], minimum_valid_until_ts: int ) -> Dict[str, FetchKeyResult]: diff --git a/synapse/server.py b/synapse/server.py index b272e02a8d4..f4ed81a54ec 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -346,14 +346,8 @@ def shutdown(self) -> None: # TODO: Cleanup replication pieces - from synapse.util.batching_queue import number_of_keys + self.get_keyring().shutdown() - number_of_keys.clear() - from synapse.util.batching_queue import number_queued - - number_queued.clear() - - # TODO: unregister all metrics that bind to HS resources! logger.info("Stopping later gauges: %d", len(self._later_gauges)) for later_gauge in self._later_gauges: later_gauge.unregister() diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 470906484ba..30203774d9d 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -123,11 +123,18 @@ def __init__( name=self._name, **{SERVER_NAME_LABEL: self.server_name} ).set_function(lambda: len(self._next_values)) - # TODO: (devon) what do with this global? self._number_in_flight_metric: Gauge = number_in_flight.labels( name=self._name, **{SERVER_NAME_LABEL: self.server_name} ) + def shutdown(self) -> None: + number_queued.labels( + name=self._name, **{SERVER_NAME_LABEL: self.server_name} + ).set_function(lambda: 0) + number_of_keys.labels( + name=self._name, **{SERVER_NAME_LABEL: self.server_name} + ).set_function(lambda: 0) + async def add_to_queue(self, value: V, key: Hashable = ()) -> R: """Adds the value to the queue with the given key, returning the result of the processing function for the batch that included the given value. From a3160908efdd298daa1bb1e4a36670f072e2ce36 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 15:13:59 -0600 Subject: [PATCH 017/181] Add shutdown call to keyring mock --- tests/handlers/test_typing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 614b12c62aa..d48a8294025 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -79,8 +79,9 @@ def make_homeserver( ) -> HomeServer: # we mock out the keyring so as to skip the authentication check on the # federation API call. - mock_keyring = Mock(spec=["verify_json_for_server"]) + mock_keyring = Mock(spec=["verify_json_for_server", "shutdown"]) mock_keyring.verify_json_for_server = AsyncMock(return_value=True) + mock_keyring.shutdown = Mock() # we mock out the federation client too self.mock_federation_client = AsyncMock(spec=["put_json"]) From 667351ac5c64b745c98919a6982318c2e0e59edf Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 16:02:39 -0600 Subject: [PATCH 018/181] Add manhole port to shutdown list --- synapse/app/_base.py | 5 ++--- synapse/app/generic_worker.py | 12 +++++++----- synapse/app/homeserver.py | 12 +++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 43f82cd28c0..8f037bae999 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -318,7 +318,7 @@ def listen_manhole( port: int, manhole_settings: ManholeConfig, manhole_globals: dict, -) -> None: +) -> List[Port]: # twisted.conch.manhole 21.1.0 uses "int_from_bytes", which produces a confusing # warning. It's fixed by https://github.com/twisted/twisted/pull/1522), so # suppress the warning for now. @@ -330,8 +330,7 @@ def listen_manhole( from synapse.util.manhole import manhole - # TODO: return port - listen_tcp( + return listen_tcp( bind_addresses, port, manhole(settings=manhole_settings, globals=manhole_globals), diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 4f5bea6bd67..9b4059402b5 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -277,11 +277,13 @@ def start_listening(self) -> None: self._listen_http(listener) elif listener.type == "manhole": if isinstance(listener, TCPListenerConfig): - _base.listen_manhole( - listener.bind_addresses, - listener.port, - manhole_settings=self.config.server.manhole_settings, - manhole_globals={"hs": self}, + self._listening_services.extend( + _base.listen_manhole( + listener.bind_addresses, + listener.port, + manhole_settings=self.config.server.manhole_settings, + manhole_globals={"hs": self}, + ) ) else: raise ConfigError( diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index ca0dc8d5707..68673f19f7f 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -297,11 +297,13 @@ def start_listening(self) -> None: ) elif listener.type == "manhole": if isinstance(listener, TCPListenerConfig): - _base.listen_manhole( - listener.bind_addresses, - listener.port, - manhole_settings=self.config.server.manhole_settings, - manhole_globals={"hs": self}, + self._listening_services.extend( + _base.listen_manhole( + listener.bind_addresses, + listener.port, + manhole_settings=self.config.server.manhole_settings, + manhole_globals={"hs": self}, + ) ) else: raise ConfigError( From 470978a9a90ea7d8491d68deec199ce22921a992 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 16:07:15 -0600 Subject: [PATCH 019/181] Remove outdated TODO --- synapse/notifier.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index 77209d28259..a4387e585f9 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -253,7 +253,6 @@ def __init__(self, hs: "HomeServer"): self.federation_sender = None if hs.should_send_federation(): - # TODO: (devon) why is this one already a weakref proxy? self.federation_sender = hs.get_federation_sender() self.state_handler = hs.get_state_handler() From 35ee71ad3c61776beb5fe231ade56f7c6d6bc4fe Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 16:08:24 -0600 Subject: [PATCH 020/181] Reintroduce code that no longer blocks clean shutdown --- synapse/app/phone_stats_home.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 4d24b1cfea9..72406cec1fa 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -233,8 +233,7 @@ def performance_stats_init() -> None: hs.get_datastores().main.reap_monthly_active_users, ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, ) - # TODO: (devon) how does this hold onto a DB pool reference? - # hs.register_background_process(hs.get_datastores().main.reap_monthly_active_users()) + hs.register_background_process(hs.get_datastores().main.reap_monthly_active_users()) def generate_monthly_active_users() -> "defer.Deferred[None]": async def _generate_monthly_active_users() -> None: From da4aa351e03114cd9261aa3d70efa0a656f2e148 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 16:58:09 -0600 Subject: [PATCH 021/181] Address more TODO comments --- synapse/server.py | 10 +++------- synapse/util/caches/__init__.py | 4 ++-- synapse/util/metrics.py | 17 ++++++++++------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index f4ed81a54ec..dc6a72765dc 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -167,6 +167,7 @@ from synapse.synapse_rust.rendezvous import RendezvousHandler from synapse.types import DomainSpecificString, ISynapseReactor from synapse.util import Clock +from synapse.util.caches import CACHE_METRIC_REGISTRY from synapse.util.distributor import Distributor from synapse.util.macaroons import MacaroonGenerator from synapse.util.ratelimitutils import FederationRateLimiter @@ -369,21 +370,16 @@ def shutdown(self) -> None: process.cancel() self._background_processes.clear() - from synapse.util.caches import CACHE_METRIC_REGISTRY - logger.info( "Clearing cache metrics: %d", len(CACHE_METRIC_REGISTRY._pre_update_hooks) ) - # TODO: Do this better, ie. don't clear metrics for other tenants - # only clear them for this server - CACHE_METRIC_REGISTRY.clear() + CACHE_METRIC_REGISTRY.clear(self.config.server.server_name) for shutdown_handler in self._async_shutdown_handlers: try: logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - # TODO: we should probably run these - # yield defer.ensureDeferred(shutdown_handler.func()) + defer.ensureDeferred(shutdown_handler.func()) except Exception: pass self._async_shutdown_handlers.clear() diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 710a29e3f0f..c35b2583333 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -243,8 +243,8 @@ def register_cache( server_name=server_name, collect_callback=collect_callback, ) - metric_name = "cache_%s_%s_%s" % (cache_type, cache_name, server_name) - CACHE_METRIC_REGISTRY.register_hook(metric_name, metric.collect) + # metric_name = "cache_%s_%s_%s" % (cache_type, cache_name, server_name) + CACHE_METRIC_REGISTRY.register_hook(server_name, metric.collect) return metric diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 6b89ba20633..534445c45c3 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -27,6 +27,7 @@ Callable, Dict, Generator, + List, Optional, Protocol, Type, @@ -289,24 +290,26 @@ class DynamicCollectorRegistry(CollectorRegistry): def __init__(self) -> None: super().__init__() - self._pre_update_hooks: Dict[str, Callable[[], None]] = {} + self._pre_update_hooks: Dict[str, List[Callable[[], None]]] = {} def collect(self) -> Generator[Metric, None, None]: """ Collects metrics, calling pre-update hooks first. """ - for pre_update_hook in self._pre_update_hooks.values(): - pre_update_hook() + for pre_update_hooks in self._pre_update_hooks.values(): + for pre_update_hook in pre_update_hooks: + pre_update_hook() yield from super().collect() - def register_hook(self, metric_name: str, hook: Callable[[], None]) -> None: + def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: """ Registers a hook that is called before metric collection. """ - self._pre_update_hooks[metric_name] = hook + server_hooks = self._pre_update_hooks.setdefault(server_name, []) + server_hooks.append(hook) - def clear(self) -> None: - self._pre_update_hooks.clear() + def clear(self, server_name: str) -> None: + self._pre_update_hooks.pop(server_name) From ad6f02e6955f26f41fc6ccc774274ceb344798b2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 19 Aug 2025 16:58:30 -0600 Subject: [PATCH 022/181] Reset accidental constant value change --- synapse/storage/databases/main/roommember.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index fce6edad652..3c2f72c14d1 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -113,7 +113,7 @@ def __init__( self._count_known_servers_task: Optional[LoopingCall] = ( self.hs.get_clock().looping_call( self._count_known_servers, - 6 * 1000, + 60 * 1000, ) ) self.hs.get_clock().call_later( From a77e41b98bf7c762c6d3b411bd900fb0812774f3 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 09:43:34 -0600 Subject: [PATCH 023/181] Cleanly shutdown metrics servers --- synapse/app/_base.py | 12 +++++++++--- synapse/app/generic_worker.py | 8 +++++--- synapse/app/homeserver.py | 25 +++++-------------------- synapse/server.py | 30 ++++++++++++++++++++++++++++-- 4 files changed, 47 insertions(+), 28 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 8f037bae999..4b918570913 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -24,6 +24,7 @@ import signal import socket import sys +from threading import Thread import traceback import warnings from textwrap import indent @@ -39,6 +40,7 @@ Tuple, cast, ) +from wsgiref.simple_server import WSGIServer from cryptography.utils import CryptographyDeprecationWarning from typing_extensions import ParamSpec @@ -289,7 +291,7 @@ async def wrapper() -> None: reactor.callWhenRunning(lambda: defer.ensureDeferred(wrapper())) -def listen_metrics(bind_addresses: StrCollection, port: int) -> None: +def listen_metrics(bind_addresses: StrCollection, port: int) -> List[Tuple[WSGIServer, Thread]]: """ Start Prometheus metrics server. @@ -307,10 +309,14 @@ def listen_metrics(bind_addresses: StrCollection, port: int) -> None: from synapse.metrics import RegistryProxy - # TODO: track for shutdown + servers: List[Tuple[WSGIServer, Thread]] = [] for host in bind_addresses: logger.info("Starting metrics listener on %s:%d", host, port) - start_http_server_prometheus(port, addr=host, registry=RegistryProxy) + server, thread = start_http_server_prometheus( + port, addr=host, registry=RegistryProxy + ) + servers.append((server, thread)) + return servers def listen_manhole( diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 9b4059402b5..21dc2381b1f 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -297,9 +297,11 @@ def start_listening(self) -> None: ) else: if isinstance(listener, TCPListenerConfig): - _base.listen_metrics( - listener.bind_addresses, - listener.port, + self._metrics_listeners.extend( + _base.listen_metrics( + listener.bind_addresses, + listener.port, + ) ) else: raise ConfigError( diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 68673f19f7f..b6b20d1990b 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -87,23 +87,6 @@ class SynapseHomeServer(HomeServer): def shutdown(self) -> None: super().shutdown() - logger.info("Shutting down listening services") - for listener in self._listening_services: - # During unit tests, an incomplete `_FakePort` is used for listeners so - # check listener type here to ensure shutdown procedure is only applied to - # actual `Port` instances. - if type(listener) is Port: - # logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) - # Preferred over connectionLost since it allows buffers to flush - listener.unregisterProducer() - listener.loseConnection() - - # NOTE: not guaranteed to immediately shutdown - # Sometimes takes a second for some deferred to fire that cancels the socket - # But seems to always do so within a minute - # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. - self._listening_services.clear() - def _listener_http( self, config: HomeServerConfig, @@ -316,9 +299,11 @@ def start_listening(self) -> None: ) else: if isinstance(listener, TCPListenerConfig): - _base.listen_metrics( - listener.bind_addresses, - listener.port, + self._metrics_listeners.extend( + _base.listen_metrics( + listener.bind_addresses, + listener.port, + ) ) else: raise ConfigError( diff --git a/synapse/server.py b/synapse/server.py index dc6a72765dc..41c506af042 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -28,6 +28,7 @@ import abc import functools import logging +from threading import Thread from typing import ( TYPE_CHECKING, Any, @@ -35,10 +36,12 @@ Dict, List, Optional, + Tuple, Type, TypeVar, cast, ) +from wsgiref.simple_server import WSGIServer from attr import dataclass from typing_extensions import TypeAlias @@ -50,6 +53,7 @@ from twisted.web.iweb import IPolicyForHTTPS from twisted.web.resource import Resource +from synapse.app._base import unregister_sighups from synapse.api.auth import Auth from synapse.api.auth.internal import InternalAuth from synapse.api.auth.mas import MasDelegatedAuth @@ -322,6 +326,7 @@ def __init__( self.signing_key = config.key.signing_key[0] self.config = config self._listening_services: List[Port] = [] + self._metrics_listeners: List[Tuple[WSGIServer, Thread]] = [] self.start_time: Optional[int] = None self._instance_id = random_string(5) @@ -393,13 +398,34 @@ def shutdown(self) -> None: pass self._sync_shutdown_handlers.clear() - from synapse.app._base import unregister_sighups - unregister_sighups(self.config.server.server_name) for call in self.get_reactor().getDelayedCalls(): call.cancel() + logger.info("Shutting down listening services") + for listener in self._listening_services: + # During unit tests, an incomplete `_FakePort` is used for listeners so + # check listener type here to ensure shutdown procedure is only applied to + # actual `Port` instances. + if type(listener) is Port: + # logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) + # Preferred over connectionLost since it allows buffers to flush + listener.unregisterProducer() + listener.loseConnection() + + # NOTE: not guaranteed to immediately shutdown + # Sometimes takes a second for some deferred to fire that cancels the socket + # But seems to always do so within a minute + # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. + self._listening_services.clear() + + logger.info("Shutting down metrics listeners") + for (server, thread) in self._metrics_listeners: + server.shutdown() + thread.join() + self._metrics_listeners.clear() + def register_async_shutdown_handler( self, desc: "LiteralString", shutdown_func: Callable[..., Any] ) -> None: From 44e48f69582122658ecaef9e269559a3ae1d075b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 11:40:07 -0600 Subject: [PATCH 024/181] Fix linter errors --- synapse/app/_base.py | 6 ++++-- synapse/server.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 4b918570913..b9542568f21 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -24,10 +24,10 @@ import signal import socket import sys -from threading import Thread import traceback import warnings from textwrap import indent +from threading import Thread from typing import ( TYPE_CHECKING, Any, @@ -291,7 +291,9 @@ async def wrapper() -> None: reactor.callWhenRunning(lambda: defer.ensureDeferred(wrapper())) -def listen_metrics(bind_addresses: StrCollection, port: int) -> List[Tuple[WSGIServer, Thread]]: +def listen_metrics( + bind_addresses: StrCollection, port: int +) -> List[Tuple[WSGIServer, Thread]]: """ Start Prometheus metrics server. diff --git a/synapse/server.py b/synapse/server.py index 41c506af042..09a4122edcb 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -53,13 +53,13 @@ from twisted.web.iweb import IPolicyForHTTPS from twisted.web.resource import Resource -from synapse.app._base import unregister_sighups from synapse.api.auth import Auth from synapse.api.auth.internal import InternalAuth from synapse.api.auth.mas import MasDelegatedAuth from synapse.api.auth_blocking import AuthBlocking from synapse.api.filtering import Filtering from synapse.api.ratelimiting import Ratelimiter, RequestRatelimiter +from synapse.app._base import unregister_sighups from synapse.appservice.api import ApplicationServiceApi from synapse.appservice.scheduler import ApplicationServiceScheduler from synapse.config.homeserver import HomeServerConfig @@ -421,7 +421,7 @@ def shutdown(self) -> None: self._listening_services.clear() logger.info("Shutting down metrics listeners") - for (server, thread) in self._metrics_listeners: + for server, thread in self._metrics_listeners: server.shutdown() thread.join() self._metrics_listeners.clear() From fa978970e6cd0e51373732ccdcb9949a3ad3cedd Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 11:44:07 -0600 Subject: [PATCH 025/181] Remove shutdown function override --- synapse/app/homeserver.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b6b20d1990b..7cbb8835471 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -84,9 +84,6 @@ def gz_wrap(r: Resource) -> Resource: class SynapseHomeServer(HomeServer): DATASTORE_CLASS = DataStore - def shutdown(self) -> None: - super().shutdown() - def _listener_http( self, config: HomeServerConfig, From 51d0757300747f7e0823c17f1988bb55fd95dbfa Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 11:55:38 -0600 Subject: [PATCH 026/181] Refactor shutdown handler registration function --- synapse/app/_base.py | 1 + synapse/handlers/presence.py | 6 ++++-- synapse/server.py | 10 +++++++--- synapse/storage/databases/main/client_ips.py | 2 ++ synapse/storage/databases/main/lock.py | 4 +++- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index b9542568f21..57d73f230c5 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -718,6 +718,7 @@ def setup_sdnotify(hs: "HomeServer") -> None: # we're not using systemd. sdnotify(b"READY=1\nMAINPID=%i" % (os.getpid(),)) + # TODO: fixme hs.get_reactor().addSystemEventTrigger( "before", "shutdown", sdnotify, b"STOPPING=1" ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index bb49969900c..50da3ca650e 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -528,7 +528,7 @@ def __init__(self, hs: "HomeServer"): self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS_MS) hs.register_async_shutdown_handler( - "generic_presence.on_shutdown", self._on_shutdown + "before", "shutdown", "generic_presence.on_shutdown", self._on_shutdown ) @wrap_as_background_process("WorkerPresenceHandler._on_shutdown") @@ -828,7 +828,9 @@ def __init__(self, hs: "HomeServer"): # have not yet been persisted self.unpersisted_users_changes: Set[str] = set() - hs.register_async_shutdown_handler("presence.on_shutdown", self._on_shutdown) + hs.register_async_shutdown_handler( + "before", "shutdown", "presence.on_shutdown", self._on_shutdown + ) # Keeps track of the number of *ongoing* syncs on this process. While # this is non zero a user will never go offline. diff --git a/synapse/server.py b/synapse/server.py index 09a4122edcb..79e0aec3272 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -427,11 +427,15 @@ def shutdown(self) -> None: self._metrics_listeners.clear() def register_async_shutdown_handler( - self, desc: "LiteralString", shutdown_func: Callable[..., Any] + self, + phase: str, + eventType: str, + desc: "LiteralString", + shutdown_func: Callable[..., Any], ) -> None: id = self.get_reactor().addSystemEventTrigger( - "before", - "shutdown", + phase, + eventType, run_as_background_process, desc, self.config.server.server_name, diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 4a9a57ff455..2d8cf9f98c8 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -454,6 +454,8 @@ def __init__( self._clock.looping_call(self._update_client_ips_batch, 5 * 1000) hs.register_async_shutdown_handler( + "before", + "shutdown", "ClientIpWorkerStore _update_client_ips_batch", self._update_client_ips_batch, ) diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index bfdc6b3068c..2d7bc4ab192 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -99,7 +99,9 @@ def __init__( # lead to a race, as we may drop the lock while we are still processing. # However, a) it should be a small window, b) the lock is best effort # anyway and c) we want to really avoid leaking locks when we restart. - hs.register_async_shutdown_handler("LockStore _on_shutdown", self._on_shutdown) + hs.register_async_shutdown_handler( + "before", "shutdown", "LockStore _on_shutdown", self._on_shutdown + ) self._acquiring_locks: Set[Tuple[str, str]] = set() From 02112fa55c6ae02d71966c510eb6c34e155f0ddc Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 13:44:18 -0600 Subject: [PATCH 027/181] Add args to registered shutdown handlers --- synapse/app/_base.py | 6 ++---- synapse/server.py | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 57d73f230c5..59f3709ce34 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -618,9 +618,8 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: hs.get_pusherpool().start() # Log when we start the shut down process. - # TODO: fixme - hs.get_reactor().addSystemEventTrigger( - "before", "shutdown", logger.info, "Shutting down..." + hs.register_sync_shutdown_handler( + "before", "shutdown", "shutdown log entry", logger.info, "Shutting down..." ) setup_sentry(hs) @@ -718,7 +717,6 @@ def setup_sdnotify(hs: "HomeServer") -> None: # we're not using systemd. sdnotify(b"READY=1\nMAINPID=%i" % (os.getpid(),)) - # TODO: fixme hs.get_reactor().addSystemEventTrigger( "before", "shutdown", sdnotify, b"STOPPING=1" ) diff --git a/synapse/server.py b/synapse/server.py index 79e0aec3272..f934057bb5d 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -262,6 +262,8 @@ class ShutdownInfo: desc: str func: Callable[..., Any] trigger_id: Any + args: Tuple[object, ...] + kwargs: Dict[str, object] class HomeServer(metaclass=abc.ABCMeta): @@ -384,7 +386,7 @@ def shutdown(self) -> None: try: logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - defer.ensureDeferred(shutdown_handler.func()) + defer.ensureDeferred(shutdown_handler.func(*shutdown_handler.args, **shutdown_handler.kwargs)) except Exception: pass self._async_shutdown_handlers.clear() @@ -393,7 +395,7 @@ def shutdown(self) -> None: try: logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - shutdown_handler.func() + shutdown_handler.func(*shutdown_handler.args, **shutdown_handler.kwargs) except Exception: pass self._sync_shutdown_handlers.clear() @@ -432,6 +434,8 @@ def register_async_shutdown_handler( eventType: str, desc: "LiteralString", shutdown_func: Callable[..., Any], + *args: object, + **kwargs: object, ) -> None: id = self.get_reactor().addSystemEventTrigger( phase, @@ -442,7 +446,7 @@ def register_async_shutdown_handler( shutdown_func, ) self._async_shutdown_handlers.append( - ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id) + ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs) ) def register_sync_shutdown_handler( @@ -451,14 +455,18 @@ def register_sync_shutdown_handler( eventType: str, desc: "LiteralString", shutdown_func: Callable[..., Any], + *args: object, + **kwargs: object, ) -> None: id = self.get_reactor().addSystemEventTrigger( phase, eventType, shutdown_func, + args, + kwargs, ) self._sync_shutdown_handlers.append( - ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id) + ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs) ) def register_later_gauge(self, later_gauge: LaterGauge) -> None: From 60c8088404d798c3209fa5f5ae47048cfdc64f94 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 13:45:46 -0600 Subject: [PATCH 028/181] Fix formatting --- synapse/server.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index f934057bb5d..426904258cf 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -386,7 +386,11 @@ def shutdown(self) -> None: try: logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - defer.ensureDeferred(shutdown_handler.func(*shutdown_handler.args, **shutdown_handler.kwargs)) + defer.ensureDeferred( + shutdown_handler.func( + *shutdown_handler.args, **shutdown_handler.kwargs + ) + ) except Exception: pass self._async_shutdown_handlers.clear() @@ -446,7 +450,9 @@ def register_async_shutdown_handler( shutdown_func, ) self._async_shutdown_handlers.append( - ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs) + ShutdownInfo( + desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs + ) ) def register_sync_shutdown_handler( @@ -466,7 +472,9 @@ def register_sync_shutdown_handler( kwargs, ) self._sync_shutdown_handlers.append( - ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs) + ShutdownInfo( + desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs + ) ) def register_later_gauge(self, later_gauge: LaterGauge) -> None: From 42882e7c016231cf6be5296eb2acdf3ea85db509 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 13:53:18 -0600 Subject: [PATCH 029/181] Optionally freeze gc objects at startup --- synapse/app/_base.py | 28 +++++++++++++++------------- synapse/app/homeserver.py | 4 ++-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 59f3709ce34..3d6aadff45c 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -18,6 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # +import atexit import gc import logging import os @@ -515,7 +516,7 @@ def refresh_certificate(hs: "HomeServer") -> None: logger.info("Context factories updated.") -async def start(hs: "HomeServer") -> None: +async def start(hs: "HomeServer", freeze: bool = True) -> None: """ Start a Synapse server or worker. @@ -631,18 +632,19 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: await hs.get_common_usage_metrics_manager().setup() start_phone_stats_home(hs) - # We now freeze all allocated objects in the hopes that (almost) - # everything currently allocated are things that will be used for the - # rest of time. Doing so means less work each GC (hopefully). - # - # PyPy does not (yet?) implement gc.freeze() - # if hasattr(gc, "freeze"): - # gc.collect() - # gc.freeze() - # - # # Speed up shutdowns by freezing all allocated objects. This moves everything - # # into the permanent generation and excludes them from the final GC. - # atexit.register(gc.freeze) + if freeze: + # We now freeze all allocated objects in the hopes that (almost) + # everything currently allocated are things that will be used for the + # rest of time. Doing so means less work each GC (hopefully). + # + # PyPy does not (yet?) implement gc.freeze() + if hasattr(gc, "freeze"): + gc.collect() + gc.freeze() + + # Speed up shutdowns by freezing all allocated objects. This moves everything + # into the permanent generation and excludes them from the final GC. + atexit.register(gc.freeze) def reload_cache_config(config: HomeServerConfig) -> None: diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7cbb8835471..9d24eba258f 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -314,7 +314,7 @@ def start_listening(self) -> None: def setup( - config_options: List[str], reactor: Optional[ISynapseReactor] = None + config_options: List[str], reactor: Optional[ISynapseReactor] = None, freeze: bool = True, ) -> SynapseHomeServer: """ Args: @@ -391,7 +391,7 @@ async def start() -> None: # Loading the provider metadata also ensures the provider config is valid. await oidc.load_metadata() - await _base.start(hs) + await _base.start(hs, freeze) hs.get_datastores().main.db_pool.updates.start_doing_background_updates() From 7804e16c863be02337a59dc2617702be5114553e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 13:54:04 -0600 Subject: [PATCH 030/181] Apply formatting fixes --- synapse/app/_base.py | 2 +- synapse/app/homeserver.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3d6aadff45c..24cbcce23ed 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -641,7 +641,7 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: if hasattr(gc, "freeze"): gc.collect() gc.freeze() - + # Speed up shutdowns by freezing all allocated objects. This moves everything # into the permanent generation and excludes them from the final GC. atexit.register(gc.freeze) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 9d24eba258f..48715f7cfc6 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -314,7 +314,9 @@ def start_listening(self) -> None: def setup( - config_options: List[str], reactor: Optional[ISynapseReactor] = None, freeze: bool = True, + config_options: List[str], + reactor: Optional[ISynapseReactor] = None, + freeze: bool = True, ) -> SynapseHomeServer: """ Args: From 1bd2ad3b47cde5191900ebe79755b5728701a9a6 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 16:37:02 -0600 Subject: [PATCH 031/181] Remove unnecessary cleanup step --- synapse/server.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index 426904258cf..c26b894e54a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -406,9 +406,6 @@ def shutdown(self) -> None: unregister_sighups(self.config.server.server_name) - for call in self.get_reactor().getDelayedCalls(): - call.cancel() - logger.info("Shutting down listening services") for listener in self._listening_services: # During unit tests, an incomplete `_FakePort` is used for listeners so From 25fdddd521edd2d4242ea3730c63a3d49f8bf2ba Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 17:03:44 -0600 Subject: [PATCH 032/181] Revert unnecessary logic changes --- synapse/federation/transport/server/__init__.py | 6 +++--- synapse/handlers/directory.py | 3 +-- synapse/handlers/federation_event.py | 1 - synapse/handlers/presence.py | 1 - synapse/metrics/common_usage_metrics.py | 1 - synapse/replication/http/_base.py | 5 ++--- synapse/replication/tcp/handler.py | 6 +++--- synapse/rest/client/sync.py | 2 +- synapse/storage/databases/main/events.py | 4 ++-- synapse/storage/databases/main/roommember.py | 10 +++------- synapse/util/caches/__init__.py | 1 - synapse/util/caches/expiringcache.py | 4 ---- 12 files changed, 15 insertions(+), 29 deletions(-) diff --git a/synapse/federation/transport/server/__init__.py b/synapse/federation/transport/server/__init__.py index dbdbd90e2bd..174d02ab6bb 100644 --- a/synapse/federation/transport/server/__init__.py +++ b/synapse/federation/transport/server/__init__.py @@ -72,11 +72,11 @@ def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None) self.authenticator = Authenticator(hs) self.ratelimiter = hs.get_federation_ratelimiter() - self.register_servlets(hs) + self.register_servlets() - def register_servlets(self, hs: "HomeServer") -> None: + def register_servlets(self) -> None: register_servlets( - hs=hs, + self.hs, resource=self, ratelimiter=self.ratelimiter, authenticator=self.authenticator, diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 5d4982ea11b..11284ccd0bc 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -47,9 +47,8 @@ class DirectoryHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs self.auth = hs.get_auth() - self.hostname = hs.hostname + self.hs = hs self.state = hs.get_state_handler() self.appservice_handler = hs.get_application_service_handler() self.event_creation_handler = hs.get_event_creation_handler() diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 540f8438700..da9ca566ff6 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -171,7 +171,6 @@ def __init__(self, hs: "HomeServer"): ) self._notifier = hs.get_notifier() - self.hs = hs self._server_name = hs.hostname self._is_mine_id = hs.is_mine_id self._is_mine_server_name = hs.is_mine_server_name diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 50da3ca650e..25bf6c2f3d8 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1822,7 +1822,6 @@ def __init__(self, hs: "HomeServer"): # # AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler self.server_name = hs.hostname - self.hs = hs self.get_presence_handler = hs.get_presence_handler self.get_presence_router = hs.get_presence_router self.server_name = hs.hostname diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index 691c6fc7ad9..cd1c3c86499 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -52,7 +52,6 @@ def __init__(self, hs: "HomeServer") -> None: self.server_name = hs.hostname self._store = hs.get_datastores().main self._clock = hs.get_clock() - self.hs = hs async def get_metrics(self) -> CommonUsageMetrics: """Get the CommonUsageMetrics object. If no collection has happened yet, do it diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 402fd318349..0850a99e0c2 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -206,7 +206,6 @@ def make_client(cls, hs: "HomeServer") -> Callable: parameter to specify which instance to hit (the instance must be in the `instance_map` config). """ - _hs = hs server_name = hs.hostname clock = hs.get_clock() client = hs.get_replication_client() @@ -230,8 +229,8 @@ async def send_request( *, instance_name: str = MAIN_PROCESS_INSTANCE_NAME, **kwargs: Any ) -> Any: # We have to pull these out here to avoid circular dependencies... - streams = _hs.get_replication_command_handler().get_streams_to_replicate() - replication = _hs.get_replication_data_handler() + streams = hs.get_replication_command_handler().get_streams_to_replicate() + replication = hs.get_replication_data_handler() with outgoing_gauge.track_inprogress(): if instance_name == local_instance_name: diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 99805335ffc..35acb96a431 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -421,7 +421,7 @@ def start_replication(self, hs: "HomeServer") -> None: reactor = hs.get_reactor() redis_config = hs.config.redis if redis_config.redis_path is not None: - self._connection = reactor.connectUNIX( + reactor.connectUNIX( redis_config.redis_path, self._factory, timeout=30, @@ -430,7 +430,7 @@ def start_replication(self, hs: "HomeServer") -> None: elif hs.config.redis.redis_use_tls: ssl_context_factory = ClientContextFactory(hs.config.redis) - self._connection = reactor.connectSSL( + reactor.connectSSL( redis_config.redis_host, redis_config.redis_port, self._factory, @@ -439,7 +439,7 @@ def start_replication(self, hs: "HomeServer") -> None: bindAddress=None, ) else: - self._connection = reactor.connectTCP( + reactor.connectTCP( redis_config.redis_host, redis_config.redis_port, self._factory, diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index d7f1fa81695..c9fb9dc4d3a 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -691,7 +691,7 @@ def __init__(self, hs: "HomeServer"): # derived information from rooms (see how `_generate_sync_entry_for_rooms()` # prepares a bunch of data for `_generate_sync_entry_for_device_list()`). self.only_member_events_filter_collection = FilterCollection( - hs, + self.hs, { "room": { # We only care about membership events for the `device_lists`. diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index c6fe8a6901b..2478367f0d6 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -381,7 +381,7 @@ async def _persist_events_and_state_updates( if context.app_service: origin_type = "local" origin_entity = context.app_service.id - elif self.is_mine_id(event.sender): + elif self.hs.is_mine_id(event.sender): origin_type = "local" origin_entity = "*client*" else: @@ -2077,7 +2077,7 @@ def _update_current_state_txn( # Check if any of the remote membership changes requires us to # unsubscribe from their device lists. self.store.handle_potentially_left_users_txn( - txn, {m for m in members_to_cache_bust if not self.is_mine_id(m)} + txn, {m for m in members_to_cache_bust if not self.hs.is_mine_id(m)} ) @classmethod diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 3c2f72c14d1..5dd42ae2e24 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -39,8 +39,6 @@ import attr -from twisted.internet.task import LoopingCall - from synapse.api.constants import EventTypes, Membership from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS @@ -110,11 +108,9 @@ def __init__( and self.hs.config.metrics.metrics_flags.known_servers ): self._known_servers_count = 1 - self._count_known_servers_task: Optional[LoopingCall] = ( - self.hs.get_clock().looping_call( - self._count_known_servers, - 60 * 1000, - ) + self.hs.get_clock().looping_call( + self._count_known_servers, + 60 * 1000, ) self.hs.get_clock().call_later( 1, diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index c35b2583333..a7984981e2f 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -243,7 +243,6 @@ def register_cache( server_name=server_name, collect_callback=collect_callback, ) - # metric_name = "cache_%s_%s_%s" % (cache_type, cache_name, server_name) CACHE_METRIC_REGISTRY.register_hook(server_name, metric.collect) return metric diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index c7476a10c52..283f4908cd2 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -22,7 +22,6 @@ import logging from collections import OrderedDict from typing import ( - TYPE_CHECKING, Any, Generic, Iterable, @@ -42,9 +41,6 @@ from synapse.util import Clock from synapse.util.caches import EvictionReason, register_cache -if TYPE_CHECKING: - pass - logger = logging.getLogger(__name__) From b980858ee18cc729bd85eecef4057ece2c6216fc Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 20 Aug 2025 17:31:35 -0600 Subject: [PATCH 033/181] PR cleanup changes --- synapse/appservice/scheduler.py | 4 --- synapse/crypto/keyring.py | 3 +++ synapse/server.py | 42 ++++++++++++++++++++---------- synapse/storage/database.py | 4 ++- synapse/util/__init__.py | 28 ++++++++++++++++++-- synapse/util/batching_queue.py | 4 +++ tests/appservice/test_scheduler.py | 1 - 7 files changed, 64 insertions(+), 22 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 638ef3789aa..01f77c4cb6d 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -363,7 +363,6 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.store = hs.get_datastores().main self.as_api = hs.get_application_service_api() - self.hs = hs # map from service id to recoverer instance self.recoverers: Dict[str, "_Recoverer"] = {} @@ -447,7 +446,6 @@ def start_recoverer(self, service: ApplicationService) -> None: logger.info("Starting recoverer for AS ID %s", service.id) assert service.id not in self.recoverers recoverer = self.RECOVERER_CLASS( - self.hs, self.server_name, self.clock, self.store, @@ -494,7 +492,6 @@ class _Recoverer: def __init__( self, - hs: "HomeServer", server_name: str, clock: Clock, store: DataStore, @@ -502,7 +499,6 @@ def __init__( service: ApplicationService, callback: Callable[["_Recoverer"], Awaitable[None]], ): - self.hs = hs self.server_name = server_name self.clock = clock self.store = store diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index 81cf93989bc..d7b62f33fb5 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -195,6 +195,9 @@ def __init__( ) def shutdown(self) -> None: + """ + Prepares the KeyRing for garbage collection by shutting down it's queues. + """ self._fetch_keys_queue.shutdown() for key_fetcher in self._key_fetchers: key_fetcher.shutdown() diff --git a/synapse/server.py b/synapse/server.py index c26b894e54a..080402f6d45 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -259,6 +259,8 @@ def _get(self: "HomeServer") -> T: @dataclass class ShutdownInfo: + """Information for callable functions called at time of shutdown.""" + desc: str func: Callable[..., Any] trigger_id: Any @@ -350,41 +352,40 @@ def __init__( self._sync_shutdown_handlers: List[ShutdownInfo] = [] def shutdown(self) -> None: + """ + Cleanly stops all aspects of the HomeServer and removes any references that + have been handed out in order to allow the HomeServer object to be garbage + collected. + + You must ensure the HomeServer object to not be frozen in the garbage collector + in order for it to be cleaned up. By default, Synapse freezes the HomeServer + object in the garbage collector. + """ + logger.info("Received shutdown request") # TODO: Cleanup replication pieces self.get_keyring().shutdown() - logger.info("Stopping later gauges: %d", len(self._later_gauges)) for later_gauge in self._later_gauges: later_gauge.unregister() self._later_gauges.clear() - logger.info("Unregistering db background updates") for db in self.get_datastores().databases: db.stop_background_updates() - logger.info("Stopping looping calls") self.get_clock().cancel_all_looping_calls() - logger.info("Stopping call_later calls") self.get_clock().cancel_all_delayed_calls() - logger.info( - "Stopping background processes: %d", len(self._background_processes) - ) for process in self._background_processes: process.cancel() self._background_processes.clear() - logger.info( - "Clearing cache metrics: %d", len(CACHE_METRIC_REGISTRY._pre_update_hooks) - ) CACHE_METRIC_REGISTRY.clear(self.config.server.server_name) for shutdown_handler in self._async_shutdown_handlers: try: - logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) defer.ensureDeferred( shutdown_handler.func( @@ -397,7 +398,6 @@ def shutdown(self) -> None: for shutdown_handler in self._sync_shutdown_handlers: try: - logger.info("Shutting down %s", shutdown_handler.desc) self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) shutdown_handler.func(*shutdown_handler.args, **shutdown_handler.kwargs) except Exception: @@ -406,7 +406,6 @@ def shutdown(self) -> None: unregister_sighups(self.config.server.server_name) - logger.info("Shutting down listening services") for listener in self._listening_services: # During unit tests, an incomplete `_FakePort` is used for listeners so # check listener type here to ensure shutdown procedure is only applied to @@ -423,7 +422,6 @@ def shutdown(self) -> None: # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. self._listening_services.clear() - logger.info("Shutting down metrics listeners") for server, thread in self._metrics_listeners: server.shutdown() thread.join() @@ -438,6 +436,10 @@ def register_async_shutdown_handler( *args: object, **kwargs: object, ) -> None: + """ + Register a system event trigger with the HomeServer so it can be cleanly + removed when the HomeServer is shutdown. + """ id = self.get_reactor().addSystemEventTrigger( phase, eventType, @@ -461,6 +463,10 @@ def register_sync_shutdown_handler( *args: object, **kwargs: object, ) -> None: + """ + Register a system event trigger with the HomeServer so it can be cleanly + removed when the HomeServer is shutdown. + """ id = self.get_reactor().addSystemEventTrigger( phase, eventType, @@ -475,9 +481,17 @@ def register_sync_shutdown_handler( ) def register_later_gauge(self, later_gauge: LaterGauge) -> None: + """ + Register a LaterGauge specific to this instance with the HomeServer so it + can be cleanly removed when the HomeServer is shutdown. + """ self._later_gauges.append(later_gauge) def register_background_process(self, process: defer.Deferred) -> None: + """ + Register a background process with the HomeServer so it can be cleanly + removed when the HomeServer is shutdown. + """ self._background_processes.append(process) def register_module_web_resource(self, path: str, resource: Resource) -> None: diff --git a/synapse/storage/database.py b/synapse/storage/database.py index d53ecfdba41..03c820f444f 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -653,8 +653,10 @@ def __init__( ) def stop_background_updates(self) -> None: + """ + Stops the database from running any further background updates. + """ self.updates.enabled = False - self.updates._background_update_handlers.clear() def name(self) -> str: "Return the name of this database" diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 659f4405e86..f76fa000418 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -207,6 +207,13 @@ def _looping_call_common( return call def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: + """ + Stop all running looping calls. + + Args: + consumeErrors: Whether to re-raise errors encountered when cancelling the + scheduled call. + """ for call in self._looping_calls: try: call.stop() @@ -243,14 +250,31 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: self._delayed_calls[id] = call return call - def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: + def cancel_call_later( + self, timer: IDelayedCall, consumeErrors: bool = False + ) -> None: + """ + Stop the specified scheduled calls. + + Args: + timer: The scheduled call to stop. + consumeErrors: Whether to re-raise errors encountered when cancelling the + scheduled call. + """ try: timer.cancel() except Exception: - if not ignore_errs: + if not consumeErrors: raise def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: + """ + Stop all scheduled calls. + + Args: + ignore_errs: Whether to re-raise errors encountered when cancelling the + scheduled call. + """ for call in self._delayed_calls.values(): try: call.cancel() diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 30203774d9d..451d4aa0d9a 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -128,6 +128,10 @@ def __init__( ) def shutdown(self) -> None: + """ + Prepares the object for garbage collection by removing any handed out + references. + """ number_queued.labels( name=self._name, **{SERVER_NAME_LABEL: self.server_name} ).set_function(lambda: 0) diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index b345d45a623..6a9096a70c2 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -177,7 +177,6 @@ def setUp(self) -> None: self.service = Mock() self.callback = AsyncMock() self.recoverer = _Recoverer( - hs=self.hs, server_name="test_server", clock=cast(Clock, self.clock), as_api=self.as_api, From 25e199a3454baee812f530744d984c5856e6de6c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 27 Aug 2025 17:01:28 -0600 Subject: [PATCH 034/181] Cleanly shutdown open connections to Synapse --- .../sender/per_destination_queue.py | 11 +- synapse/handlers/worker_lock.py | 18 ++-- synapse/http/client.py | 17 ++- synapse/http/matrixfederationclient.py | 33 ++++-- synapse/http/proxy.py | 2 +- synapse/http/site.py | 100 +++++++++++++++--- synapse/media/_base.py | 3 +- synapse/notifier.py | 2 +- synapse/push/emailpusher.py | 2 +- synapse/push/httppusher.py | 2 +- synapse/replication/tcp/client.py | 2 +- synapse/server.py | 49 +++++---- synapse/util/__init__.py | 5 +- synapse/util/async_helpers.py | 18 ++-- tests/replication/_base.py | 6 +- tests/util/test_async_helpers.py | 25 +++-- 16 files changed, 195 insertions(+), 100 deletions(-) diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 4c844d403a2..e04e1c57e5d 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -94,6 +94,7 @@ def __init__( destination: str, ): self.server_name = hs.hostname + self._hs = hs self._clock = hs.get_clock() self._storage_controllers = hs.get_storage_controllers() self._store = hs.get_datastores().main @@ -311,10 +312,12 @@ def attempt_new_transaction(self) -> None: logger.debug("TX [%s] Starting transaction loop", self._destination) - run_as_background_process( - "federation_transaction_transmission_loop", - self.server_name, - self._transaction_transmission_loop, + self._hs.register_background_process( + run_as_background_process( + "federation_transaction_transmission_loop", + self.server_name, + self._transaction_transmission_loop, + ) ) async def _transaction_transmission_loop(self) -> None: diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 0b375790dd9..61443978d70 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -37,12 +37,12 @@ import attr from twisted.internet import defer -from twisted.internet.interfaces import IReactorTime from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import start_active_span from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.databases.main.lock import Lock, LockStore +from synapse.util import Clock from synapse.util.async_helpers import timeout_deferred from synapse.util.constants import ONE_MINUTE_SECONDS @@ -69,7 +69,7 @@ def __init__(self, hs: "HomeServer") -> None: self.server_name = ( hs.hostname ) # nb must be called this for @wrap_as_background_process - self._reactor = hs.get_reactor() + self._clock = hs.get_clock() self._store = hs.get_datastores().main self._clock = hs.get_clock() self._notifier = hs.get_notifier() @@ -98,7 +98,7 @@ def acquire_lock(self, lock_name: str, lock_key: str) -> "WaitingLock": """ lock = WaitingLock( - reactor=self._reactor, + clock=self._clock, store=self._store, handler=self, lock_name=lock_name, @@ -129,7 +129,7 @@ def acquire_read_write_lock( """ lock = WaitingLock( - reactor=self._reactor, + clock=self._clock, store=self._store, handler=self, lock_name=lock_name, @@ -160,7 +160,7 @@ def acquire_multi_read_write_lock( lock = WaitingMultiLock( lock_names=lock_names, write=write, - reactor=self._reactor, + clock=self._clock, store=self._store, handler=self, ) @@ -207,7 +207,7 @@ async def _cleanup_locks(self) -> None: @attr.s(auto_attribs=True, eq=False) class WaitingLock: - reactor: IReactorTime + clock: Clock store: LockStore handler: WorkerLocksHandler lock_name: str @@ -249,7 +249,7 @@ async def __aenter__(self) -> None: await timeout_deferred( deferred=self.deferred, timeout=self._get_next_retry_interval(), - reactor=self.reactor, + clock=self.clock, ) except Exception: pass @@ -290,7 +290,7 @@ class WaitingMultiLock: write: bool - reactor: IReactorTime + clock: Clock store: LockStore handler: WorkerLocksHandler @@ -326,7 +326,7 @@ async def __aenter__(self) -> None: await timeout_deferred( deferred=self.deferred, timeout=self._get_next_retry_interval(), - reactor=self.reactor, + clock=self.clock, ) except Exception: pass diff --git a/synapse/http/client.py b/synapse/http/client.py index 1f6d4dcd860..ca0f5052247 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -54,7 +54,6 @@ IOpenSSLContextFactory, IReactorCore, IReactorPluggableNameResolver, - IReactorTime, IResolutionReceiver, ITCPTransport, ) @@ -87,7 +86,7 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.metrics import SERVER_NAME_LABEL from synapse.types import ISynapseReactor, StrSequence -from synapse.util import json_decoder +from synapse.util import Clock, json_decoder from synapse.util.async_helpers import timeout_deferred if TYPE_CHECKING: @@ -165,16 +164,14 @@ def _is_ip_blocked( _EPSILON = 0.00000001 -def _make_scheduler( - reactor: IReactorTime, -) -> Callable[[Callable[[], object]], IDelayedCall]: +def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCall]: """Makes a schedular suitable for a Cooperator using the given reactor. (This is effectively just a copy from `twisted.internet.task`) """ def _scheduler(x: Callable[[], object]) -> IDelayedCall: - return reactor.callLater(_EPSILON, x) + return clock.call_later(_EPSILON, x) return _scheduler @@ -367,7 +364,7 @@ def __init__( # We use this for our body producers to ensure that they use the correct # reactor. - self._cooperator = Cooperator(scheduler=_make_scheduler(hs.get_reactor())) + self._cooperator = Cooperator(scheduler=_make_scheduler(hs.get_clock())) async def request( self, @@ -438,7 +435,7 @@ async def request( request_deferred = timeout_deferred( request_deferred, 60, - self.hs.get_reactor(), + self.hs.get_clock(), ) # turn timeouts into RequestTimedOutErrors @@ -763,7 +760,7 @@ async def get_file( d = read_body_with_max_size(response, output_stream, max_size) # Ensure that the body is not read forever. - d = timeout_deferred(d, 30, self.hs.get_reactor()) + d = timeout_deferred(d, 30, self.hs.get_clock()) length = await make_deferred_yieldable(d) except BodyExceededMaxSize: @@ -959,7 +956,7 @@ async def request( request_deferred = timeout_deferred( request_deferred, 60, - self.hs.get_reactor(), + self.hs.get_clock(), ) # turn timeouts into RequestTimedOutErrors diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index afe305ab41d..24e96385916 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -89,7 +89,7 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags from synapse.metrics import SERVER_NAME_LABEL from synapse.types import JsonDict -from synapse.util import json_decoder +from synapse.util import Clock, json_decoder from synapse.util.async_helpers import AwakenableSleeper, Linearizer, timeout_deferred from synapse.util.metrics import Measure from synapse.util.stringutils import parse_and_validate_server_name @@ -270,6 +270,7 @@ def _validate(v: Any) -> bool: async def _handle_response( + clock: Clock, reactor: IReactorTime, timeout_sec: float, request: MatrixFederationRequest, @@ -299,7 +300,7 @@ async def _handle_response( check_content_type_is(response.headers, parser.CONTENT_TYPE) d = read_body_with_max_size(response, parser, max_response_size) - d = timeout_deferred(d, timeout=timeout_sec, reactor=reactor) + d = timeout_deferred(d, timeout=timeout_sec, clock=clock) length = await make_deferred_yieldable(d) @@ -470,9 +471,9 @@ def __init__( self.max_long_retries = hs.config.federation.max_long_retries self.max_short_retries = hs.config.federation.max_short_retries - self._cooperator = Cooperator(scheduler=_make_scheduler(self.reactor)) + self._cooperator = Cooperator(scheduler=_make_scheduler(self.clock)) - self._sleeper = AwakenableSleeper(self.reactor) + self._sleeper = AwakenableSleeper(self.clock) self._simple_http_client = SimpleHttpClient( hs, @@ -735,7 +736,7 @@ async def _send_request( request_deferred = timeout_deferred( request_deferred, timeout=_sec_timeout, - reactor=self.reactor, + clock=self.clock, ) response = await make_deferred_yieldable(request_deferred) @@ -793,7 +794,9 @@ async def _send_request( # Update transactions table? d = treq.content(response) d = timeout_deferred( - d, timeout=_sec_timeout, reactor=self.reactor + d, + timeout=_sec_timeout, + clock=self.clock, ) try: @@ -1074,6 +1077,7 @@ async def put_json( parser = cast(ByteParser[T], JsonParser()) body = await _handle_response( + self.clock, self.reactor, _sec_timeout, request, @@ -1152,7 +1156,13 @@ async def post_json( _sec_timeout = self.default_timeout_seconds body = await _handle_response( - self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser() + self.clock, + self.reactor, + _sec_timeout, + request, + response, + start_ms, + parser=JsonParser(), ) return body @@ -1358,6 +1368,7 @@ async def get_json_with_headers( parser = cast(ByteParser[T], JsonParser()) body = await _handle_response( + self.clock, self.reactor, _sec_timeout, request, @@ -1431,7 +1442,13 @@ async def delete_json( _sec_timeout = self.default_timeout_seconds body = await _handle_response( - self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser() + self.clock, + self.reactor, + _sec_timeout, + request, + response, + start_ms, + parser=JsonParser(), ) return body diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index 9b044f3b0ae..4ae48e56619 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -166,7 +166,7 @@ async def _async_render(self, request: "SynapseRequest") -> Tuple[int, Any]: # so that it has enough time to complete and pass us the data before we give # up. timeout=90, - reactor=self.reactor, + clock=self._clock, ) response = await make_deferred_yieldable(request_deferred) diff --git a/synapse/http/site.py b/synapse/http/site.py index 55088fc190e..4b317749466 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -22,7 +22,7 @@ import logging import time from http import HTTPStatus -from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Generator, List, Optional, Tuple, Union import attr from zope.interface import implementer @@ -30,6 +30,7 @@ from twisted.internet.address import UNIXAddress from twisted.internet.defer import Deferred from twisted.internet.interfaces import IAddress +from twisted.internet.protocol import Protocol from twisted.python.failure import Failure from twisted.web.http import HTTPChannel from twisted.web.resource import IResource, Resource @@ -655,6 +656,59 @@ class _XForwardedForAddress: host: str +class SynapseProtocol(HTTPChannel): + """ + Synapse-specific twisted http Protocol. + + This is a small wrapper around the twisted HTTPChannel so we can track active + connections in order to close any outstanding connections on shutdown. + """ + + def __init__( + self, + factory: "SynapseSite", + server_name: str, + max_request_body_size: int, + request_id_header: Optional[str], + request_class: type, + ): + super().__init__() + self.factory: SynapseSite = factory + self.site = factory + self.server_name = server_name + self.max_request_body_size = max_request_body_size + self.request_id_header = request_id_header + self.request_class = request_class + + def connectionMade(self) -> None: + """ + Add the connection to the factory's connection list when it's established. + """ + super().connectionMade() + self.factory.addConnection(self) + + def connectionLost(self, reason: Failure) -> None: # type: ignore[override] + """ + Remove the connection from the factory's connection list, when it's lost. + """ + super().connectionLost(reason) + self.factory.removeConnection(self) + + def requestFactory(self, http_channel: HTTPChannel, queued: bool) -> SynapseRequest: # type: ignore[override] + """ + Use our own custom SynapseRequest type instead of the regular + twisted.web.server.Request. + """ + return self.request_class( + self, + self.factory, + our_server_name=self.server_name, + max_request_body_size=self.max_request_body_size, + queued=queued, + request_id_header=self.request_id_header, + ) + + class SynapseSite(ProxySite): """ Synapse-specific twisted http Site @@ -705,23 +759,39 @@ def __init__( assert config.http_options is not None proxied = config.http_options.x_forwarded - request_class = XForwardedForRequest if proxied else SynapseRequest - - request_id_header = config.http_options.request_id_header - - def request_factory(channel: HTTPChannel, queued: bool) -> Request: - return request_class( - channel, - self, - our_server_name=self.server_name, - max_request_body_size=max_request_body_size, - queued=queued, - request_id_header=request_id_header, - ) + self.request_class = XForwardedForRequest if proxied else SynapseRequest + + self.request_id_header = config.http_options.request_id_header + self.max_request_body_size = max_request_body_size - self.requestFactory = request_factory # type: ignore self.access_logger = logging.getLogger(logger_name) self.server_version_string = server_version_string.encode("ascii") + self.connections: List[Protocol] = [] + + def buildProtocol(self, addr: IAddress) -> SynapseProtocol: + protocol = SynapseProtocol( + self, + self.server_name, + self.max_request_body_size, + self.request_id_header, + self.request_class, + ) + return protocol + + def addConnection(self, protocol: Protocol) -> None: + self.connections.append(protocol) + + def removeConnection(self, protocol: Protocol) -> None: + if protocol in self.connections: + self.connections.remove(protocol) + + def stopFactory(self) -> None: + super().stopFactory() + + for protocol in self.connections[:]: + if protocol.transport is not None: + protocol.transport.loseConnection() + self.connections.clear() def log(self, request: SynapseRequest) -> None: # type: ignore[override] pass diff --git a/synapse/media/_base.py b/synapse/media/_base.py index 29911dab77e..8c27f78c8a0 100644 --- a/synapse/media/_base.py +++ b/synapse/media/_base.py @@ -704,6 +704,7 @@ class ThreadedFileSender: def __init__(self, hs: "HomeServer") -> None: self.reactor = hs.get_reactor() + self.clock = hs.get_clock() self.thread_pool = hs.get_media_sender_thread_pool() self.file: Optional[BinaryIO] = None @@ -712,7 +713,7 @@ def __init__(self, hs: "HomeServer") -> None: # Signals if the thread should keep reading/sending data. Set means # continue, clear means pause. - self.wakeup_event = DeferredEvent(self.reactor) + self.wakeup_event = DeferredEvent(self.clock) # Signals if the thread should terminate, e.g. because the consumer has # gone away. diff --git a/synapse/notifier.py b/synapse/notifier.py index a4387e585f9..5a55b584d69 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -673,7 +673,7 @@ async def wait_for_events( listener = timeout_deferred( listener, (end_time - now) / 1000.0, - self.hs.get_reactor(), + self.hs.get_clock(), ) log_kv( diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 09ca14584a2..8f5409bcd96 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -228,7 +228,7 @@ async def _unsafe_process(self) -> None: self.timed_call = None if soonest_due_at is not None: - self.timed_call = self.hs.get_reactor().callLater( + self.timed_call = self.hs.get_clock().call_later( self.seconds_until(soonest_due_at), self.on_timer ) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 5946a6e9724..9e91c0e5610 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -336,7 +336,7 @@ async def _unsafe_process(self) -> None: ) else: logger.info("Push failed: delaying for %ds", self.backoff_delay) - self.timed_call = self.hs.get_reactor().callLater( + self.timed_call = self.hs.get_clock().call_later( self.backoff_delay, self.on_timer ) self.backoff_delay = min( diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 9583a76779a..6107be0ca65 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -337,7 +337,7 @@ async def wait_for_stream_position( # to wedge here forever. deferred: "Deferred[None]" = Deferred() deferred = timeout_deferred( - deferred, _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, self._reactor + deferred, _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, self._clock ) waiting_list = self._streams_to_waiters.setdefault( diff --git a/synapse/server.py b/synapse/server.py index 080402f6d45..6494475c73f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -364,6 +364,31 @@ def shutdown(self) -> None: logger.info("Received shutdown request") + for listener in self._listening_services: + # During unit tests, an incomplete `_FakePort` is used for listeners so + # check listener type here to ensure shutdown procedure is only applied to + # actual `Port` instances. + if type(listener) is Port: + port_shutdown = listener.stopListening() + if port_shutdown is not None: + + def on_error(_: T) -> None: + logger.error("Port shutdown with error") + + def on_success(_: T) -> None: + logger.error("Port shutdown successfully") + + port_shutdown.addCallback(on_success) + port_shutdown.addErrback(on_error) + self._listening_services.clear() + + logger.error("Shutting down metrics listeners") + for server, thread in self._metrics_listeners: + logger.error("Shutting down metrics listener") + server.shutdown() + thread.join() + self._metrics_listeners.clear() + # TODO: Cleanup replication pieces self.get_keyring().shutdown() @@ -371,6 +396,9 @@ def shutdown(self) -> None: for later_gauge in self._later_gauges: later_gauge.unregister() self._later_gauges.clear() + # TODO: What about the other gauge types? + # from synapse.metrics import all_gauges + # all_gauges.clear() for db in self.get_datastores().databases: db.stop_background_updates() @@ -406,27 +434,6 @@ def shutdown(self) -> None: unregister_sighups(self.config.server.server_name) - for listener in self._listening_services: - # During unit tests, an incomplete `_FakePort` is used for listeners so - # check listener type here to ensure shutdown procedure is only applied to - # actual `Port` instances. - if type(listener) is Port: - # logger.info("Shutting down %s %d", listener._type, listener._realPortNumber) - # Preferred over connectionLost since it allows buffers to flush - listener.unregisterProducer() - listener.loseConnection() - - # NOTE: not guaranteed to immediately shutdown - # Sometimes takes a second for some deferred to fire that cancels the socket - # But seems to always do so within a minute - # twisted.internet.error.AlreadyCancelled: Tried to cancel an already-cancelled event. - self._listening_services.clear() - - for server, thread in self._metrics_listeners: - server.shutdown() - thread.join() - self._metrics_listeners.clear() - def register_async_shutdown_handler( self, phase: str, diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index f76fa000418..1773a9a9eae 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -241,9 +241,8 @@ def call_later( self._call_id += 1 def wrapped_callback(*args: Any, **kwargs: Any) -> None: - with context.PreserveLoggingContext(): - callback(*args, **kwargs) - self._delayed_calls.pop(id) + callback(*args, **kwargs) + self._delayed_calls.pop(id) with context.PreserveLoggingContext(): call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index e596e1ed209..44696fc50e2 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -760,7 +760,9 @@ async def _ctx_manager() -> AsyncIterator[None]: def timeout_deferred( - deferred: "defer.Deferred[_T]", timeout: float, reactor: IReactorTime + deferred: "defer.Deferred[_T]", + timeout: float, + clock: Clock, ) -> "defer.Deferred[_T]": """The in built twisted `Deferred.addTimeout` fails to time out deferreds that have a canceller that throws exceptions. This method creates a new @@ -802,7 +804,7 @@ def time_it_out() -> None: if not new_d.called: new_d.errback(defer.TimeoutError("Timed out after %gs" % (timeout,))) - delayed_call = reactor.callLater(timeout, time_it_out) + delayed_call = clock.call_later(timeout, time_it_out) def convert_cancelled(value: Failure) -> Failure: # if the original deferred was cancelled, and our timeout has fired, then @@ -944,9 +946,9 @@ class AwakenableSleeper: currently sleeping. """ - def __init__(self, reactor: IReactorTime) -> None: + def __init__(self, clock: Clock) -> None: self._streams: Dict[str, Set[defer.Deferred[None]]] = {} - self._reactor = reactor + self._clock = clock def wake(self, name: str) -> None: """Wake everything related to `name` that is currently sleeping.""" @@ -965,7 +967,7 @@ async def sleep(self, name: str, delay_ms: int) -> None: # Create a deferred that gets called in N seconds sleep_deferred: "defer.Deferred[None]" = defer.Deferred() - call = self._reactor.callLater(delay_ms / 1000, sleep_deferred.callback, None) + call = self._clock.call_later(delay_ms / 1000, sleep_deferred.callback, None) # Create a deferred that will get called if `wake` is called with # the same `name`. @@ -999,8 +1001,8 @@ async def sleep(self, name: str, delay_ms: int) -> None: class DeferredEvent: """Like threading.Event but for async code""" - def __init__(self, reactor: IReactorTime) -> None: - self._reactor = reactor + def __init__(self, clock: Clock) -> None: + self._clock = clock self._deferred: "defer.Deferred[None]" = defer.Deferred() def set(self) -> None: @@ -1020,7 +1022,7 @@ async def wait(self, timeout_seconds: float) -> bool: # Create a deferred that gets called in N seconds sleep_deferred: "defer.Deferred[None]" = defer.Deferred() - call = self._reactor.callLater(timeout_seconds, sleep_deferred.callback, None) + call = self._clock.call_later(timeout_seconds, sleep_deferred.callback, None) try: await make_deferred_yieldable( diff --git a/tests/replication/_base.py b/tests/replication/_base.py index 453eb7750b3..0689fa404a6 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -174,7 +174,7 @@ def handle_http_replication_attempt(self) -> SynapseRequest: # Set up the server side protocol server_address = IPv4Address("TCP", host, port) - channel = self.site.buildProtocol((host, port)) + channel = self.site.buildProtocol((host, port)) # type: ignore[arg-type] # hook into the channel's request factory so that we can keep a record # of the requests @@ -186,7 +186,7 @@ def request_factory(*args: Any, **kwargs: Any) -> SynapseRequest: requests.append(request) return request - channel.requestFactory = request_factory + channel.requestFactory = request_factory # type: ignore[method-assign] # Connect client to server and vice versa. client_to_server_transport = FakeTransport( @@ -428,7 +428,7 @@ def _handle_http_replication_attempt(self, hs: HomeServer, repl_port: int) -> No # Set up the server side protocol server_address = IPv4Address("TCP", host, port) - channel = self._hs_to_site[hs].buildProtocol((host, port)) + channel = self._hs_to_site[hs].buildProtocol((host, port)) # type: ignore[arg-type] # Connect client to server and vice versa. client_to_server_transport = FakeTransport( diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index cfd2882410e..98184f795a5 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -24,7 +24,6 @@ from twisted.internet import defer from twisted.internet.defer import CancelledError, Deferred, ensureDeferred -from twisted.internet.task import Clock from twisted.python.failure import Failure from synapse.logging.context import ( @@ -149,7 +148,7 @@ def test_cancellation(self) -> None: class TimeoutDeferredTest(TestCase): def setUp(self) -> None: - self.clock = Clock() + self.reactor, self.clock = get_clock() def test_times_out(self) -> None: """Basic test case that checks that the original deferred is cancelled and that @@ -167,7 +166,7 @@ def canceller(_d: Deferred) -> None: self.assertNoResult(timing_out_d) self.assertFalse(cancelled, "deferred was cancelled prematurely") - self.clock.pump((1.0,)) + self.reactor.pump((1.0,)) self.assertTrue(cancelled, "deferred was not cancelled by timeout") self.failureResultOf(timing_out_d, defer.TimeoutError) @@ -184,7 +183,7 @@ def canceller(_d: Deferred) -> None: self.assertNoResult(timing_out_d) - self.clock.pump((1.0,)) + self.reactor.pump((1.0,)) self.failureResultOf(timing_out_d, defer.TimeoutError) @@ -221,7 +220,7 @@ def errback(res: Failure, deferred_name: str) -> Failure: self.assertIs(current_context(), SENTINEL_CONTEXT) timing_out_d.addErrback(errback, "timingout") - self.clock.pump((1.0,)) + self.reactor.pump((2.0,)) self.assertTrue( blocking_was_cancelled, "non-completing deferred was not cancelled" @@ -526,8 +525,8 @@ class AwakenableSleeperTests(TestCase): "Tests AwakenableSleeper" def test_sleep(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d = defer.ensureDeferred(sleeper.sleep("name", 1000)) @@ -541,8 +540,8 @@ def test_sleep(self) -> None: self.assertTrue(d.called) def test_explicit_wake(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d = defer.ensureDeferred(sleeper.sleep("name", 1000)) @@ -558,8 +557,8 @@ def test_explicit_wake(self) -> None: reactor.advance(0.6) def test_multiple_sleepers_timeout(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d1 = defer.ensureDeferred(sleeper.sleep("name", 1000)) @@ -578,8 +577,8 @@ def test_multiple_sleepers_timeout(self) -> None: self.assertTrue(d2.called) def test_multiple_sleepers_wake(self) -> None: - reactor, _ = get_clock() - sleeper = AwakenableSleeper(reactor) + reactor, clock = get_clock() + sleeper = AwakenableSleeper(clock) d1 = defer.ensureDeferred(sleeper.sleep("name", 1000)) From 9607670aa631c1b46a788698f3b3449ae6e26c4b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 15:04:35 -0600 Subject: [PATCH 035/181] Update comment to reflect new variable --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 24cbcce23ed..9029f07a024 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -100,7 +100,7 @@ logger = logging.getLogger(__name__) -# list of tuples of function, args list, kwargs dict +# dict of server_name to tuples of function, args list, kwargs dict _sighup_callbacks: Dict[ str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] ] = {} From e4233b92131140b93c6d498e043eb135c5722674 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 15:08:49 -0600 Subject: [PATCH 036/181] Use instance_id instead of server_name --- synapse/app/_base.py | 12 ++++++------ synapse/config/logger.py | 2 +- synapse/server.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 9029f07a024..689df0f44d6 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -108,7 +108,7 @@ def register_sighup( - server_name: str, func: Callable[P, None], *args: P.args, **kwargs: P.kwargs + instance_id: str, func: Callable[P, None], *args: P.args, **kwargs: P.kwargs ) -> None: """ Register a function to be called when a SIGHUP occurs. @@ -118,11 +118,11 @@ def register_sighup( *args, **kwargs: args and kwargs to be passed to the target function. """ - _sighup_callbacks.setdefault(server_name, []).append((func, args, kwargs)) + _sighup_callbacks.setdefault(instance_id, []).append((func, args, kwargs)) -def unregister_sighups(server_name: str) -> None: - _sighup_callbacks.pop(server_name, []) +def unregister_sighups(instance_id: str) -> None: + _sighup_callbacks.pop(instance_id, []) def start_worker_reactor( @@ -581,8 +581,8 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: signal.signal(signal.SIGHUP, run_sighup) - register_sighup(hs.config.server.server_name, refresh_certificate, hs) - register_sighup(hs.config.server.server_name, reload_cache_config, hs.config) + register_sighup(hs.get_instance_id(), refresh_certificate, hs) + register_sighup(hs.get_instance_id(), reload_cache_config, hs.config) # Apply the cache config. hs.config.caches.resize_all_caches() diff --git a/synapse/config/logger.py b/synapse/config/logger.py index 4bd46aa01f2..c315a3ccde9 100644 --- a/synapse/config/logger.py +++ b/synapse/config/logger.py @@ -349,7 +349,7 @@ def setup_logging( from synapse.app import _base as appbase appbase.register_sighup( - config.server.server_name, _reload_logging_config, log_config_path + hs.get_instance_id(), _reload_logging_config, log_config_path ) # Log immediately so we can grep backwards. diff --git a/synapse/server.py b/synapse/server.py index 6494475c73f..ee79a15a5b4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -432,7 +432,7 @@ def on_success(_: T) -> None: pass self._sync_shutdown_handlers.clear() - unregister_sighups(self.config.server.server_name) + unregister_sighups(self._instance_id) def register_async_shutdown_handler( self, From 4759c558ed5e13e946ae4abea9a14f9b9d72d388 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 15:09:22 -0600 Subject: [PATCH 037/181] Update comment again --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 689df0f44d6..c87c1ca43f8 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -100,7 +100,7 @@ logger = logging.getLogger(__name__) -# dict of server_name to tuples of function, args list, kwargs dict +# dict of instance_id to tuples of function, args list, kwargs dict _sighup_callbacks: Dict[ str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] ] = {} From 947cac97c6e62b26f360a19b1b813b25a6890b8d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 15:37:28 -0600 Subject: [PATCH 038/181] Update docstring for setup function --- synapse/app/homeserver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 48715f7cfc6..7316f1ea846 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -321,6 +321,8 @@ def setup( """ Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. + reactor: Optionally provide a reactor to use. + freeze: Whether to freeze all objects in the garbage collector. Returns: A homeserver instance. From 849a093cfa481193de4bb948c22eb842b05b6bc5 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 17:40:52 -0600 Subject: [PATCH 039/181] Move server shutdown in tests --- tests/server.py | 3 +++ tests/unittest.py | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/server.py b/tests/server.py index 3a81a4c6d9b..81e09a4a002 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1145,6 +1145,9 @@ def setup_test_homeserver( reactor=reactor, ) + # Register the cleanup hook + cleanup_func(hs.shutdown) + # Install @cache_in_self attributes for key, val in kwargs.items(): setattr(hs, "_" + key, val) diff --git a/tests/unittest.py b/tests/unittest.py index a5f1aea3aa9..5e6957dc6d2 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -473,7 +473,6 @@ async def get_requester(*args: Any, **kwargs: Any) -> Requester: def tearDown(self) -> None: # Reset to not use frozen dicts. events.USE_FROZEN_DICTS = False - self.hs.shutdown() def wait_on_thread(self, deferred: Deferred, timeout: int = 10) -> None: """ From 4b67de55cd1c14982eddb3381bc3af9e07180b28 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 17:59:09 -0600 Subject: [PATCH 040/181] Assert metrics aren't being clobbered --- synapse/push/bulk_push_rule_evaluator.py | 2 +- synapse/server.py | 2 +- synapse/util/metrics.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index bb9d5dbcaab..eae069f817b 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -150,7 +150,7 @@ def __init__(self, hs: "HomeServer"): cache_name="room_push_rule_cache", cache=[], # Meaningless size, as this isn't a cache that stores values, resizable=False, - server_name=self.server_name, + server_name=self.hs.get_instance_id(), ) async def _get_rules_for_event( diff --git a/synapse/server.py b/synapse/server.py index ee79a15a5b4..c8aff9e18f6 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -410,7 +410,7 @@ def on_success(_: T) -> None: process.cancel() self._background_processes.clear() - CACHE_METRIC_REGISTRY.clear(self.config.server.server_name) + CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver_instance_id(self.config.server.server_name) for shutdown_handler in self._async_shutdown_handlers: try: diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 534445c45c3..766a9049041 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -308,8 +308,9 @@ def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: Registers a hook that is called before metric collection. """ + assert self._pre_update_hooks.get(server_name) is None server_hooks = self._pre_update_hooks.setdefault(server_name, []) server_hooks.append(hook) - def clear(self, server_name: str) -> None: + def unregister_hooks_for_homeserver_instance_id(self, server_name: str) -> None: self._pre_update_hooks.pop(server_name) From e99f9cdfc7e21e3977dad44c9663e9a62345329d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 18:05:30 -0600 Subject: [PATCH 041/181] Don't renam arg --- synapse/util/__init__.py | 4 ++-- synapse/util/metrics.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 1773a9a9eae..16d6d58a912 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -250,7 +250,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: return call def cancel_call_later( - self, timer: IDelayedCall, consumeErrors: bool = False + self, timer: IDelayedCall, ignore_errs: bool = False ) -> None: """ Stop the specified scheduled calls. @@ -263,7 +263,7 @@ def cancel_call_later( try: timer.cancel() except Exception: - if not consumeErrors: + if not ignore_errs: raise def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 766a9049041..c24a956005f 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -308,7 +308,6 @@ def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: Registers a hook that is called before metric collection. """ - assert self._pre_update_hooks.get(server_name) is None server_hooks = self._pre_update_hooks.setdefault(server_name, []) server_hooks.append(hook) From 0f26d201898d533a08ec1c23c453eee140310626 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 18:15:14 -0600 Subject: [PATCH 042/181] Use gauge.remove --- synapse/server.py | 4 +++- synapse/util/__init__.py | 4 +--- synapse/util/batching_queue.py | 8 ++------ 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index c8aff9e18f6..ad90466d39c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -410,7 +410,9 @@ def on_success(_: T) -> None: process.cancel() self._background_processes.clear() - CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver_instance_id(self.config.server.server_name) + CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver_instance_id( + self.config.server.server_name + ) for shutdown_handler in self._async_shutdown_handlers: try: diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 16d6d58a912..0183ab4cfea 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -249,9 +249,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: self._delayed_calls[id] = call return call - def cancel_call_later( - self, timer: IDelayedCall, ignore_errs: bool = False - ) -> None: + def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: """ Stop the specified scheduled calls. diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index 451d4aa0d9a..97ea5bb3cb6 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -132,12 +132,8 @@ def shutdown(self) -> None: Prepares the object for garbage collection by removing any handed out references. """ - number_queued.labels( - name=self._name, **{SERVER_NAME_LABEL: self.server_name} - ).set_function(lambda: 0) - number_of_keys.labels( - name=self._name, **{SERVER_NAME_LABEL: self.server_name} - ).set_function(lambda: 0) + number_queued.remove(self._name, self.server_name) + number_of_keys.remove(self._name, self.server_name) async def add_to_queue(self, value: V, key: Hashable = ()) -> R: """Adds the value to the queue with the given key, returning the result From 649c18290328eeee9070def92222d383879ac2fc Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 18:16:42 -0600 Subject: [PATCH 043/181] Rename var for clarity --- synapse/util/metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index c24a956005f..e07fe84eddd 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -290,14 +290,14 @@ class DynamicCollectorRegistry(CollectorRegistry): def __init__(self) -> None: super().__init__() - self._pre_update_hooks: Dict[str, List[Callable[[], None]]] = {} + self._server_id_to_pre_update_hooks: Dict[str, List[Callable[[], None]]] = {} def collect(self) -> Generator[Metric, None, None]: """ Collects metrics, calling pre-update hooks first. """ - for pre_update_hooks in self._pre_update_hooks.values(): + for pre_update_hooks in self._server_id_to_pre_update_hooks.values(): for pre_update_hook in pre_update_hooks: pre_update_hook() @@ -308,8 +308,8 @@ def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: Registers a hook that is called before metric collection. """ - server_hooks = self._pre_update_hooks.setdefault(server_name, []) + server_hooks = self._server_id_to_pre_update_hooks.setdefault(server_name, []) server_hooks.append(hook) def unregister_hooks_for_homeserver_instance_id(self, server_name: str) -> None: - self._pre_update_hooks.pop(server_name) + self._server_id_to_pre_update_hooks.pop(server_name) From 93a218323bcd9b0e19d17ca2c25b6a526e346ef5 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 18:24:01 -0600 Subject: [PATCH 044/181] Add field descriptions to docs --- synapse/server.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index ad90466d39c..1a90ec8bbde 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -259,7 +259,15 @@ def _get(self: "HomeServer") -> T: @dataclass class ShutdownInfo: - """Information for callable functions called at time of shutdown.""" + """Information for callable functions called at time of shutdown. + + Attributes: + desc: a short description of the event trigger. + func: the object to call before shutdown. + trigger_id: an ID returned when registering this event trigger. + args: the arguments to call the function with. + kwargs: the keyword arguments to call the function with. + """ desc: str func: Callable[..., Any] From 4e7ddb34c78486b494a7328e3d34817c895160b6 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 29 Aug 2025 18:28:18 -0600 Subject: [PATCH 045/181] Tighten type of event trigger id --- synapse/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index 1a90ec8bbde..785ac7c35fd 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -47,6 +47,7 @@ from typing_extensions import TypeAlias from twisted.internet import defer +from twisted.internet.base import _SystemEventID from twisted.internet.interfaces import IOpenSSLContextFactory from twisted.internet.tcp import Port from twisted.python.threadpool import ThreadPool @@ -271,7 +272,7 @@ class ShutdownInfo: desc: str func: Callable[..., Any] - trigger_id: Any + trigger_id: _SystemEventID args: Tuple[object, ...] kwargs: Dict[str, object] From cc598a6e0aea2e4652d3f7cd71757e06f481767f Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 3 Sep 2025 14:30:07 -0600 Subject: [PATCH 046/181] Fix linter error --- synapse/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index 86e6ae5fe56..af74d3f914a 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -427,7 +427,7 @@ def on_success(_: T) -> None: for process in self._background_processes: try: process.cancel() - except: + except Exception: pass self._background_processes.clear() From b697f9a86407c435185c29e97f216b2825c8bda0 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 11:59:57 -0600 Subject: [PATCH 047/181] Remove background process registration --- synapse/app/phone_stats_home.py | 2 +- synapse/federation/send_queue.py | 3 +++ synapse/federation/sender/__init__.py | 9 +++++++++ .../federation/sender/per_destination_queue.py | 18 ++++++++++-------- synapse/server.py | 18 +++--------------- synapse/storage/background_updates.py | 12 +++++------- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 72406cec1fa..69d3ac78fd4 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -233,7 +233,7 @@ def performance_stats_init() -> None: hs.get_datastores().main.reap_monthly_active_users, ONE_HOUR_SECONDS * MILLISECONDS_PER_SECOND, ) - hs.register_background_process(hs.get_datastores().main.reap_monthly_active_users()) + hs.get_datastores().main.reap_monthly_active_users() def generate_monthly_active_users() -> "defer.Deferred[None]": async def _generate_monthly_active_users() -> None: diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 2fdee9ac549..2b751afc14f 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -144,6 +144,9 @@ def register(queue_name: QueueNames, queue: Sized) -> None: self.clock.looping_call(self._clear_queue, 30 * 1000) + def shutdown(self) -> None: + pass + def _next_pos(self) -> int: pos = self.pos self.pos += 1 diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 278a9573310..a03b49c77f8 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -231,6 +231,11 @@ class AbstractFederationSender(metaclass=abc.ABCMeta): + @abc.abstractmethod + def shutdown(self) -> None: + """Stops this federation sender instance from sending further transactions.""" + raise NotImplementedError() + @abc.abstractmethod def notify_new_events(self, max_token: RoomStreamToken) -> None: """This gets called when we have some new events we might want to @@ -464,6 +469,10 @@ def __init__(self, hs: "HomeServer"): self._wake_destinations_needing_catchup, ) + def shutdown(self) -> None: + for queue in self._per_destination_queues.values(): + queue.disable() + def _get_per_destination_queue( self, destination: str ) -> Optional[PerDestinationQueue]: diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index e04e1c57e5d..5c18bfe9c55 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -118,6 +118,7 @@ def __init__( self._destination = destination self.transmission_loop_running = False + self._transmission_loop_enabled = True # Flag to signal to any running transmission loop that there is new data # queued up to be sent. @@ -172,6 +173,9 @@ def __init__( def __str__(self) -> str: return "PerDestinationQueue[%s]" % self._destination + def disable(self) -> None: + self._transmission_loop_enabled = False + def pending_pdu_count(self) -> int: return len(self._pending_pdus) @@ -312,12 +316,10 @@ def attempt_new_transaction(self) -> None: logger.debug("TX [%s] Starting transaction loop", self._destination) - self._hs.register_background_process( - run_as_background_process( - "federation_transaction_transmission_loop", - self.server_name, - self._transaction_transmission_loop, - ) + run_as_background_process( + "federation_transaction_transmission_loop", + self.server_name, + self._transaction_transmission_loop, ) async def _transaction_transmission_loop(self) -> None: @@ -342,7 +344,7 @@ async def _transaction_transmission_loop(self) -> None: # not caught up yet return - while True: + while self._transmission_loop_enabled: self._new_data_to_send = False async with _TransactionQueueManager(self) as ( @@ -472,7 +474,7 @@ async def _catch_up_transmission_loop(self) -> None: last_successful_stream_ordering: int = _tmp_last_successful_stream_ordering # get at most 50 catchup room/PDUs - while True: + while self._transmission_loop_enabled: event_ids = await self._store.get_catch_up_room_event_ids( self._destination, last_successful_stream_ordering ) diff --git a/synapse/server.py b/synapse/server.py index af74d3f914a..445428f9d06 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -358,7 +358,6 @@ def __init__( # This attribute is set by the free function `refresh_certificate`. self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None - self._background_processes: List[defer.Deferred] = [] self._async_shutdown_handlers: List[ShutdownInfo] = [] self._sync_shutdown_handlers: List[ShutdownInfo] = [] @@ -421,16 +420,12 @@ def on_success(_: T) -> None: for db in self.get_datastores().databases: db.stop_background_updates() + if self.should_send_federation(): + self.get_federation_sender().shutdown() + self.get_clock().cancel_all_looping_calls() self.get_clock().cancel_all_delayed_calls() - for process in self._background_processes: - try: - process.cancel() - except Exception: - pass - self._background_processes.clear() - for shutdown_handler in self._async_shutdown_handlers: try: self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) @@ -506,13 +501,6 @@ def register_sync_shutdown_handler( ) ) - def register_background_process(self, process: defer.Deferred) -> None: - """ - Register a background process with the HomeServer so it can be cleanly - removed when the HomeServer is shutdown. - """ - self._background_processes.append(process) - def register_module_web_resource(self, path: str, resource: Resource) -> None: """Allows a module to register a web resource to be served at the given path. diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 12ad973f982..acc0abee63e 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -395,13 +395,11 @@ def start_doing_background_updates(self) -> None: # if we start a new background update, not all updates are done. self._all_done = False sleep = self.sleep_enabled - self.hs.register_background_process( - run_as_background_process( - "background_updates", - self.server_name, - self.run_background_updates, - sleep, - ) + run_as_background_process( + "background_updates", + self.server_name, + self.run_background_updates, + sleep, ) async def run_background_updates(self, sleep: bool) -> None: From 2619070c144ec2f02767be9350907faf67c2c91f Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 12:48:05 -0600 Subject: [PATCH 048/181] Make clock a required arg for Linearizer --- synapse/federation/federation_server.py | 2 +- synapse/handlers/appservice.py | 3 ++- synapse/handlers/device.py | 6 ++++-- synapse/handlers/e2e_keys.py | 4 ++-- synapse/handlers/federation.py | 5 +++-- synapse/handlers/federation_event.py | 2 +- synapse/handlers/message.py | 4 +++- synapse/handlers/presence.py | 3 ++- synapse/handlers/read_marker.py | 4 +--- synapse/handlers/room_member.py | 7 +++++-- synapse/handlers/sso.py | 2 +- synapse/http/matrixfederationclient.py | 2 +- synapse/media/media_repository.py | 3 ++- synapse/replication/tcp/client.py | 3 ++- synapse/rest/client/push_rule.py | 2 +- synapse/state/__init__.py | 3 ++- synapse/storage/controllers/state.py | 3 ++- synapse/util/async_helpers.py | 8 +------- tests/server.py | 9 +++++++++ tests/util/test_linearizer.py | 21 +++++++++++++-------- 20 files changed, 58 insertions(+), 38 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index d35d62766ba..27a3a664bd1 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -159,7 +159,7 @@ def __init__(self, hs: "HomeServer"): # with FederationHandlerRegistry. hs.get_directory_handler() - self._server_linearizer = Linearizer("fed_server", clock=hs.get_clock()) + self._server_linearizer = Linearizer(hs.get_clock(), "fed_server") # origins that we are currently processing a transaction from. # a dict from origin to txn id. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index bf36cf39a19..00fe763e708 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -98,7 +98,8 @@ def __init__(self, hs: "HomeServer"): self.is_processing = False self._ephemeral_events_linearizer = Linearizer( - name="appservice_ephemeral_events", clock=hs.get_clock() + hs.get_clock(), + name="appservice_ephemeral_events", ) def notify_interested_services(self, max_token: RoomStreamToken) -> None: diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 90a6c17cc5b..187e5e59eb2 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1451,10 +1451,12 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): self.device_handler = device_handler self._remote_edu_linearizer = Linearizer( - name="remote_device_list", clock=hs.get_clock() + hs.get_clock(), + name="remote_device_list", ) self._resync_linearizer = Linearizer( - name="remote_device_resync", clock=hs.get_clock() + hs.get_clock(), + name="remote_device_resync", ) # user_id -> list of updates waiting to be handled. diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index ac3348dbc1e..835bdc8953d 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -112,9 +112,9 @@ def __init__(self, hs: "HomeServer"): # Limit the number of in-flight requests from a single device. self._query_devices_linearizer = Linearizer( + hs.get_clock(), name="query_devices", max_count=10, - clock=hs.get_clock(), ) self._query_appservices_for_otks = ( @@ -1767,7 +1767,7 @@ def __init__(self, hs: "HomeServer"): self._device_handler = device_handler self._remote_edu_linearizer = Linearizer( - name="remote_signing_key", clock=hs.get_clock() + hs.get_clock(), name="remote_signing_key" ) # user_id -> list of updates waiting to be handled. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 18f078df6f4..9185d436c8f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -160,7 +160,7 @@ def __init__(self, hs: "HomeServer"): self._notifier = hs.get_notifier() self._worker_locks = hs.get_worker_locks_handler() - self._room_backfill = Linearizer("room_backfill", clock=hs.get_clock()) + self._room_backfill = Linearizer(hs.get_clock(), "room_backfill") self._third_party_event_rules = ( hs.get_module_api_callbacks().third_party_event_rules @@ -180,7 +180,8 @@ def __init__(self, hs: "HomeServer"): # When the lock is held for a given room, no other concurrent code may # partial state or un-partial state the room. self._is_partial_state_room_linearizer = Linearizer( - name="_is_partial_state_room_linearizer", clock=hs.get_clock() + hs.get_clock(), + name="_is_partial_state_room_linearizer", ) # if this is the main process, fire off a background process to resume diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 45de6d10fe4..486736c8f88 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -192,7 +192,7 @@ def __init__(self, hs: "HomeServer"): # federation event staging area. self.room_queues: Dict[str, List[Tuple[EventBase, str]]] = {} - self._room_pdu_linearizer = Linearizer("fed_room_pdu", clock=hs.get_clock()) + self._room_pdu_linearizer = Linearizer(hs.get_clock(), "fed_room_pdu") async def on_receive_pdu(self, origin: str, pdu: EventBase) -> None: """Process a PDU received via a federation /send/ transaction diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 28f0d2f6140..3469dfb1a1f 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -513,7 +513,9 @@ def __init__(self, hs: "HomeServer"): # We limit concurrent event creation for a room to 1. This prevents state resolution # from occurring when sending bursts of events to a local room self.limiter = Linearizer( - max_count=1, name="room_event_creation_limit", clock=hs.get_clock() + hs.get_clock(), + max_count=1, + name="room_event_creation_limit", ) self._bulk_push_rule_evaluator = hs.get_bulk_push_rule_evaluator() diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 0d9ce028f0a..974050a0e5b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -862,7 +862,8 @@ def __init__(self, hs: "HomeServer"): self.external_process_last_updated_ms: Dict[str, int] = {} self.external_sync_linearizer = Linearizer( - name="external_sync_linearizer", clock=hs.get_clock() + hs.get_clock(), + name="external_sync_linearizer", ) if self._track_presence: diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 85d2dd62bb7..6a311055dc2 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -36,9 +36,7 @@ class ReadMarkerHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.account_data_handler = hs.get_account_data_handler() - self.read_marker_linearizer = Linearizer( - name="read_marker", clock=hs.get_clock() - ) + self.read_marker_linearizer = Linearizer(hs.get_clock(), name="read_marker") async def received_client_read_marker( self, room_id: str, user_id: str, event_id: str diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 623823acb02..a7e29ebc90d 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -115,10 +115,13 @@ def __init__(self, hs: "HomeServer"): self._membership_types_to_include_profile_data_in.add(Membership.INVITE) self.member_linearizer: Linearizer = Linearizer( - name="member", clock=hs.get_clock() + hs.get_clock(), + name="member", ) self.member_as_limiter = Linearizer( - max_count=10, name="member_as_limiter", clock=hs.get_clock() + hs.get_clock(), + max_count=10, + name="member_as_limiter", ) self.clock = hs.get_clock() diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index eec420cbb17..27fe035ebf0 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -224,7 +224,7 @@ def __init__(self, hs: "HomeServer"): ) # a lock on the mappings - self._mapping_lock = Linearizer(name="sso_user_mapping", clock=hs.get_clock()) + self._mapping_lock = Linearizer(hs.get_clock(), name="sso_user_mapping") # a map from session id to session data self._username_mapping_sessions: Dict[str, UsernameMappingSession] = {} diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 24e96385916..ce450af55f8 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -483,7 +483,7 @@ def __init__( ) self.remote_download_linearizer = Linearizer( - "remote_download_linearizer", 6, clock=hs.get_clock() + hs.get_clock(), "remote_download_linearizer", 6 ) def wake_destination(self, destination: str) -> None: diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 6ccf8ccb284..a7f17a29d1c 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -109,7 +109,8 @@ def __init__(self, hs: "HomeServer"): self.thumbnail_requirements = hs.config.media.thumbnail_requirements self.remote_media_linearizer = Linearizer( - name="media_remote", clock=hs.get_clock() + hs.get_clock(), + name="media_remote", ) self.recently_accessed_remotes: Set[Tuple[str, str]] = set() diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 6107be0ca65..46998db6cae 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -430,7 +430,8 @@ def __init__(self, hs: "HomeServer"): self.federation_position: Optional[int] = None self._fed_position_linearizer = Linearizer( - name="_fed_position_linearizer", clock=hs.get_clock() + hs.get_clock(), + name="_fed_position_linearizer", ) async def process_replication_rows( diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index 58cbb8a5869..e1698616e74 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -63,7 +63,7 @@ def __init__(self, hs: "HomeServer"): hs.get_instance_name() in hs.config.worker.writers.push_rules ) self._push_rules_handler = hs.get_push_rules_handler() - self._push_rule_linearizer = Linearizer(name="push_rules", clock=hs.get_clock()) + self._push_rule_linearizer = Linearizer(hs.get_clock(), name="push_rules") async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: if not self._is_push_worker: diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 8e7af360b80..45993476216 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -643,7 +643,8 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.resolve_linearizer = Linearizer( - name="state_resolve_lock", clock=hs.get_clock() + hs.get_clock(), + name="state_resolve_lock", ) # dict of set of event_ids -> _StateCacheEntry. diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 7528b3d2a68..97d17803643 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -78,7 +78,8 @@ def __init__(self, hs: "HomeServer", stores: "Databases"): # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. self._joined_host_linearizer = Linearizer( - "_JoinedHostsCache", clock=hs.get_clock() + hs.get_clock(), + "_JoinedHostsCache", ) def notify_event_un_partial_stated(self, event_id: str) -> None: diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 44696fc50e2..148c46048b8 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -47,7 +47,6 @@ Tuple, TypeVar, Union, - cast, overload, ) @@ -56,7 +55,6 @@ from twisted.internet import defer from twisted.internet.defer import CancelledError -from twisted.internet.interfaces import IReactorTime from twisted.python.failure import Failure from synapse.logging.context import ( @@ -532,9 +530,9 @@ class Linearizer: def __init__( self, + clock: Clock, name: Optional[str] = None, max_count: int = 1, - clock: Optional[Clock] = None, ): """ Args: @@ -545,10 +543,6 @@ def __init__( else: self.name = name - if not clock: - from twisted.internet import reactor - - clock = Clock(cast(IReactorTime, reactor)) self._clock = clock self.max_count = max_count diff --git a/tests/server.py b/tests/server.py index 81e09a4a002..cba8b0281e7 100644 --- a/tests/server.py +++ b/tests/server.py @@ -625,6 +625,15 @@ def connectTCP( return conn + def runUntilCurrent(self) -> None: + max_time = 0.0 + for call in self.getDelayedCalls(): + time_until_call = call.getTime() - self.seconds() + if time_until_call > max_time: + max_time = time_until_call + + self.advance(max_time) + def advance(self, amount: float) -> None: # first advance our reactor's time, and run any "callLater" callbacks that # makes ready diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py index 7510657b858..4fa1d0eae07 100644 --- a/tests/util/test_linearizer.py +++ b/tests/util/test_linearizer.py @@ -29,6 +29,7 @@ from synapse.util.async_helpers import Linearizer from tests import unittest +from tests.server import get_clock class UnblockFunction(Protocol): @@ -36,6 +37,10 @@ def __call__(self, pump_reactor: bool = True) -> None: ... class LinearizerTestCase(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.reactor, self.clock = get_clock() + def _start_task( self, linearizer: Linearizer, key: Hashable ) -> Tuple["Deferred[None]", "Deferred[None]", UnblockFunction]: @@ -74,12 +79,12 @@ def unblock(pump_reactor: bool = True) -> None: def _pump(self) -> None: """Pump the reactor to advance `Linearizer`s.""" assert isinstance(reactor, ReactorBase) - while reactor.getDelayedCalls(): - reactor.runUntilCurrent() + while self.reactor.getDelayedCalls(): + self.reactor.runUntilCurrent() def test_linearizer(self) -> None: """Tests that a task is queued up behind an earlier task.""" - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() @@ -100,7 +105,7 @@ def test_linearizer_is_queued(self) -> None: Runs through the same scenario as `test_linearizer`. """ - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() @@ -131,7 +136,7 @@ def test_lots_of_queued_things(self) -> None: The stack should *not* explode when the slow thing completes. """ - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = "" async def func(i: int) -> None: @@ -151,7 +156,7 @@ async def func(i: int) -> None: def test_multiple_entries(self) -> None: """Tests a `Linearizer` with a concurrency above 1.""" - limiter = Linearizer(max_count=3) + limiter = Linearizer(self.clock, max_count=3) key = object() @@ -192,7 +197,7 @@ def test_multiple_entries(self) -> None: def test_cancellation(self) -> None: """Tests cancellation while waiting for a `Linearizer`.""" - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() @@ -226,7 +231,7 @@ def test_cancellation(self) -> None: def test_cancellation_during_sleep(self) -> None: """Tests cancellation during the sleep just after waiting for a `Linearizer`.""" - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() From 35f88f45271c9028bc8627291324053f0f186d32 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 13:28:13 -0600 Subject: [PATCH 049/181] Return None instead of raising exception --- synapse/util/metrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index e07fe84eddd..65657b4e98b 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -312,4 +312,4 @@ def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: server_hooks.append(hook) def unregister_hooks_for_homeserver_instance_id(self, server_name: str) -> None: - self._server_id_to_pre_update_hooks.pop(server_name) + self._server_id_to_pre_update_hooks.pop(server_name, None) From 0e67230b2a3c7fd90e42e6d761a7073bb8cb1280 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 13:35:47 -0600 Subject: [PATCH 050/181] Clarify our_server_name --- synapse/http/site.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index 4b317749466..cb291bf2fbd 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -667,7 +667,7 @@ class SynapseProtocol(HTTPChannel): def __init__( self, factory: "SynapseSite", - server_name: str, + our_server_name: str, max_request_body_size: int, request_id_header: Optional[str], request_class: type, @@ -675,7 +675,7 @@ def __init__( super().__init__() self.factory: SynapseSite = factory self.site = factory - self.server_name = server_name + self.our_server_name = our_server_name self.max_request_body_size = max_request_body_size self.request_id_header = request_id_header self.request_class = request_class @@ -702,7 +702,7 @@ def requestFactory(self, http_channel: HTTPChannel, queued: bool) -> SynapseRequ return self.request_class( self, self.factory, - our_server_name=self.server_name, + our_server_name=self.our_server_name, max_request_body_size=self.max_request_body_size, queued=queued, request_id_header=self.request_id_header, From 1a22e8f0f0965ec84952bb4e8782c4b3af29d4fa Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 13:42:04 -0600 Subject: [PATCH 051/181] Rename factory arg to site --- synapse/http/site.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index cb291bf2fbd..4b26e59b0f6 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -666,15 +666,15 @@ class SynapseProtocol(HTTPChannel): def __init__( self, - factory: "SynapseSite", + site: "SynapseSite", our_server_name: str, max_request_body_size: int, request_id_header: Optional[str], request_class: type, ): super().__init__() - self.factory: SynapseSite = factory - self.site = factory + self.factory: SynapseSite = site + self.site = site self.our_server_name = our_server_name self.max_request_body_size = max_request_body_size self.request_id_header = request_id_header From 4d2e59526a451b4f690048f7683ccf26ece65c48 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 14:00:19 -0600 Subject: [PATCH 052/181] Add comment documenting SynapseSite shutdown --- synapse/http/site.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/http/site.py b/synapse/http/site.py index 4b26e59b0f6..94879e76028 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -788,6 +788,11 @@ def removeConnection(self, protocol: Protocol) -> None: def stopFactory(self) -> None: super().stopFactory() + # Shutdown any connections which are still active. + # These can be long lived HTTP connections which wouldn't normally be closed + # when calling `shutdown` on the respective `Port`. + # Closing the connections here is required for us to fully shutdown the + # `SynapseHomeServer` in order for it to be garbage collected. for protocol in self.connections[:]: if protocol.transport is not None: protocol.transport.loseConnection() From a5ff83edb31009a1088502abd36cf942407c3a68 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 14:31:40 -0600 Subject: [PATCH 053/181] Add docs about using Clock instead of reactor directly --- synapse/util/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 0183ab4cfea..648664af958 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -113,6 +113,11 @@ class Clock: """ A Clock wraps a Twisted reactor and provides utilities on top of it. + This clock should be used in place of calls to the base reactor wherever `LoopingCall` + or `DelayedCall` are made (such as when calling `reactor.callLater`. This is to + ensure the calls made by this `HomeServer` instance are tracked and can be cleaned + up during `HomeServer.shutdown()`. + Args: reactor: The Twisted reactor to use. """ From 62978f159ad2d452b2dcdcfe546e5953554b0c67 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 4 Sep 2025 14:59:08 -0600 Subject: [PATCH 054/181] Remove unnecessary cleanup changes --- synapse/federation/federation_server.py | 2 +- synapse/handlers/appservice.py | 1 - synapse/handlers/device.py | 2 -- synapse/handlers/e2e_keys.py | 5 +---- synapse/handlers/federation.py | 3 +-- synapse/handlers/federation_event.py | 2 +- synapse/handlers/message.py | 1 - synapse/handlers/presence.py | 1 - synapse/handlers/read_marker.py | 2 +- synapse/handlers/room_member.py | 2 -- synapse/handlers/sso.py | 2 +- synapse/http/matrixfederationclient.py | 4 +--- synapse/media/media_repository.py | 1 - synapse/replication/tcp/client.py | 1 - synapse/rest/client/push_rule.py | 2 +- synapse/rest/synapse/mas/_base.py | 4 +--- synapse/state/__init__.py | 1 - synapse/storage/controllers/state.py | 1 - synapse/util/async_helpers.py | 8 +++++++- tests/util/test_linearizer.py | 21 ++++++++------------- 20 files changed, 24 insertions(+), 42 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 27a3a664bd1..a8d5c3c280a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -159,7 +159,7 @@ def __init__(self, hs: "HomeServer"): # with FederationHandlerRegistry. hs.get_directory_handler() - self._server_linearizer = Linearizer(hs.get_clock(), "fed_server") + self._server_linearizer = Linearizer("fed_server") # origins that we are currently processing a transaction from. # a dict from origin to txn id. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 00fe763e708..70968caaa2b 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -98,7 +98,6 @@ def __init__(self, hs: "HomeServer"): self.is_processing = False self._ephemeral_events_linearizer = Linearizer( - hs.get_clock(), name="appservice_ephemeral_events", ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 187e5e59eb2..2b305054b77 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1451,11 +1451,9 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): self.device_handler = device_handler self._remote_edu_linearizer = Linearizer( - hs.get_clock(), name="remote_device_list", ) self._resync_linearizer = Linearizer( - hs.get_clock(), name="remote_device_resync", ) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 835bdc8953d..b9abad21881 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -112,7 +112,6 @@ def __init__(self, hs: "HomeServer"): # Limit the number of in-flight requests from a single device. self._query_devices_linearizer = Linearizer( - hs.get_clock(), name="query_devices", max_count=10, ) @@ -1766,9 +1765,7 @@ def __init__(self, hs: "HomeServer"): assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler - self._remote_edu_linearizer = Linearizer( - hs.get_clock(), name="remote_signing_key" - ) + self._remote_edu_linearizer = Linearizer(name="remote_signing_key") # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[str, List[Tuple[JsonDict, JsonDict]]] = {} diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9185d436c8f..3ba5a821323 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -160,7 +160,7 @@ def __init__(self, hs: "HomeServer"): self._notifier = hs.get_notifier() self._worker_locks = hs.get_worker_locks_handler() - self._room_backfill = Linearizer(hs.get_clock(), "room_backfill") + self._room_backfill = Linearizer("room_backfill") self._third_party_event_rules = ( hs.get_module_api_callbacks().third_party_event_rules @@ -180,7 +180,6 @@ def __init__(self, hs: "HomeServer"): # When the lock is held for a given room, no other concurrent code may # partial state or un-partial state the room. self._is_partial_state_room_linearizer = Linearizer( - hs.get_clock(), name="_is_partial_state_room_linearizer", ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 486736c8f88..9f774fa9f74 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -192,7 +192,7 @@ def __init__(self, hs: "HomeServer"): # federation event staging area. self.room_queues: Dict[str, List[Tuple[EventBase, str]]] = {} - self._room_pdu_linearizer = Linearizer(hs.get_clock(), "fed_room_pdu") + self._room_pdu_linearizer = Linearizer("fed_room_pdu") async def on_receive_pdu(self, origin: str, pdu: EventBase) -> None: """Process a PDU received via a federation /send/ transaction diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 3469dfb1a1f..7a342565acf 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -513,7 +513,6 @@ def __init__(self, hs: "HomeServer"): # We limit concurrent event creation for a room to 1. This prevents state resolution # from occurring when sending bursts of events to a local room self.limiter = Linearizer( - hs.get_clock(), max_count=1, name="room_event_creation_limit", ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 974050a0e5b..06ab3454cd7 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -862,7 +862,6 @@ def __init__(self, hs: "HomeServer"): self.external_process_last_updated_ms: Dict[str, int] = {} self.external_sync_linearizer = Linearizer( - hs.get_clock(), name="external_sync_linearizer", ) diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index 6a311055dc2..fb39c8e04b5 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -36,7 +36,7 @@ class ReadMarkerHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.account_data_handler = hs.get_account_data_handler() - self.read_marker_linearizer = Linearizer(hs.get_clock(), name="read_marker") + self.read_marker_linearizer = Linearizer(name="read_marker") async def received_client_read_marker( self, room_id: str, user_id: str, event_id: str diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index a7e29ebc90d..3fc7a992643 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -115,11 +115,9 @@ def __init__(self, hs: "HomeServer"): self._membership_types_to_include_profile_data_in.add(Membership.INVITE) self.member_linearizer: Linearizer = Linearizer( - hs.get_clock(), name="member", ) self.member_as_limiter = Linearizer( - hs.get_clock(), max_count=10, name="member_as_limiter", ) diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 27fe035ebf0..05a9da69949 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -224,7 +224,7 @@ def __init__(self, hs: "HomeServer"): ) # a lock on the mappings - self._mapping_lock = Linearizer(hs.get_clock(), name="sso_user_mapping") + self._mapping_lock = Linearizer(name="sso_user_mapping") # a map from session id to session data self._username_mapping_sessions: Dict[str, UsernameMappingSession] = {} diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index ce450af55f8..5c876025187 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -482,9 +482,7 @@ def __init__( use_proxy=True, ) - self.remote_download_linearizer = Linearizer( - hs.get_clock(), "remote_download_linearizer", 6 - ) + self.remote_download_linearizer = Linearizer("remote_download_linearizer", 6) def wake_destination(self, destination: str) -> None: """Called when the remote server may have come back online.""" diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index a7f17a29d1c..c575443f06d 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -109,7 +109,6 @@ def __init__(self, hs: "HomeServer"): self.thumbnail_requirements = hs.config.media.thumbnail_requirements self.remote_media_linearizer = Linearizer( - hs.get_clock(), name="media_remote", ) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 46998db6cae..786703a7662 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -430,7 +430,6 @@ def __init__(self, hs: "HomeServer"): self.federation_position: Optional[int] = None self._fed_position_linearizer = Linearizer( - hs.get_clock(), name="_fed_position_linearizer", ) diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index e1698616e74..af042504c9b 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -63,7 +63,7 @@ def __init__(self, hs: "HomeServer"): hs.get_instance_name() in hs.config.worker.writers.push_rules ) self._push_rules_handler = hs.get_push_rules_handler() - self._push_rule_linearizer = Linearizer(hs.get_clock(), name="push_rules") + self._push_rule_linearizer = Linearizer(name="push_rules") async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: if not self._is_push_worker: diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py index 07332af1773..7346198b750 100644 --- a/synapse/rest/synapse/mas/_base.py +++ b/synapse/rest/synapse/mas/_base.py @@ -42,9 +42,7 @@ def __init__(self, hs: "HomeServer"): self._is_request_from_mas = auth.is_request_using_the_admin_token - DirectServeJsonResource.__init__( - self, extract_context=True, clock=hs.get_clock() - ) + DirectServeJsonResource.__init__(self, extract_context=True) self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.hostname = hs.hostname diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 45993476216..54fededcfbe 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -643,7 +643,6 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.resolve_linearizer = Linearizer( - hs.get_clock(), name="state_resolve_lock", ) diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 97d17803643..9576ba86889 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -78,7 +78,6 @@ def __init__(self, hs: "HomeServer", stores: "Databases"): # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. self._joined_host_linearizer = Linearizer( - hs.get_clock(), "_JoinedHostsCache", ) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 148c46048b8..44696fc50e2 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -47,6 +47,7 @@ Tuple, TypeVar, Union, + cast, overload, ) @@ -55,6 +56,7 @@ from twisted.internet import defer from twisted.internet.defer import CancelledError +from twisted.internet.interfaces import IReactorTime from twisted.python.failure import Failure from synapse.logging.context import ( @@ -530,9 +532,9 @@ class Linearizer: def __init__( self, - clock: Clock, name: Optional[str] = None, max_count: int = 1, + clock: Optional[Clock] = None, ): """ Args: @@ -543,6 +545,10 @@ def __init__( else: self.name = name + if not clock: + from twisted.internet import reactor + + clock = Clock(cast(IReactorTime, reactor)) self._clock = clock self.max_count = max_count diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py index 4fa1d0eae07..7510657b858 100644 --- a/tests/util/test_linearizer.py +++ b/tests/util/test_linearizer.py @@ -29,7 +29,6 @@ from synapse.util.async_helpers import Linearizer from tests import unittest -from tests.server import get_clock class UnblockFunction(Protocol): @@ -37,10 +36,6 @@ def __call__(self, pump_reactor: bool = True) -> None: ... class LinearizerTestCase(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.reactor, self.clock = get_clock() - def _start_task( self, linearizer: Linearizer, key: Hashable ) -> Tuple["Deferred[None]", "Deferred[None]", UnblockFunction]: @@ -79,12 +74,12 @@ def unblock(pump_reactor: bool = True) -> None: def _pump(self) -> None: """Pump the reactor to advance `Linearizer`s.""" assert isinstance(reactor, ReactorBase) - while self.reactor.getDelayedCalls(): - self.reactor.runUntilCurrent() + while reactor.getDelayedCalls(): + reactor.runUntilCurrent() def test_linearizer(self) -> None: """Tests that a task is queued up behind an earlier task.""" - linearizer = Linearizer(self.clock) + linearizer = Linearizer() key = object() @@ -105,7 +100,7 @@ def test_linearizer_is_queued(self) -> None: Runs through the same scenario as `test_linearizer`. """ - linearizer = Linearizer(self.clock) + linearizer = Linearizer() key = object() @@ -136,7 +131,7 @@ def test_lots_of_queued_things(self) -> None: The stack should *not* explode when the slow thing completes. """ - linearizer = Linearizer(self.clock) + linearizer = Linearizer() key = "" async def func(i: int) -> None: @@ -156,7 +151,7 @@ async def func(i: int) -> None: def test_multiple_entries(self) -> None: """Tests a `Linearizer` with a concurrency above 1.""" - limiter = Linearizer(self.clock, max_count=3) + limiter = Linearizer(max_count=3) key = object() @@ -197,7 +192,7 @@ def test_multiple_entries(self) -> None: def test_cancellation(self) -> None: """Tests cancellation while waiting for a `Linearizer`.""" - linearizer = Linearizer(self.clock) + linearizer = Linearizer() key = object() @@ -231,7 +226,7 @@ def test_cancellation(self) -> None: def test_cancellation_during_sleep(self) -> None: """Tests cancellation during the sleep just after waiting for a `Linearizer`.""" - linearizer = Linearizer(self.clock) + linearizer = Linearizer() key = object() From a7f394814ddbb7061dec1055f01f14326c7417eb Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 09:05:28 -0600 Subject: [PATCH 055/181] Add TODO about shutdown freeze detection --- synapse/server.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/server.py b/synapse/server.py index 445428f9d06..de9133ccf12 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -374,6 +374,15 @@ def shutdown(self) -> None: logger.info("Received shutdown request") + # TODO: It would be desireable to be able to report an error if the HomeServer + # object is frozen in the garbage collector as that would prevent it from being + # collected after being shutdown. + # In theory the following should work, but it doesn't seem to make a difference + # when I test it locally. + # + # if gc.is_tracked(self): + # logger.error("HomeServer object is tracked by garbage collection so cannot be fully cleaned up") + for listener in self._listening_services: # During unit tests, an incomplete `_FakePort` is used for listeners so # check listener type here to ensure shutdown procedure is only applied to From 42f3e8fadd1e07e3862e86fc8075ae165c1ec566 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 09:09:15 -0600 Subject: [PATCH 056/181] Callout full name of _FakePort in comment --- synapse/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index de9133ccf12..095f14b27ea 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -384,9 +384,9 @@ def shutdown(self) -> None: # logger.error("HomeServer object is tracked by garbage collection so cannot be fully cleaned up") for listener in self._listening_services: - # During unit tests, an incomplete `_FakePort` is used for listeners so - # check listener type here to ensure shutdown procedure is only applied to - # actual `Port` instances. + # During unit tests, an incomplete `twisted.pair.testing._FakePort` is used + # for listeners so check listener type here to ensure shutdown procedure is + # only applied to actual `Port` instances. if type(listener) is Port: port_shutdown = listener.stopListening() if port_shutdown is not None: From f00d90b0acd69166ec52f7d1f59509b7a6220b57 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 09:20:32 -0600 Subject: [PATCH 057/181] Reword SynapseProtocol docstrings --- synapse/http/site.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/http/site.py b/synapse/http/site.py index 94879e76028..7a9d609db5a 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -682,6 +682,11 @@ def __init__( def connectionMade(self) -> None: """ + Called when a connection is made. + + This may be considered the initializer of the protocol, because + it is called when the connection is completed. + Add the connection to the factory's connection list when it's established. """ super().connectionMade() @@ -689,6 +694,8 @@ def connectionMade(self) -> None: def connectionLost(self, reason: Failure) -> None: # type: ignore[override] """ + Called when the connection is shut down. + Remove the connection from the factory's connection list, when it's lost. """ super().connectionLost(reason) @@ -696,6 +703,8 @@ def connectionLost(self, reason: Failure) -> None: # type: ignore[override] def requestFactory(self, http_channel: HTTPChannel, queued: bool) -> SynapseRequest: # type: ignore[override] """ + A callable used to build `twisted.web.iweb.IRequest` objects. + Use our own custom SynapseRequest type instead of the regular twisted.web.server.Request. """ From 57b030d1cce0ec01be66f5e8b8c5c84222a06e48 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 09:31:16 -0600 Subject: [PATCH 058/181] Remove unused code --- tests/server.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/server.py b/tests/server.py index cba8b0281e7..81e09a4a002 100644 --- a/tests/server.py +++ b/tests/server.py @@ -625,15 +625,6 @@ def connectTCP( return conn - def runUntilCurrent(self) -> None: - max_time = 0.0 - for call in self.getDelayedCalls(): - time_until_call = call.getTime() - self.seconds() - if time_until_call > max_time: - max_time = time_until_call - - self.advance(max_time) - def advance(self, amount: float) -> None: # first advance our reactor's time, and run any "callLater" callbacks that # makes ready From ed1b436362a9c644fd83232e11cb674641e41b8a Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 09:48:19 -0600 Subject: [PATCH 059/181] Add comment explaining need for type ignore --- tests/replication/_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/replication/_base.py b/tests/replication/_base.py index d6c33766b8a..537ac63ac1f 100644 --- a/tests/replication/_base.py +++ b/tests/replication/_base.py @@ -173,6 +173,12 @@ def handle_http_replication_attempt(self) -> SynapseRequest: # Set up the server side protocol server_address = IPv4Address("TCP", host, port) + # The type ignore is here because mypy doesn't think the host/port tuple is of + # the correct type, even though it is the exact example given for + # `twisted.internet.interfaces.IAddress`. + # Mypy was happy with the type before we overrode `buildProtocol` in + # `SynapseSite`, probably because there was enough inheritance indirection before + # withe the argument not having a type associated with it. channel = self.site.buildProtocol((host, port)) # type: ignore[arg-type] # hook into the channel's request factory so that we can keep a record From 7fc17a887c7b9421e28817ffe2fb4f505ae72986 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 16:48:27 +0000 Subject: [PATCH 060/181] Update synapse/_scripts/synapse_port_db.py Co-authored-by: Eric Eastwood --- synapse/_scripts/synapse_port_db.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 47a1b429b0d..0f7d177be04 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -345,9 +345,6 @@ def should_send_federation(self) -> bool: def get_replication_notifier(self) -> ReplicationNotifier: return ReplicationNotifier() - def register_later_gauge(self, later_gauge: LaterGauge) -> None: - pass - class Porter: def __init__( From 27f40390b3c7a9378b52b28ffe9eb2e9d2e8d83d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 16:48:38 +0000 Subject: [PATCH 061/181] Update synapse/app/_base.py Co-authored-by: Eric Eastwood --- synapse/app/_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index c87c1ca43f8..a95071071e0 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -100,10 +100,12 @@ logger = logging.getLogger(__name__) -# dict of instance_id to tuples of function, args list, kwargs dict _sighup_callbacks: Dict[ str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] ] = {} +""" +Mapping from homeserver instance_id to tuples of function, args list, kwargs dict +""" P = ParamSpec("P") From e8a145b3eb74a7fa23e71dc6fc9169270acaf588 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 10:49:14 -0600 Subject: [PATCH 062/181] More docstrings --- synapse/crypto/keyring.py | 3 +++ synapse/federation/send_queue.py | 1 + synapse/server.py | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index d7b62f33fb5..a881534da2a 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -493,6 +493,9 @@ def __init__(self, hs: "HomeServer"): ) def shutdown(self) -> None: + """ + Prepares the KeyFetcher for garbage collection by shutting down it's queue. + """ self._queue.shutdown() async def get_keys( diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 2b751afc14f..f2717c9f7bc 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -145,6 +145,7 @@ def register(queue_name: QueueNames, queue: Sized) -> None: self.clock.looping_call(self._clear_queue, 30 * 1000) def shutdown(self) -> None: + """Stops this federation sender instance from sending further transactions.""" pass def _next_pos(self) -> int: diff --git a/synapse/server.py b/synapse/server.py index 095f14b27ea..f0192929b00 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -574,7 +574,7 @@ def setup(self) -> None: # Make sure we actually do some clean-up, rather than leak data. # """ # - # # NOTE: This is a chicken an egg problem. + # # NOTE: This is a chicken and egg problem. # # __del__ will never be called since the HomeServer cannot be garbage collected # # until the shutdown function has been called. So it makes no sense to call # # shutdown inside of __del__, even though that is a logical place to assume it From 070f3026e5068bcce99f951ebce5fb1e7d72cf4d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 10:51:22 -0600 Subject: [PATCH 063/181] Fix linter errors --- synapse/_scripts/synapse_port_db.py | 1 - synapse/federation/send_queue.py | 1 - 2 files changed, 2 deletions(-) diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 0f7d177be04..a81db3cfbfb 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -58,7 +58,6 @@ make_deferred_yieldable, run_in_background, ) -from synapse.metrics import LaterGauge from synapse.notifier import ReplicationNotifier from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn from synapse.storage.databases.main import FilteringWorkerStore diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index f2717c9f7bc..759df9836b9 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -146,7 +146,6 @@ def register(queue_name: QueueNames, queue: Sized) -> None: def shutdown(self) -> None: """Stops this federation sender instance from sending further transactions.""" - pass def _next_pos(self) -> int: pos = self.pos From 073734a409020e958388c8cccca35d726b9f95e9 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 10:52:37 -0600 Subject: [PATCH 064/181] Add instance_id to docstring --- synapse/app/_base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index a95071071e0..505838a804f 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -116,6 +116,7 @@ def register_sighup( Register a function to be called when a SIGHUP occurs. Args: + instance_id: Unique ID for this Synapse process instance. func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ From a53a7dfa88f04ac5bb3d4eed065c3c2949e59fb9 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 10:54:26 -0600 Subject: [PATCH 065/181] Add return docstring --- synapse/app/_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 505838a804f..3cd2b887ccf 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -310,6 +310,9 @@ def listen_metrics( bytecode at a time), this still works because the metrics thread can preempt the Twisted reactor thread between bytecode boundaries and the metrics thread gets scheduled with roughly equal priority to the Twisted reactor thread. + + Returns: + List of WSGIServer with the thread they are running on. """ from prometheus_client import start_http_server as start_http_server_prometheus From cb2f562e5515989650c855aa871cb836997e7fe8 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 10:59:16 -0600 Subject: [PATCH 066/181] Add docstring for freeze --- synapse/app/_base.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3cd2b887ccf..d35a8a11542 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -533,6 +533,9 @@ async def start(hs: "HomeServer", freeze: bool = True) -> None: Args: hs: homeserver instance + freeze: whether to freeze the homeserver base objects in the garbage collector. + May improve garbage collection performance by marking objects with an effectively + static lifetime as frozen so they don't need to be considered for cleanup. """ server_name = hs.hostname reactor = hs.get_reactor() From 41734ca91a156fa90267d223e2ef58615bc46344 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 11:00:29 -0600 Subject: [PATCH 067/181] Add docstring for unregister_sighups --- synapse/app/_base.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index d35a8a11542..5014605f87c 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -125,6 +125,12 @@ def register_sighup( def unregister_sighups(instance_id: str) -> None: + """ + Unregister all sighup functions associated with this Synapse instance. + + Args: + instance_id: Unique ID for this Synapse process instance. + """ _sighup_callbacks.pop(instance_id, []) From 219d00293a2ecbbce97c52c029aed40cfd43d964 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 17:01:23 +0000 Subject: [PATCH 068/181] Update synapse/app/_base.py Co-authored-by: Eric Eastwood --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 5014605f87c..a42338ad280 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -573,7 +573,7 @@ async def _handle_sighup(*args: Any, **kwargs: Any) -> None: sdnotify(b"RELOADING=1") for _, v in _sighup_callbacks.items(): - for i, args, kwargs in v: + for func, args, kwargs in v: i(*args, **kwargs) sdnotify(b"READY=1") From cf10f45b0e2ef82ff09d76e4b037d5e7fde15aef Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 11:03:32 -0600 Subject: [PATCH 069/181] Rename sighup map --- synapse/app/_base.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index a42338ad280..8a37d1ddf05 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -100,7 +100,7 @@ logger = logging.getLogger(__name__) -_sighup_callbacks: Dict[ +_instance_id_to_sighup_callbacks_map: Dict[ str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] ] = {} """ @@ -121,17 +121,19 @@ def register_sighup( *args, **kwargs: args and kwargs to be passed to the target function. """ - _sighup_callbacks.setdefault(instance_id, []).append((func, args, kwargs)) + _instance_id_to_sighup_callbacks_map.setdefault(instance_id, []).append( + (func, args, kwargs) + ) def unregister_sighups(instance_id: str) -> None: """ Unregister all sighup functions associated with this Synapse instance. - + Args: instance_id: Unique ID for this Synapse process instance. """ - _sighup_callbacks.pop(instance_id, []) + _instance_id_to_sighup_callbacks_map.pop(instance_id, []) def start_worker_reactor( @@ -572,9 +574,9 @@ async def _handle_sighup(*args: Any, **kwargs: Any) -> None: # we're not using systemd. sdnotify(b"RELOADING=1") - for _, v in _sighup_callbacks.items(): + for _, v in _instance_id_to_sighup_callbacks_map.items(): for func, args, kwargs in v: - i(*args, **kwargs) + func(*args, **kwargs) sdnotify(b"READY=1") From 7b64868b6efe2770fbd9ba2c65bf8ad8a7a522b8 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 14:31:59 -0600 Subject: [PATCH 070/181] Make homeserver shutdown async --- synapse/_scripts/generate_workers_map.py | 2 +- synapse/server.py | 45 +++++++++++------------- tests/server.py | 6 ++-- 3 files changed, 25 insertions(+), 28 deletions(-) diff --git a/synapse/_scripts/generate_workers_map.py b/synapse/_scripts/generate_workers_map.py index d24679cdcdd..58618d848c2 100755 --- a/synapse/_scripts/generate_workers_map.py +++ b/synapse/_scripts/generate_workers_map.py @@ -157,7 +157,7 @@ def get_registered_paths_for_default( # TODO We only do this to avoid an error, but don't need the database etc hs.setup() registered_paths = get_registered_paths_for_hs(hs) - hs.shutdown() + hs.cleanup_metrics() return registered_paths diff --git a/synapse/server.py b/synapse/server.py index f0192929b00..45e2d14086b 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -361,7 +361,7 @@ def __init__( self._async_shutdown_handlers: List[ShutdownInfo] = [] self._sync_shutdown_handlers: List[ShutdownInfo] = [] - def shutdown(self) -> None: + async def shutdown(self) -> None: """ Cleanly stops all aspects of the HomeServer and removes any references that have been handed out in order to allow the HomeServer object to be garbage @@ -390,20 +390,12 @@ def shutdown(self) -> None: if type(listener) is Port: port_shutdown = listener.stopListening() if port_shutdown is not None: - - def on_error(_: T) -> None: - logger.error("Port shutdown with error") - - def on_success(_: T) -> None: - logger.error("Port shutdown successfully") - - port_shutdown.addCallback(on_success) - port_shutdown.addErrback(on_error) + await port_shutdown self._listening_services.clear() - logger.error("Shutting down metrics listeners") + logger.info("Shutting down metrics listeners") for server, thread in self._metrics_listeners: - logger.error("Shutting down metrics listener") + logger.info("Shutting down metrics listener") server.shutdown() thread.join() self._metrics_listeners.clear() @@ -412,19 +404,7 @@ def on_success(_: T) -> None: self.get_keyring().shutdown() - # Cleanup metrics associated with the homeserver - for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): - later_gauge.unregister_hooks_for_homeserver_instance_id( - self.get_instance_id() - ) - - CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver_instance_id( - self.config.server.server_name - ) - - # TODO: What about the other gauge types? - # from synapse.metrics import all_gauges - # all_gauges.clear() + self.cleanup_metrics() for db in self.get_datastores().databases: db.stop_background_updates() @@ -484,6 +464,21 @@ def register_async_shutdown_handler( ) ) + def cleanup_metrics(self) -> None: + # Cleanup metrics associated with the homeserver + for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): + later_gauge.unregister_hooks_for_homeserver_instance_id( + self.get_instance_id() + ) + + CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver_instance_id( + self.config.server.server_name + ) + + # TODO: What about the other gauge types? + # from synapse.metrics import all_gauges + # all_gauges.clear() + def register_sync_shutdown_handler( self, phase: str, diff --git a/tests/server.py b/tests/server.py index 81e09a4a002..a863bf3a42e 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1145,8 +1145,10 @@ def setup_test_homeserver( reactor=reactor, ) - # Register the cleanup hook - cleanup_func(hs.shutdown) + # Register the cleanup hook for homeserver metrics. + # We only need to cleanup homeserver metrics here since they can build up over time + # to a point where we run out of memory in CI. + cleanup_func(hs.cleanup_metrics) # Install @cache_in_self attributes for key, val in kwargs.items(): From 1f334efcb6be53ed307954cec5cb747c51dd40c6 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 14:40:43 -0600 Subject: [PATCH 071/181] Move up unregister_sighups call --- synapse/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index 45e2d14086b..48e2bf4ba67 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -374,6 +374,8 @@ async def shutdown(self) -> None: logger.info("Received shutdown request") + unregister_sighups(self._instance_id) + # TODO: It would be desireable to be able to report an error if the HomeServer # object is frozen in the garbage collector as that would prevent it from being # collected after being shutdown. @@ -435,8 +437,6 @@ async def shutdown(self) -> None: pass self._sync_shutdown_handlers.clear() - unregister_sighups(self._instance_id) - def register_async_shutdown_handler( self, phase: str, From ac8ecb1e248a006ca98346c3c6e70e68ed49ca66 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 16:42:28 -0600 Subject: [PATCH 072/181] Call hs.shutdown during tests --- tests/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/server.py b/tests/server.py index a863bf3a42e..f46289881f2 100644 --- a/tests/server.py +++ b/tests/server.py @@ -56,7 +56,7 @@ import twisted from twisted.enterprise import adbapi -from twisted.internet import address, tcp, threads, udp +from twisted.internet import address, defer, tcp, threads, udp from twisted.internet._resolver import SimpleResolverComplexifier from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet.defer import Deferred, fail, maybeDeferred, succeed @@ -1148,7 +1148,7 @@ def setup_test_homeserver( # Register the cleanup hook for homeserver metrics. # We only need to cleanup homeserver metrics here since they can build up over time # to a point where we run out of memory in CI. - cleanup_func(hs.cleanup_metrics) + cleanup_func(lambda: (defer.ensureDeferred(hs.shutdown()), None)[1]) # Install @cache_in_self attributes for key, val in kwargs.items(): From c3856ac65f1833f0df4ed13810551012c8eb17ac Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 17:03:25 -0600 Subject: [PATCH 073/181] Update test shutdown comment --- tests/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/server.py b/tests/server.py index f46289881f2..5a1dd779e91 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1145,9 +1145,9 @@ def setup_test_homeserver( reactor=reactor, ) - # Register the cleanup hook for homeserver metrics. - # We only need to cleanup homeserver metrics here since they can build up over time - # to a point where we run out of memory in CI. + # Register the cleanup hook for the homeserver. + # A full `hs.shutdown()` is necessary otherwise CI tests will fail while exhibiting + # strange behaviours. cleanup_func(lambda: (defer.ensureDeferred(hs.shutdown()), None)[1]) # Install @cache_in_self attributes From 4a6ead1e9089de67bd29b96b95a71c15b1398129 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 17:06:27 -0600 Subject: [PATCH 074/181] Further explain docstring args --- synapse/app/homeserver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7316f1ea846..7ec22d3a6f0 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -321,8 +321,11 @@ def setup( """ Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. - reactor: Optionally provide a reactor to use. - freeze: Whether to freeze all objects in the garbage collector. + reactor: Optionally provide a reactor to use. Can be useful in different + scenarios that you want to control over the reactor, such as tests. + freeze: Whether to freeze all objects in the garbage collector. May result in + less work for the garbage collector since the `SynapseHomeServer` generally has + a static lifetime. Returns: A homeserver instance. From 672adc2a9bab11e11c8ff4e6efe4ed9086bdb562 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 17:11:44 -0600 Subject: [PATCH 075/181] Fix function name --- synapse/federation/sender/__init__.py | 2 +- synapse/federation/sender/per_destination_queue.py | 4 +++- synapse/server.py | 2 +- synapse/util/metrics.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index a03b49c77f8..c644f5a28f3 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -471,7 +471,7 @@ def __init__(self, hs: "HomeServer"): def shutdown(self) -> None: for queue in self._per_destination_queues.values(): - queue.disable() + queue.shutdown() def _get_per_destination_queue( self, destination: str diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 5c18bfe9c55..20787fb0801 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -79,6 +79,7 @@ class PerDestinationQueue: """ Manages the per-destination transmission queues. + Runs until `shutdown()` is called on the queue. Args: hs @@ -173,7 +174,8 @@ def __init__( def __str__(self) -> str: return "PerDestinationQueue[%s]" % self._destination - def disable(self) -> None: + def shutdown(self) -> None: + """Instruct the queue to stop processing any further requests""" self._transmission_loop_enabled = False def pending_pdu_count(self) -> int: diff --git a/synapse/server.py b/synapse/server.py index 48e2bf4ba67..cd8c9959680 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -471,7 +471,7 @@ def cleanup_metrics(self) -> None: self.get_instance_id() ) - CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver_instance_id( + CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver( self.config.server.server_name ) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 65657b4e98b..8ca9d73b6fa 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -311,5 +311,5 @@ def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: server_hooks = self._server_id_to_pre_update_hooks.setdefault(server_name, []) server_hooks.append(hook) - def unregister_hooks_for_homeserver_instance_id(self, server_name: str) -> None: + def unregister_hooks_for_homeserver(self, server_name: str) -> None: self._server_id_to_pre_update_hooks.pop(server_name, None) From 79e84ebb6f55ce235075066cd152019b90912e4c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 17:13:09 -0600 Subject: [PATCH 076/181] Update var name --- synapse/util/metrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 8ca9d73b6fa..fd8955c9d0c 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -290,14 +290,14 @@ class DynamicCollectorRegistry(CollectorRegistry): def __init__(self) -> None: super().__init__() - self._server_id_to_pre_update_hooks: Dict[str, List[Callable[[], None]]] = {} + self._server_name_to_pre_update_hooks: Dict[str, List[Callable[[], None]]] = {} def collect(self) -> Generator[Metric, None, None]: """ Collects metrics, calling pre-update hooks first. """ - for pre_update_hooks in self._server_id_to_pre_update_hooks.values(): + for pre_update_hooks in self._server_name_to_pre_update_hooks.values(): for pre_update_hook in pre_update_hooks: pre_update_hook() @@ -308,8 +308,8 @@ def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: Registers a hook that is called before metric collection. """ - server_hooks = self._server_id_to_pre_update_hooks.setdefault(server_name, []) + server_hooks = self._server_name_to_pre_update_hooks.setdefault(server_name, []) server_hooks.append(hook) def unregister_hooks_for_homeserver(self, server_name: str) -> None: - self._server_id_to_pre_update_hooks.pop(server_name, None) + self._server_name_to_pre_update_hooks.pop(server_name, None) From f814dd041167c8ed4e0d6fa860d6210b8cd312fa Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 17:17:27 -0600 Subject: [PATCH 077/181] Fix incorrect servername change --- synapse/push/bulk_push_rule_evaluator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index ffd9c5083af..ea9169aef02 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -150,7 +150,7 @@ def __init__(self, hs: "HomeServer"): cache_name="room_push_rule_cache", cache=[], # Meaningless size, as this isn't a cache that stores values, resizable=False, - server_name=self.hs.get_instance_id(), + server_name=self.server_name, ) async def _get_rules_for_event( From ccf2585a4c1009cd469dab21aab404a38b3cd561 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 23:18:00 +0000 Subject: [PATCH 078/181] Update synapse/server.py Co-authored-by: Eric Eastwood --- synapse/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index cd8c9959680..7cba045f154 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -372,7 +372,7 @@ async def shutdown(self) -> None: object in the garbage collector. """ - logger.info("Received shutdown request") + logger.info("Received shutdown request for %s (%s).", self.hostname, self.get_instance_id()) unregister_sighups(self._instance_id) From 0c63671545eb2b367a548428ea49a52690f8b46e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 5 Sep 2025 17:20:42 -0600 Subject: [PATCH 079/181] Fix linter error --- synapse/server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index 7cba045f154..69e63e4d220 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -372,7 +372,11 @@ async def shutdown(self) -> None: object in the garbage collector. """ - logger.info("Received shutdown request for %s (%s).", self.hostname, self.get_instance_id()) + logger.info( + "Received shutdown request for %s (%s).", + self.hostname, + self.get_instance_id(), + ) unregister_sighups(self._instance_id) From 267da3eaed51a4c35b1d9b690fc4bc9ed15fd41b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 8 Sep 2025 17:06:25 -0600 Subject: [PATCH 080/181] Refactor clock variable names --- synapse/util/__init__.py | 22 ++++++---- tests/app/test_homeserver_start.py | 67 ++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 648664af958..6efd8cacba4 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -123,10 +123,14 @@ class Clock: """ _reactor: IReactorTime - _call_id: int = 0 + _delayed_call_id: int = 0 + """Unique ID used to track delayed calls""" _looping_calls: List[LoopingCall] = [] - _delayed_calls: Dict[int, IDelayedCall] = {} + """List of active looping calls""" + + _call_id_to_delayed_call: Dict[int, IDelayedCall] = {} + """Mapping from unique call ID to delayed call""" def __init__(self, reactor: IReactorTime) -> None: self._reactor = reactor @@ -134,7 +138,7 @@ def __init__(self, reactor: IReactorTime) -> None: async def sleep(self, seconds: float) -> None: d: defer.Deferred[float] = defer.Deferred() with context.PreserveLoggingContext(): - self._reactor.callLater(seconds, d.callback, seconds) + self.call_later(seconds, d.callback, seconds) await d def time(self) -> float: @@ -242,16 +246,16 @@ def call_later( **kwargs: Key arguments to pass to function. """ - id = self._call_id - self._call_id += 1 + id = self._delayed_call_id + self._delayed_call_id += 1 def wrapped_callback(*args: Any, **kwargs: Any) -> None: callback(*args, **kwargs) - self._delayed_calls.pop(id) + self._call_id_to_delayed_call.pop(id) with context.PreserveLoggingContext(): call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) - self._delayed_calls[id] = call + self._call_id_to_delayed_call[id] = call return call def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: @@ -277,13 +281,13 @@ def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: ignore_errs: Whether to re-raise errors encountered when cancelling the scheduled call. """ - for call in self._delayed_calls.values(): + for call in self._call_id_to_delayed_call.values(): try: call.cancel() except Exception: if not ignore_errs: raise - self._delayed_calls.clear() + self._call_id_to_delayed_call.clear() def log_failure( diff --git a/tests/app/test_homeserver_start.py b/tests/app/test_homeserver_start.py index 9dc20800b22..ac2563ab181 100644 --- a/tests/app/test_homeserver_start.py +++ b/tests/app/test_homeserver_start.py @@ -19,10 +19,15 @@ # # +import gc + import synapse.app.homeserver from synapse.config._base import ConfigError +from synapse.server import HomeServer from tests.config.utils import ConfigFileTestCase +from tests.server import get_clock +from tests.unittest import HomeserverTestCase class HomeserverAppStartTestCase(ConfigFileTestCase): @@ -38,3 +43,65 @@ def test_wrong_start_caught(self) -> None: # Ensure that starting master process with worker config raises an exception with self.assertRaises(ConfigError): synapse.app.homeserver.setup(["-c", self.config_file]) + + +class HomeserverCleanShutdownTests(HomeserverTestCase): + def setUp(self) -> None: + # Override setup to manually create homeserver instance ourself + pass + + def test_homeserver_can_be_garbage_collected(self) -> None: + self.reactor, self.clock = get_clock() + self._hs_args = {"clock": self.clock, "reactor": self.reactor} + self.hs = self.make_homeserver(self.reactor, self.clock) + + # TODO: make legit homeserver so I'm testing actual shutdown, not some fake test + # shutdown scenario + + if self.hs is None: + raise Exception("No homeserver returned from make_homeserver.") + + if not isinstance(self.hs, HomeServer): + raise Exception("A homeserver wasn't returned, but %r" % (self.hs,)) + + self.hs.shutdown() + # del self.helper + # del self.site + # del self.resource + # del self.servlets + del self.hs + + gc.collect() + + gc_objects = gc.get_objects() + for obj in gc_objects: + if isinstance(obj, HomeServer): + # import objgraph + # objgraph.show_backrefs(obj, max_depth=10, too_many=10, extra_info=lambda x: bla(x)) + raise Exception( + "Found HomeServer instance referenced by garbage collector" + ) + + +def bla(x): + if isinstance(x, list): + for k, v in globals().items(): + if v is x: + print(f"Found in globals: {k}") + # print("List:") + # print(x[:3]) + # print(type(x), len(x)) + # for ref in gc.get_referrers(x): + # print(type(ref), repr(ref)[:300]) + # print(objgraph.at(id(x))) + if isinstance(x, dict): + for k, v in globals().items(): + if v is x: + print(f"Found in globals: {k}") + # print("Dict:") + # print(x.__repr__) + # print(type(x), len(x)) + # for ref in gc.get_referrers(x): + # print(type(ref), repr(ref)[:300]) + # print(hex(id(x))) + return hex(id(x)) From 5dce3938e339f453171125b3e94c682f04bfb242 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 8 Sep 2025 17:06:47 -0600 Subject: [PATCH 081/181] Remove old test --- tests/app/test_homeserver_start.py | 107 ----------------------------- 1 file changed, 107 deletions(-) delete mode 100644 tests/app/test_homeserver_start.py diff --git a/tests/app/test_homeserver_start.py b/tests/app/test_homeserver_start.py deleted file mode 100644 index ac2563ab181..00000000000 --- a/tests/app/test_homeserver_start.py +++ /dev/null @@ -1,107 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2021 The Matrix.org Foundation C.I.C. -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -import gc - -import synapse.app.homeserver -from synapse.config._base import ConfigError -from synapse.server import HomeServer - -from tests.config.utils import ConfigFileTestCase -from tests.server import get_clock -from tests.unittest import HomeserverTestCase - - -class HomeserverAppStartTestCase(ConfigFileTestCase): - def test_wrong_start_caught(self) -> None: - # Generate a config with a worker_app - self.generate_config() - # Add a blank line as otherwise the next addition ends up on a line with a comment - self.add_lines_to_config([" "]) - self.add_lines_to_config(["worker_app: test_worker_app"]) - self.add_lines_to_config(["worker_log_config: /data/logconfig.config"]) - self.add_lines_to_config(["instance_map:"]) - self.add_lines_to_config([" main:", " host: 127.0.0.1", " port: 1234"]) - # Ensure that starting master process with worker config raises an exception - with self.assertRaises(ConfigError): - synapse.app.homeserver.setup(["-c", self.config_file]) - - -class HomeserverCleanShutdownTests(HomeserverTestCase): - def setUp(self) -> None: - # Override setup to manually create homeserver instance ourself - pass - - def test_homeserver_can_be_garbage_collected(self) -> None: - self.reactor, self.clock = get_clock() - self._hs_args = {"clock": self.clock, "reactor": self.reactor} - self.hs = self.make_homeserver(self.reactor, self.clock) - - # TODO: make legit homeserver so I'm testing actual shutdown, not some fake test - # shutdown scenario - - if self.hs is None: - raise Exception("No homeserver returned from make_homeserver.") - - if not isinstance(self.hs, HomeServer): - raise Exception("A homeserver wasn't returned, but %r" % (self.hs,)) - - self.hs.shutdown() - # del self.helper - # del self.site - # del self.resource - # del self.servlets - del self.hs - - gc.collect() - - gc_objects = gc.get_objects() - for obj in gc_objects: - if isinstance(obj, HomeServer): - # import objgraph - # objgraph.show_backrefs(obj, max_depth=10, too_many=10, extra_info=lambda x: bla(x)) - raise Exception( - "Found HomeServer instance referenced by garbage collector" - ) - - -def bla(x): - if isinstance(x, list): - for k, v in globals().items(): - if v is x: - print(f"Found in globals: {k}") - # print("List:") - # print(x[:3]) - # print(type(x), len(x)) - # for ref in gc.get_referrers(x): - # print(type(ref), repr(ref)[:300]) - # print(objgraph.at(id(x))) - if isinstance(x, dict): - for k, v in globals().items(): - if v is x: - print(f"Found in globals: {k}") - # print("Dict:") - # print(x.__repr__) - # print(type(x), len(x)) - # for ref in gc.get_referrers(x): - # print(type(ref), repr(ref)[:300]) - # print(hex(id(x))) - return hex(id(x)) From 9202f50db5e19913b98801a77411b7868048c0d2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 8 Sep 2025 17:07:09 -0600 Subject: [PATCH 082/181] Fully cleanup federation on shutdown --- synapse/federation/sender/__init__.py | 7 ++++-- .../sender/per_destination_queue.py | 25 ++++++++++++++++--- .../federation/sender/transaction_manager.py | 12 +++++++++ synapse/federation/transport/client.py | 3 +++ synapse/http/matrixfederationclient.py | 18 +++++++++++-- synapse/server.py | 5 +++- 6 files changed, 61 insertions(+), 9 deletions(-) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index c644f5a28f3..d1aca48c1be 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -469,7 +469,10 @@ def __init__(self, hs: "HomeServer"): self._wake_destinations_needing_catchup, ) + self._is_shutdown = False + def shutdown(self) -> None: + self._is_shutdown = True for queue in self._per_destination_queues.values(): queue.shutdown() @@ -520,7 +523,7 @@ def notify_new_events(self, max_token: RoomStreamToken) -> None: async def _process_event_queue_loop(self) -> None: try: self._is_processing = True - while True: + while not self._is_shutdown: last_token = await self.store.get_federation_out_pos("events") ( next_token, @@ -1106,7 +1109,7 @@ async def _wake_destinations_needing_catchup(self) -> None: last_processed: Optional[str] = None - while True: + while not self._is_shutdown: destinations_to_wake = ( await self.store.get_catch_up_outstanding_destinations(last_processed) ) diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 20787fb0801..12c2d91a012 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -28,6 +28,8 @@ import attr from prometheus_client import Counter +from twisted.internet import defer + from synapse.api.constants import EduTypes from synapse.api.errors import ( FederationDeniedError, @@ -120,6 +122,7 @@ def __init__( self._destination = destination self.transmission_loop_running = False self._transmission_loop_enabled = True + self.active_transmission_loop: Optional[defer.Deferred] = None # Flag to signal to any running transmission loop that there is new data # queued up to be sent. @@ -177,6 +180,16 @@ def __str__(self) -> str: def shutdown(self) -> None: """Instruct the queue to stop processing any further requests""" self._transmission_loop_enabled = False + # The transaction manager must be shutdown before cancelling the active + # transmission loop. Otherwise the transmission loop can enter a new cycle of + # sleeping before retrying since the shutdown flag of the _transaction_manager + # hasn't been set yet. + self._transaction_manager.shutdown() + try: + if self.active_transmission_loop is not None: + self.active_transmission_loop.cancel() + except Exception: + pass def pending_pdu_count(self) -> int: return len(self._pending_pdus) @@ -316,9 +329,13 @@ def attempt_new_transaction(self) -> None: ) return + if not self._transmission_loop_enabled: + logger.warning("Shutdown has been requested. Not sending transaction") + return + logger.debug("TX [%s] Starting transaction loop", self._destination) - run_as_background_process( + self.active_transmission_loop = run_as_background_process( "federation_transaction_transmission_loop", self.server_name, self._transaction_transmission_loop, @@ -328,7 +345,6 @@ async def _transaction_transmission_loop(self) -> None: pending_pdus: List[EventBase] = [] try: self.transmission_loop_running = True - # This will throw if we wouldn't retry. We do this here so we fail # quickly, but we will later check this again in the http client, # hence why we throw the result away. @@ -359,8 +375,8 @@ async def _transaction_transmission_loop(self) -> None: # If we've gotten told about new things to send during # checking for things to send, we try looking again. # Otherwise new PDUs or EDUs might arrive in the meantime, - # but not get sent because we hold the - # `transmission_loop_running` flag. + # but not get sent because we currently have an + # `_active_transmission_loop` running. if self._new_data_to_send: continue else: @@ -449,6 +465,7 @@ async def _transaction_transmission_loop(self) -> None: ) finally: # We want to be *very* sure we clear this after we stop processing + self.active_transmission_loop = None self.transmission_loop_running = False async def _catch_up_transmission_loop(self) -> None: diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index 63ed13c6fa6..7d8cabf690d 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -72,6 +72,12 @@ def __init__(self, hs: "synapse.server.HomeServer"): # HACK to get unique tx id self._next_txn_id = int(self.clock.time_msec()) + self._is_shutdown = False + + def shutdown(self) -> None: + self._is_shutdown = True + self._transport_layer.shutdown() + @measure_func("_send_new_transaction") async def send_new_transaction( self, @@ -86,6 +92,12 @@ async def send_new_transaction( edus: List of EDUs to send """ + if self._is_shutdown: + logger.warning( + "TransactionManager has been shutdown, not sending transaction" + ) + return + # Make a transaction-sending opentracing span. This span follows on from # all the edus in that transaction. This needs to be done since there is # no active span here, so if the edus were not received by the remote the diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index 62bf96ce913..95113ff6139 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -70,6 +70,9 @@ def __init__(self, hs: "HomeServer"): self.client = hs.get_federation_http_client() self._is_mine_server_name = hs.is_mine_server_name + def shutdown(self) -> None: + self.client.shutdown() + async def get_room_state_ids( self, destination: str, room_id: str, event_id: str ) -> JsonDict: diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5c876025187..b239c51bab7 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -483,6 +483,10 @@ def __init__( ) self.remote_download_linearizer = Linearizer("remote_download_linearizer", 6) + self._is_shutdown = False + + def shutdown(self) -> None: + self._is_shutdown = True def wake_destination(self, destination: str) -> None: """Called when the remote server may have come back online.""" @@ -674,7 +678,7 @@ async def _send_request( (b"", b"", path_bytes, None, query_bytes, b"") ) - while True: + while not self._is_shutdown: try: json = request.get_json() if json: @@ -856,13 +860,23 @@ async def _send_request( ) delay_seconds *= random.uniform(0.8, 1.4) - logger.debug( + logger.info( "{%s} [%s] Waiting %ss before re-sending...", request.txn_id, request.destination, delay_seconds, ) + if self._is_shutdown: + # Immediately fail sending the request instead of starting a + # potentially long sleep after the server has requested + # shutdown. + # This is the code path followed when the + # `federation_transaction_transmission_loop` has been + # cancelled. + raise + + logger.warning("Sleepy time") # Sleep for the calculated delay, or wake up immediately # if we get notified that the server is back up. await self._sleeper.sleep( diff --git a/synapse/server.py b/synapse/server.py index 69e63e4d220..f67b1450b5c 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -416,7 +416,10 @@ async def shutdown(self) -> None: db.stop_background_updates() if self.should_send_federation(): - self.get_federation_sender().shutdown() + try: + self.get_federation_sender().shutdown() + except Exception: + pass self.get_clock().cancel_all_looping_calls() self.get_clock().cancel_all_delayed_calls() From 9403bbdd9df3f70705ff2a35f2f8ca91d65d217d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 9 Sep 2025 09:49:11 -0600 Subject: [PATCH 083/181] Make clock variables instance specific --- synapse/util/__init__.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 6efd8cacba4..7f73465a765 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -123,17 +123,17 @@ class Clock: """ _reactor: IReactorTime - _delayed_call_id: int = 0 - """Unique ID used to track delayed calls""" - - _looping_calls: List[LoopingCall] = [] - """List of active looping calls""" - - _call_id_to_delayed_call: Dict[int, IDelayedCall] = {} - """Mapping from unique call ID to delayed call""" def __init__(self, reactor: IReactorTime) -> None: self._reactor = reactor + self._delayed_call_id: int = 0 + """Unique ID used to track delayed calls""" + + self._looping_calls: List[LoopingCall] = [] + """List of active looping calls""" + + self._call_id_to_delayed_call: Dict[int, IDelayedCall] = {} + """Mapping from unique call ID to delayed call""" async def sleep(self, seconds: float) -> None: d: defer.Deferred[float] = defer.Deferred() @@ -246,16 +246,16 @@ def call_later( **kwargs: Key arguments to pass to function. """ - id = self._delayed_call_id - self._delayed_call_id += 1 + call_id = self._delayed_call_id + self._delayed_call_id = self._delayed_call_id + 1 def wrapped_callback(*args: Any, **kwargs: Any) -> None: callback(*args, **kwargs) - self._call_id_to_delayed_call.pop(id) + self._call_id_to_delayed_call.pop(call_id) with context.PreserveLoggingContext(): call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) - self._call_id_to_delayed_call[id] = call + self._call_id_to_delayed_call[call_id] = call return call def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: From dafba10620cdfdc2fc5c697a23e66080b09dc5b6 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 9 Sep 2025 13:39:34 -0600 Subject: [PATCH 084/181] Readd metric_name to metrics hooks --- synapse/federation/sender/__init__.py | 6 +++-- synapse/http/matrixfederationclient.py | 1 - synapse/util/caches/__init__.py | 3 ++- synapse/util/metrics.py | 31 +++++++++++++++++++++----- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 136a937a118..66fa46f32b0 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -461,6 +461,10 @@ def __init__(self, hs: "HomeServer"): self, self.server_name, self.clock, max_delay_s=rr_txn_interval_per_room_s ) + # It is important for `_is_shutdown` to be instantiated before the looping call + # for `wake_destinations_needing_catchup`. + self._is_shutdown = False + # Regularly wake up destinations that have outstanding PDUs to be caught up self.clock.looping_call_now( run_as_background_process, @@ -470,8 +474,6 @@ def __init__(self, hs: "HomeServer"): self._wake_destinations_needing_catchup, ) - self._is_shutdown = False - def shutdown(self) -> None: self._is_shutdown = True for queue in self._per_destination_queues.values(): diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b239c51bab7..8c2528bf029 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -876,7 +876,6 @@ async def _send_request( # cancelled. raise - logger.warning("Sleepy time") # Sleep for the calculated delay, or wake up immediately # if we get notified that the server is back up. await self._sleeper.sleep( diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index a7984981e2f..08ff842af0f 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -243,7 +243,8 @@ def register_cache( server_name=server_name, collect_callback=collect_callback, ) - CACHE_METRIC_REGISTRY.register_hook(server_name, metric.collect) + metric_name = "cache_%s_%s_%s" % (cache_type, cache_name, server_name) + CACHE_METRIC_REGISTRY.register_hook(server_name, metric_name, metric.collect) return metric diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index fd8955c9d0c..7911643fbf0 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -27,7 +27,6 @@ Callable, Dict, Generator, - List, Optional, Protocol, Type, @@ -290,7 +289,13 @@ class DynamicCollectorRegistry(CollectorRegistry): def __init__(self) -> None: super().__init__() - self._server_name_to_pre_update_hooks: Dict[str, List[Callable[[], None]]] = {} + self._server_name_to_pre_update_hooks: Dict[ + str, Dict[str, Callable[[], None]] + ] = {} + """ + Mapping of server name to a mapping of metric name to metric pre-update + hook + """ def collect(self) -> Generator[Metric, None, None]: """ @@ -298,18 +303,32 @@ def collect(self) -> Generator[Metric, None, None]: """ for pre_update_hooks in self._server_name_to_pre_update_hooks.values(): - for pre_update_hook in pre_update_hooks: + for pre_update_hook in pre_update_hooks.values(): pre_update_hook() yield from super().collect() - def register_hook(self, server_name: str, hook: Callable[[], None]) -> None: + def register_hook( + self, server_name: str, metric_name: str, hook: Callable[[], None] + ) -> None: """ Registers a hook that is called before metric collection. """ - server_hooks = self._server_name_to_pre_update_hooks.setdefault(server_name, []) - server_hooks.append(hook) + server_hooks = self._server_name_to_pre_update_hooks.setdefault(server_name, {}) + if server_hooks.get(metric_name) is not None: + # TODO: This should be an `assert` since registering the same metric name + # multiple times will clobber the old metric. + # We currently rely on this behaviour as we instantiate multiple + # `SyncRestServlet`, one per listener, and in the `__init__` we setup a new + # LruCache. + # Once the above behaviour is changed, this should be changed to an `assert`. + logger.error( + "Metric named %s already registered for server %s", + metric_name, + server_name, + ) + server_hooks[metric_name] = hook def unregister_hooks_for_homeserver(self, server_name: str) -> None: self._server_name_to_pre_update_hooks.pop(server_name, None) From da6f85ee5d513f02ab1ef367a0631966dd050b4f Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 9 Sep 2025 14:05:47 -0600 Subject: [PATCH 085/181] Force shutdown handler registration to use keyword arguments --- synapse/app/_base.py | 6 +++- synapse/handlers/presence.py | 10 ++++-- synapse/server.py | 33 +++++++------------- synapse/storage/databases/main/client_ips.py | 8 ++--- synapse/storage/databases/main/lock.py | 5 ++- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index aa9f75cbf1b..914c8981f63 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -637,7 +637,11 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: # Log when we start the shut down process. hs.register_sync_shutdown_handler( - "before", "shutdown", "shutdown log entry", logger.info, "Shutting down..." + phase="before", + eventType="shutdown", + desc="shutdown log entry", + shutdown_func=logger.info, + msg="Shutting down...", ) setup_sentry(hs) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 06ab3454cd7..e8bff295bbf 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -540,7 +540,10 @@ def __init__(self, hs: "HomeServer"): self.clock.looping_call(self.send_stop_syncing, UPDATE_SYNCING_USERS_MS) hs.register_async_shutdown_handler( - "before", "shutdown", "generic_presence.on_shutdown", self._on_shutdown + phase="before", + eventType="shutdown", + desc="generic_presence.on_shutdown", + shutdown_func=self._on_shutdown, ) @wrap_as_background_process("WorkerPresenceHandler._on_shutdown") @@ -837,7 +840,10 @@ def __init__(self, hs: "HomeServer"): self.unpersisted_users_changes: Set[str] = set() hs.register_async_shutdown_handler( - "before", "shutdown", "presence.on_shutdown", self._on_shutdown + phase="before", + eventType="shutdown", + desc="presence.on_shutdown", + shutdown_func=self._on_shutdown, ) # Keeps track of the number of *ongoing* syncs on this process. While diff --git a/synapse/server.py b/synapse/server.py index 80894f57dc7..a025573afe2 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -276,7 +276,6 @@ class ShutdownInfo: desc: str func: Callable[..., Any] trigger_id: _SystemEventID - args: Tuple[object, ...] kwargs: Dict[str, object] @@ -427,11 +426,7 @@ async def shutdown(self) -> None: for shutdown_handler in self._async_shutdown_handlers: try: self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - defer.ensureDeferred( - shutdown_handler.func( - *shutdown_handler.args, **shutdown_handler.kwargs - ) - ) + defer.ensureDeferred(shutdown_handler.func(**shutdown_handler.kwargs)) except Exception: pass self._async_shutdown_handlers.clear() @@ -439,18 +434,18 @@ async def shutdown(self) -> None: for shutdown_handler in self._sync_shutdown_handlers: try: self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) - shutdown_handler.func(*shutdown_handler.args, **shutdown_handler.kwargs) + shutdown_handler.func(**shutdown_handler.kwargs) except Exception: pass self._sync_shutdown_handlers.clear() def register_async_shutdown_handler( self, + *, phase: str, eventType: str, desc: "LiteralString", shutdown_func: Callable[..., Any], - *args: object, **kwargs: object, ) -> None: """ @@ -464,11 +459,10 @@ def register_async_shutdown_handler( desc, self.config.server.server_name, shutdown_func, + **kwargs, ) self._async_shutdown_handlers.append( - ShutdownInfo( - desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs - ) + ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, kwargs=kwargs) ) def cleanup_metrics(self) -> None: @@ -488,11 +482,11 @@ def cleanup_metrics(self) -> None: def register_sync_shutdown_handler( self, + *, phase: str, eventType: str, desc: "LiteralString", shutdown_func: Callable[..., Any], - *args: object, **kwargs: object, ) -> None: """ @@ -503,13 +497,10 @@ def register_sync_shutdown_handler( phase, eventType, shutdown_func, - args, - kwargs, + **kwargs, ) self._sync_shutdown_handlers.append( - ShutdownInfo( - desc=desc, func=shutdown_func, trigger_id=id, args=args, kwargs=kwargs - ) + ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, kwargs=kwargs) ) def register_module_web_resource(self, path: str, resource: Resource) -> None: @@ -1194,10 +1185,10 @@ def get_media_sender_thread_pool(self) -> ThreadPool: media_threadpool.start() self.register_sync_shutdown_handler( - "during", - "shutdown", - "Homeserver media_threadpool.stop", - media_threadpool.stop, + phase="during", + eventType="shutdown", + desc="Homeserver media_threadpool.stop", + shutdown_func=media_threadpool.stop, ) # Register the threadpool with our metrics. diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 2d8cf9f98c8..86328af62f6 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -454,10 +454,10 @@ def __init__( self._clock.looping_call(self._update_client_ips_batch, 5 * 1000) hs.register_async_shutdown_handler( - "before", - "shutdown", - "ClientIpWorkerStore _update_client_ips_batch", - self._update_client_ips_batch, + phase="before", + eventType="shutdown", + desc="ClientIpWorkerStore _update_client_ips_batch", + shutdown_func=self._update_client_ips_batch, ) @wrap_as_background_process("prune_old_user_ips") diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index 2d7bc4ab192..50634cd5cbb 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -100,7 +100,10 @@ def __init__( # However, a) it should be a small window, b) the lock is best effort # anyway and c) we want to really avoid leaking locks when we restart. hs.register_async_shutdown_handler( - "before", "shutdown", "LockStore _on_shutdown", self._on_shutdown + phase="before", + eventType="shutdown", + desc="LockStore _on_shutdown", + shutdown_func=self._on_shutdown, ) self._acquiring_locks: Set[Tuple[str, str]] = set() From a86bfe0cb3a31806004ed0154ef321914c989702 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 9 Sep 2025 14:23:02 -0600 Subject: [PATCH 086/181] Remove unnecessary TODO --- synapse/server.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index a025573afe2..acfdea5bb3f 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -476,10 +476,6 @@ def cleanup_metrics(self) -> None: self.config.server.server_name ) - # TODO: What about the other gauge types? - # from synapse.metrics import all_gauges - # all_gauges.clear() - def register_sync_shutdown_handler( self, *, From a9df2ba5ff0a1785e6b85e71e492f816d826b6da Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 10 Sep 2025 15:37:47 -0600 Subject: [PATCH 087/181] Add lint for using our internal Clock --- scripts-dev/mypy_synapse_plugin.py | 67 +++++++++++++++++++++++++++ synapse/metrics/_gc.py | 4 +- tests/server.py | 10 ++-- tests/util/caches/test_descriptors.py | 2 +- 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 610dec415af..cff5987a6f4 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -68,6 +68,18 @@ category="per-homeserver-tenant-metrics", ) +INTERNAL_CLOCK_CALL_LATER_NOT_USED = ErrorCode( + "call-later-not-tracked", + "`synapse.util.Clock.call_later` should be used instead of `reactor.callLater`", + category="delayed-call-tracking", +) + +INTERNAL_CLOCK_LOOPING_CALL_NOT_USED = ErrorCode( + "looping-call-not-tracked", + "`synapse.util.Clock.looping_call` should be used instead of `task.LoopingCall`", + category="delayed-call-tracking", +) + class Sentinel(enum.Enum): # defining a sentinel in this way allows mypy to correctly handle the @@ -210,6 +222,9 @@ def get_function_signature_hook( # callback, let's just pass it in while we have it. return lambda ctx: check_prometheus_metric_instantiation(ctx, fullname) + if "twisted.internet.task.LoopingCall" == fullname: + return check_looping_call + return None def get_method_signature_hook( @@ -229,9 +244,61 @@ def get_method_signature_hook( ): return check_is_cacheable_wrapper + if fullname.startswith(("twisted.internet.interfaces.IReactorTime.callLater",)): + return check_call_later + return None +def check_call_later(ctx: MethodSigContext) -> CallableType: + """ + Ensure that the `reactor.callLater` callsites are used intentionally. + + Using `synapse.util.Clock.call_later` should be preferred. This is because the + `synapse.util.Clock` tracks delayed calls in order to cancel any outstanding calls + during server shutdown. Delayed calls which are either short lived (<~60s) or + frequently called and can be tracked via other means could be candidates for using + `reactor.callLater` directly. In those cases, use a type ignore comment to disable the + check, e.g. `# type: ignore[call-later-not-tracked]`. + + Args: + ctx: The `FunctionSigContext` from mypy. + fullname: The fully qualified name of the function being called, + e.g. `"twisted.internet.interfaces.IReactorTime.callLater"` + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected all `reactor.callLater` calls to use `synapse.util.Clock.call_later` instead. This is so that long lived calls can be tracked for cancellation during server shutdown", + ctx.context, + code=INTERNAL_CLOCK_CALL_LATER_NOT_USED, + ) + + return signature + + +def check_looping_call(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that the `task.LoopingCall` callsites are used intentionally. + + Using `synapse.util.Clock.looping_call` should be preferred. This is because the + `synapse.util.Clock` tracks looping calls in order to cancel any outstanding calls + during server shutdown. + + Args: + ctx: The `FunctionSigContext` from mypy. + fullname: The fully qualified name of the function being called, + e.g. `"twisted.internet.task.LoopingCall"` + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected all `task.LoopingCall` instances to use `synapse.util.Clock.looping_call` instead. This is so that long lived calls can be tracked for cancellation during server shutdown", + ctx.context, + code=INTERNAL_CLOCK_LOOPING_CALL_NOT_USED, + ) + + return signature + + def analyze_prometheus_metric_classes(ctx: ClassDefContext) -> None: """ Cross-check the list of Prometheus metric classes against the diff --git a/synapse/metrics/_gc.py b/synapse/metrics/_gc.py index e7783b05e6d..820dbca75ba 100644 --- a/synapse/metrics/_gc.py +++ b/synapse/metrics/_gc.py @@ -138,7 +138,9 @@ def _maybe_gc() -> None: gc_time.labels(i).observe(end - start) gc_unreachable.labels(i).set(unreachable) - gc_task = task.LoopingCall(_maybe_gc) + # We can ignore the lint here since this looping call does not hold a `HomeServer` + # reference so can be cleaned up by other means on shutdown. + gc_task = task.LoopingCall(_maybe_gc) # type: ignore[looping-call-not-tracked] gc_task.start(0.1) diff --git a/tests/server.py b/tests/server.py index d7293626787..f038bdf069b 100644 --- a/tests/server.py +++ b/tests/server.py @@ -781,7 +781,7 @@ def _(res: Any) -> None: d: "Deferred[None]" = Deferred() d.addCallback(lambda x: function(*args, **kwargs)) d.addBoth(_) - self._reactor.callLater(0, d.callback, True) + self._reactor.callLater(0, d.callback, True) # type: ignore[call-later-not-tracked] return d @@ -899,10 +899,10 @@ def _produce() -> None: # some implementations of IProducer (for example, FileSender) # don't return a deferred. d = maybeDeferred(self.producer.resumeProducing) - d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) + d.addCallback(lambda: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] if not streaming: - self._reactor.callLater(0.0, _produce) + self._reactor.callLater(0.0, _produce) # type: ignore[call-later-not-tracked] def write(self, byt: bytes) -> None: if self.disconnecting: @@ -914,7 +914,7 @@ def write(self, byt: bytes) -> None: # TLSMemoryBIOProtocol) get very confused if a read comes back while they are # still doing a write. Doing a callLater here breaks the cycle. if self.autoflush: - self._reactor.callLater(0.0, self.flush) + self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] def writeSequence(self, seq: Iterable[bytes]) -> None: for x in seq: @@ -944,7 +944,7 @@ def flush(self, maxbytes: Optional[int] = None) -> None: self.buffer = self.buffer[len(to_write) :] if self.buffer and self.autoflush: - self._reactor.callLater(0.0, self.flush) + self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] if not self.buffer and self.disconnecting: logger.info("FakeTransport: Buffer now empty, completing disconnect") diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py index 7865a677093..fd9e4830981 100644 --- a/tests/util/caches/test_descriptors.py +++ b/tests/util/caches/test_descriptors.py @@ -56,7 +56,7 @@ def run_on_reactor() -> "Deferred[int]": d: "Deferred[int]" = Deferred() - cast(IReactorTime, reactor).callLater(0, d.callback, 0) + cast(IReactorTime, reactor).callLater(0, d.callback, 0) # type: ignore[call-later-not-tracked] return make_deferred_yieldable(d) From 2337b64e6cbdccc6bc7052eae7608b59f61efda9 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 10 Sep 2025 16:44:51 -0600 Subject: [PATCH 088/181] Change Clock to conditionally track calls for cleanup --- synapse/app/phone_stats_home.py | 8 ++- synapse/appservice/scheduler.py | 1 + synapse/handlers/delayed_events.py | 4 +- synapse/handlers/message.py | 12 +++- synapse/handlers/presence.py | 3 +- synapse/handlers/room_member.py | 2 +- synapse/handlers/stats.py | 2 +- synapse/handlers/user_directory.py | 13 +++- synapse/handlers/worker_lock.py | 2 +- synapse/http/client.py | 2 +- .../federation/matrix_federation_agent.py | 2 - synapse/module_api/__init__.py | 1 + synapse/push/emailpusher.py | 2 +- synapse/push/httppusher.py | 2 +- synapse/server.py | 3 +- synapse/storage/database.py | 2 + synapse/storage/databases/main/cache.py | 3 +- .../storage/databases/main/registration.py | 1 + synapse/storage/databases/main/roommember.py | 1 + synapse/util/__init__.py | 61 ++++++++++++++----- synapse/util/async_helpers.py | 18 ++++-- synapse/util/caches/response_cache.py | 2 +- synapse/util/ratelimitutils.py | 2 +- synapse/util/task_scheduler.py | 2 +- tests/server.py | 2 +- tests/test_test_utils.py | 8 +-- tests/utils.py | 1 + 27 files changed, 116 insertions(+), 46 deletions(-) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 69d3ac78fd4..38ae8d85f0d 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -285,10 +285,14 @@ async def _generate_monthly_active_users() -> None: # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process - clock.call_later(0, performance_stats_init) + clock.call_later(0, False, performance_stats_init) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later( - INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, phone_stats_home, hs, stats + INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, + True, + phone_stats_home, + hs, + stats, ) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 01f77c4cb6d..405df1e6046 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -513,6 +513,7 @@ def recover(self) -> None: logger.info("Scheduling retries on %s in %fs", self.service.id, delay) self.scheduled_recovery = self.clock.call_later( delay, + True, run_as_background_process, "as-recoverer", self.server_name, diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index a6749801a50..58fffa17668 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -43,6 +43,7 @@ UserID, create_requester, ) +from synapse.util import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from synapse.util.events import generate_fake_event_id from synapse.util.metrics import Measure @@ -90,7 +91,7 @@ async def _schedule_db_events() -> None: hs.get_notifier().add_replication_callback(self.notify_new_event) # Kick off again (without blocking) to catch any missed notifications # that may have fired before the callback was added. - self._clock.call_later(0, self.notify_new_event) + self._clock.call_later(0, False, self.notify_new_event) # Delayed events that are already marked as processed on startup might not have been # sent properly on the last run of the server, so unmark them to send them again. @@ -453,6 +454,7 @@ def _schedule_next_at(self, next_send_ts: Timestamp) -> None: if self._next_delayed_event_call is None: self._next_delayed_event_call = self._clock.call_later( delay_sec, + True if delay_sec > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "_send_on_timeout", self.server_name, diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7a342565acf..71c60943364 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -81,7 +81,13 @@ create_requester, ) from synapse.types.state import StateFilter -from synapse.util import json_decoder, json_encoder, log_failure, unwrapFirstError +from synapse.util import ( + CALL_LATER_DELAY_TRACKING_THRESHOLD_S, + json_decoder, + json_encoder, + log_failure, + unwrapFirstError, +) from synapse.util.async_helpers import Linearizer, gather_results from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.metrics import measure_func @@ -441,8 +447,12 @@ def _schedule_expiry_for_event(self, event_id: str, expiry_ts: int) -> None: logger.info("Scheduling expiry for event %s in %.3fs", event_id, delay) + track_for_shutdown = ( + True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False + ) self._scheduled_expiry = self.clock.call_later( delay, + track_for_shutdown, run_as_background_process, "_expire_event", self.server_name, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index e8bff295bbf..ff020a2b485 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -876,7 +876,7 @@ def __init__(self, hs: "HomeServer"): # The initial delay is to allow disconnected clients a chance to # reconnect before we treat them as offline. self.clock.call_later( - 30, self.clock.looping_call, self._handle_timeouts, 5000 + 30, False, self.clock.looping_call, self._handle_timeouts, 5000 ) # Presence information is persisted, whether or not it is being tracked @@ -884,6 +884,7 @@ def __init__(self, hs: "HomeServer"): if self._presence_enabled: self.clock.call_later( 60, + False, self.clock.looping_call, self._persist_unpersisted_changes, 60 * 1000, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 3fc7a992643..2a95caca6a1 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -2191,7 +2191,7 @@ def __init__(self, hs: "HomeServer"): self._notifier.add_replication_callback(self.notify_new_event) # We kick this off to pick up outstanding work from before the last restart. - self._clock.call_later(0, self.notify_new_event) + self._clock.call_later(0, False, self.notify_new_event) def notify_new_event(self) -> None: """Called when there may be more deltas to process""" diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index a2602ea818e..c9e39036648 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -75,7 +75,7 @@ def __init__(self, hs: "HomeServer"): # We kick this off so that we don't have to wait for a change before # we start populating stats - self.clock.call_later(0, self.notify_new_event) + self.clock.call_later(0, False, self.notify_new_event) def notify_new_event(self) -> None: """Called when there may be more deltas to process""" diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 130099a2390..56470ce833c 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -41,6 +41,7 @@ from synapse.storage.databases.main.user_directory import SearchResult from synapse.storage.roommember import ProfileInfo from synapse.types import UserID +from synapse.util import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from synapse.util.metrics import Measure from synapse.util.retryutils import NotRetryingDestination from synapse.util.stringutils import non_null_str_or_none @@ -137,11 +138,11 @@ def __init__(self, hs: "HomeServer"): # We kick this off so that we don't have to wait for a change before # we start populating the user directory - self.clock.call_later(0, self.notify_new_event) + self.clock.call_later(0, False, self.notify_new_event) # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( - 10, self.kick_off_remote_profile_refresh_process + 10, False, self.kick_off_remote_profile_refresh_process ) async def search_users( @@ -558,12 +559,14 @@ async def _handle_possible_remote_profile_change( # the profile if the server only just sent us an event. self.clock.call_later( USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, + True, self.kick_off_remote_profile_refresh_process_for_remote_server, UserID.from_string(user_id).domain, ) # Schedule a wake-up to handle any backoffs that may occur in the future. self.clock.call_later( 2 * USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, + True, self.kick_off_remote_profile_refresh_process, ) return @@ -623,6 +626,7 @@ async def _unsafe_refresh_remote_profiles(self) -> None: # Come back later. self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, + False, self.kick_off_remote_profile_refresh_process, ) return @@ -655,8 +659,10 @@ async def _unsafe_refresh_remote_profiles(self) -> None: if not users: return _, _, next_try_at_ts = users[0] + delay = ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2 self._refresh_remote_profiles_call_later = self.clock.call_later( - ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2, + delay, + True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, self.kick_off_remote_profile_refresh_process, ) @@ -669,6 +675,7 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, + False, self.kick_off_remote_profile_refresh_process, ) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 61443978d70..ecbde72ed1a 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -197,7 +197,7 @@ def _wake_all_locks( if not deferred.called: deferred.callback(None) - self._clock.call_later(0, _wake_all_locks, locks) + self._clock.call_later(0, False, _wake_all_locks, locks) @wrap_as_background_process("_cleanup_locks") async def _cleanup_locks(self) -> None: diff --git a/synapse/http/client.py b/synapse/http/client.py index ca0f5052247..542dae7a7f2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -171,7 +171,7 @@ def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCa """ def _scheduler(x: Callable[[], object]) -> IDelayedCall: - return clock.call_later(_EPSILON, x) + return clock.call_later(_EPSILON, False, x) return _scheduler diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 6ebadf0dbf2..8353a8b1adb 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -49,7 +49,6 @@ from synapse.http.proxyagent import ProxyAgent from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.types import ISynapseReactor -from synapse.util import Clock logger = logging.getLogger(__name__) @@ -124,7 +123,6 @@ def __init__( # addresses, to prevent DNS rebinding. reactor = BlocklistingReactorWrapper(reactor, ip_allowlist, ip_blocklist) - self._clock = Clock(reactor) self._pool = HTTPConnectionPool(reactor) self._pool.retryAutomatically = False self._pool.maxPersistentPerHost = 5 diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 9309aa93942..814a0747360 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -1448,6 +1448,7 @@ def delayed_background_call( return self._clock.call_later( # convert ms to seconds as needed by call_later. msec * 0.001, + False, self.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 8f5409bcd96..9246518ef73 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -229,7 +229,7 @@ async def _unsafe_process(self) -> None: if soonest_due_at is not None: self.timed_call = self.hs.get_clock().call_later( - self.seconds_until(soonest_due_at), self.on_timer + self.seconds_until(soonest_due_at), True, self.on_timer ) async def save_last_stream_ordering_and_success( diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 9e91c0e5610..abad888cebb 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -337,7 +337,7 @@ async def _unsafe_process(self) -> None: else: logger.info("Push failed: delaying for %ds", self.backoff_delay) self.timed_call = self.hs.get_clock().call_later( - self.backoff_delay, self.on_timer + self.backoff_delay, True, self.on_timer ) self.backoff_delay = min( self.backoff_delay * 2, self.MAX_BACKOFF_SEC diff --git a/synapse/server.py b/synapse/server.py index acfdea5bb3f..f14a88eb873 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -420,8 +420,7 @@ async def shutdown(self) -> None: except Exception: pass - self.get_clock().cancel_all_looping_calls() - self.get_clock().cancel_all_delayed_calls() + self.get_clock().shutdown() for shutdown_handler in self._async_shutdown_handlers: try: diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 26e10c1353c..816af268433 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -638,6 +638,7 @@ def __init__( # background updates of tables that aren't safe to update. self._clock.call_later( 0.0, + False, run_as_background_process, "upsert_safety_check", self.server_name, @@ -687,6 +688,7 @@ async def _check_safe_to_upsert(self) -> None: if background_update_names: self._clock.call_later( 15.0, + False, run_as_background_process, "upsert_safety_check", self.server_name, diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 77949268125..98871aba94a 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -133,6 +133,7 @@ def __init__( ): self.hs.get_clock().call_later( CATCH_UP_CLEANUP_INTERVAL_MS / 1000, + True, self._clean_up_cache_invalidation_wrapper, ) @@ -796,7 +797,7 @@ async def _clean_up_cache_invalidation_wrapper(self) -> None: next_interval = REGULAR_CLEANUP_INTERVAL_MS self.hs.get_clock().call_later( - next_interval / 1000, self._clean_up_cache_invalidation_wrapper + next_interval / 1000, True, self._clean_up_cache_invalidation_wrapper ) async def _clean_up_batch_of_old_cache_invalidations( diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 117444e7b75..db4e2a82fd3 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -214,6 +214,7 @@ def __init__( if hs.config.worker.run_background_tasks: self._clock.call_later( 0.0, + False, self._set_expiration_date_when_missing, ) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9db2e14a06f..46fc5d4cb94 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -121,6 +121,7 @@ def __init__( ) self.hs.get_clock().call_later( 1, + False, self._count_known_servers, ) federation_known_servers_gauge.register_hook( diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 7f73465a765..ad30b016bcf 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -109,6 +109,9 @@ def unwrapFirstError(failure: Failure) -> Failure: P = ParamSpec("P") +CALL_LATER_DELAY_TRACKING_THRESHOLD_S = 60 + + class Clock: """ A Clock wraps a Twisted reactor and provides utilities on top of it. @@ -126,6 +129,7 @@ class Clock: def __init__(self, reactor: IReactorTime) -> None: self._reactor = reactor + self._delayed_call_id: int = 0 """Unique ID used to track delayed calls""" @@ -135,10 +139,18 @@ def __init__(self, reactor: IReactorTime) -> None: self._call_id_to_delayed_call: Dict[int, IDelayedCall] = {} """Mapping from unique call ID to delayed call""" - async def sleep(self, seconds: float) -> None: + self._is_shutdown = False + """Whether shutdown has been requested by the HomeServer""" + + def shutdown(self) -> None: + self._is_shutdown = True + self.cancel_all_looping_calls() + self.cancel_all_delayed_calls() + + async def sleep(self, seconds: float, track_for_shutdown: bool = False) -> None: d: defer.Deferred[float] = defer.Deferred() with context.PreserveLoggingContext(): - self.call_later(seconds, d.callback, seconds) + self.call_later(seconds, track_for_shutdown, d.callback, seconds) await d def time(self) -> float: @@ -208,7 +220,11 @@ def _looping_call_common( **kwargs: P.kwargs, ) -> LoopingCall: """Common functionality for `looping_call` and `looping_call_now`""" - call = task.LoopingCall(f, *args, **kwargs) + if self._is_shutdown: + raise Exception("Cannot start looping call. Clock has been shutdown") + # We can ignore the lint here since this is the one location LoopingCall's + # should be created. + call = task.LoopingCall(f, *args, **kwargs) # type: ignore[looping-call-not-tracked] call.clock = self._reactor d = call.start(msec / 1000.0, now=now) d.addErrback(log_failure, "Looping call died", consumeErrors=False) @@ -232,7 +248,12 @@ def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: self._looping_calls.clear() def call_later( - self, delay: float, callback: Callable, *args: Any, **kwargs: Any + self, + delay: float, + track_for_shutdown: bool, + callback: Callable, + *args: Any, + **kwargs: Any, ) -> IDelayedCall: """Call something later @@ -246,17 +267,27 @@ def call_later( **kwargs: Key arguments to pass to function. """ - call_id = self._delayed_call_id - self._delayed_call_id = self._delayed_call_id + 1 - - def wrapped_callback(*args: Any, **kwargs: Any) -> None: - callback(*args, **kwargs) - self._call_id_to_delayed_call.pop(call_id) - - with context.PreserveLoggingContext(): - call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) - self._call_id_to_delayed_call[call_id] = call - return call + if self._is_shutdown: + raise Exception("Cannot start delayed call. Clock has been shutdown") + + if track_for_shutdown: + call_id = self._delayed_call_id + self._delayed_call_id = self._delayed_call_id + 1 + + def wrapped_callback(*args: Any, **kwargs: Any) -> None: + callback(*args, **kwargs) + self._call_id_to_delayed_call.pop(call_id) + + with context.PreserveLoggingContext(): + # We can ignore the lint here since this is the one location callLater + # should be called. + call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] + self._call_id_to_delayed_call[call_id] = call + return call + else: + # We can ignore the lint here since this is the one location callLater should + # be called. + return self._reactor.callLater(delay, callback, *args, **kwargs) # type: ignore[call-later-not-tracked] def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: """ diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 44696fc50e2..6bf8ad72253 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -65,7 +65,7 @@ run_coroutine_in_background, run_in_background, ) -from synapse.util import Clock +from synapse.util import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock logger = logging.getLogger(__name__) @@ -804,7 +804,7 @@ def time_it_out() -> None: if not new_d.called: new_d.errback(defer.TimeoutError("Timed out after %gs" % (timeout,))) - delayed_call = clock.call_later(timeout, time_it_out) + delayed_call = clock.call_later(timeout, False, time_it_out) def convert_cancelled(value: Failure) -> Failure: # if the original deferred was cancelled, and our timeout has fired, then @@ -967,7 +967,12 @@ async def sleep(self, name: str, delay_ms: int) -> None: # Create a deferred that gets called in N seconds sleep_deferred: "defer.Deferred[None]" = defer.Deferred() - call = self._clock.call_later(delay_ms / 1000, sleep_deferred.callback, None) + call = self._clock.call_later( + delay_ms / 1000, + True if delay_ms / 1000 > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, + sleep_deferred.callback, + None, + ) # Create a deferred that will get called if `wake` is called with # the same `name`. @@ -1022,7 +1027,12 @@ async def wait(self, timeout_seconds: float) -> bool: # Create a deferred that gets called in N seconds sleep_deferred: "defer.Deferred[None]" = defer.Deferred() - call = self._clock.call_later(timeout_seconds, sleep_deferred.callback, None) + call = self._clock.call_later( + timeout_seconds, + True if timeout_seconds > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, + sleep_deferred.callback, + None, + ) try: await make_deferred_yieldable( diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 49a9151916e..f14e32e3a9d 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -198,7 +198,7 @@ def on_complete(r: RV) -> RV: # the should_cache bit, we leave it in the cache for now and schedule # its removal later. if self.timeout_sec and context.should_cache: - self.clock.call_later(self.timeout_sec, self._entry_timeout, key) + self.clock.call_later(self.timeout_sec, False, self._entry_timeout, key) else: # otherwise, remove the result immediately. self.unset(key) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 88edc071611..435c402db66 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -419,4 +419,4 @@ def start_next_request() -> None: except KeyError: pass - self.clock.call_later(0.0, start_next_request) + self.clock.call_later(0.0, False, start_next_request) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 0539989320f..135ed7f5321 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -473,7 +473,7 @@ async def wrapper() -> None: occasional_status_call.stop() # Try launch a new task since we've finished with this one. - self._clock.call_later(0.1, self._launch_scheduled_tasks) + self._clock.call_later(0.1, False, self._launch_scheduled_tasks) if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: return diff --git a/tests/server.py b/tests/server.py index f038bdf069b..c6e588e5e91 100644 --- a/tests/server.py +++ b/tests/server.py @@ -899,7 +899,7 @@ def _produce() -> None: # some implementations of IProducer (for example, FileSender) # don't return a deferred. d = maybeDeferred(self.producer.resumeProducing) - d.addCallback(lambda: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] + d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] if not streaming: self._reactor.callLater(0.0, _produce) # type: ignore[call-later-not-tracked] diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index c52f963a7ee..97fabb35b67 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -40,12 +40,12 @@ def test_later(self) -> None: def _cb0() -> None: invoked[0] = 1 - self.clock.call_later(10, _cb0) + self.clock.call_later(10, False, _cb0) def _cb1() -> None: invoked[1] = 1 - self.clock.call_later(20, _cb1) + self.clock.call_later(20, False, _cb1) self.assertFalse(invoked[0]) @@ -64,12 +64,12 @@ def test_cancel_later(self) -> None: def _cb0() -> None: invoked[0] = 1 - t0 = self.clock.call_later(10, _cb0) + t0 = self.clock.call_later(10, False, _cb0) def _cb1() -> None: invoked[1] = 1 - self.clock.call_later(20, _cb1) + self.clock.call_later(20, False, _cb1) self.clock.cancel_call_later(t0) diff --git a/tests/utils.py b/tests/utils.py index d1b66d41597..5879f3cc236 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -284,6 +284,7 @@ def time_msec(self) -> int: def call_later( self, delay: float, + _: bool, # track_for_shutdown Unused in tests callback: Callable[P, object], *args: P.args, **kwargs: P.kwargs, From b117145417f498977d0fb32cf46a70ccf96659b9 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 10 Sep 2025 16:54:34 -0600 Subject: [PATCH 089/181] Document function arg --- synapse/util/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index ad30b016bcf..37bb306d8d3 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -262,6 +262,12 @@ def call_later( Args: delay: How long to wait in seconds. + track_for_shutdown: Whether this call should be tracked for cleanup during + shutdown. Any call with a long delay, or that is created infrequently, + should be tracked. Calls which are short or of 0 delay don't require + tracking since the small delay after shutdown before they trigger is + immaterial. It's not worth the overhead to track those calls as it blows up + the tracking map on large server instances. callback: Function to call *args: Postional arguments to pass to function. **kwargs: Key arguments to pass to function. From 03016753ce821ae01ca86ea50d04bd62bf5cd1af Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 11 Sep 2025 09:36:29 -0600 Subject: [PATCH 090/181] Readd mistakenly removed test --- tests/app/test_homeserver_start.py | 40 ++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/app/test_homeserver_start.py diff --git a/tests/app/test_homeserver_start.py b/tests/app/test_homeserver_start.py new file mode 100644 index 00000000000..9dc20800b22 --- /dev/null +++ b/tests/app/test_homeserver_start.py @@ -0,0 +1,40 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright 2021 The Matrix.org Foundation C.I.C. +# Copyright (C) 2023 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import synapse.app.homeserver +from synapse.config._base import ConfigError + +from tests.config.utils import ConfigFileTestCase + + +class HomeserverAppStartTestCase(ConfigFileTestCase): + def test_wrong_start_caught(self) -> None: + # Generate a config with a worker_app + self.generate_config() + # Add a blank line as otherwise the next addition ends up on a line with a comment + self.add_lines_to_config([" "]) + self.add_lines_to_config(["worker_app: test_worker_app"]) + self.add_lines_to_config(["worker_log_config: /data/logconfig.config"]) + self.add_lines_to_config(["instance_map:"]) + self.add_lines_to_config([" main:", " host: 127.0.0.1", " port: 1234"]) + # Ensure that starting master process with worker config raises an exception + with self.assertRaises(ConfigError): + synapse.app.homeserver.setup(["-c", self.config_file]) From d03ec2f62cbaa8b79992ae68fe24ad61f7eaf1df Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 11 Sep 2025 17:58:48 -0600 Subject: [PATCH 091/181] Remove delayed calls that raise --- synapse/util/__init__.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 37bb306d8d3..5aeec18650b 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -281,8 +281,14 @@ def call_later( self._delayed_call_id = self._delayed_call_id + 1 def wrapped_callback(*args: Any, **kwargs: Any) -> None: - callback(*args, **kwargs) - self._call_id_to_delayed_call.pop(call_id) + try: + callback(*args, **kwargs) + except Exception: + raise + finally: + # We still want to remove the call from the tracking map. Even if + # the callback raises an exception. + self._call_id_to_delayed_call.pop(call_id) with context.PreserveLoggingContext(): # We can ignore the lint here since this is the one location callLater From 42d990da40188e3ea9272bd3d08a3123c70a0337 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 11 Sep 2025 17:59:09 -0600 Subject: [PATCH 092/181] Fully shutdown background updater --- synapse/storage/background_updates.py | 7 +++++++ synapse/storage/database.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index acc0abee63e..71dc9f69645 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -284,6 +284,13 @@ def __init__(self, hs: "HomeServer", database: "DatabasePool"): self.sleep_duration_ms = hs.config.background_updates.sleep_duration_ms self.sleep_enabled = hs.config.background_updates.sleep_enabled + def shutdown(self) -> None: + """ + Stop any further background updates from happening. + """ + self.enabled = False + self._background_update_handlers.clear() + def get_status(self) -> UpdaterStatus: """An integer summarising the updater status. Used as a metric.""" if self._aborted: diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 816af268433..d6d92362b9a 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -649,7 +649,7 @@ def stop_background_updates(self) -> None: """ Stops the database from running any further background updates. """ - self.updates.enabled = False + self.updates.shutdown() def name(self) -> str: "Return the name of this database" From d6bdce6fba862ff5a10acd1aa4b1b72d9ea9db4b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 11 Sep 2025 17:59:29 -0600 Subject: [PATCH 093/181] Add test for clean server shutdown --- tests/app/test_homeserver_shutdown.py | 66 +++++++++++++++++++++++++++ tests/server.py | 13 ++++++ 2 files changed, 79 insertions(+) create mode 100644 tests/app/test_homeserver_shutdown.py diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py new file mode 100644 index 00000000000..a282d6d82ab --- /dev/null +++ b/tests/app/test_homeserver_shutdown.py @@ -0,0 +1,66 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# Originally licensed under the Apache License, Version 2.0: +# . +# +# [This file includes modifications made by New Vector Limited] +# +# + +import gc +import weakref +from unittest import TestResult + +from synapse.app.homeserver import SynapseHomeServer + +from tests.server import get_clock, setup_test_homeserver +from tests.unittest import HomeserverTestCase + + +class HomeserverCleanShutdownTestCase(HomeserverTestCase): + def setUp(self) -> None: + pass + + def test_clean_homeserver_shutdown(self) -> None: + """Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected""" + self.reactor, self.clock = get_clock() + self.hs = setup_test_homeserver( + self.addCleanup, + reactor=self.reactor, + homeserver_to_use=SynapseHomeServer, + clock=self.clock, + ) + self.wait_for_background_updates() + + hs_ref = weakref.ref(self.hs) + + # Cleanup the homeserver. + # This works since we register `hs.shutdown()` as a cleanup function in + # `setup_test_homeserver`. + self._runCleanups(TestResult()) + self.get_success(self.reactor.shutdown()) + + del self.hs + + # Advance the reactor to allow for any outstanding calls to be run. + # A value of 1 is not enough, so a value of 2 is used. + self.reactor.advance(2) + + # Force garbage collection. + gc.collect() + + # Ensure the `HomeServer` hs been garbage collected by attempting to use the + # weakref to it. + if hs_ref() is not None: + self.fail("HomeServer reference should not be valid at this point") diff --git a/tests/server.py b/tests/server.py index c6e588e5e91..f67b7ac3cc6 100644 --- a/tests/server.py +++ b/tests/server.py @@ -525,6 +525,19 @@ def getHostByName( # overwrite it again. self.nameResolver = SimpleResolverComplexifier(FakeResolver()) + async def shutdown(self) -> None: + """Cleanup any outstanding resources referenced by this reactor. Useful when + trying to remove any references to the `HomeServer` that may have been + registered with this fake reactor""" + + # `MemoryReactorClock.removeSystemEventTrigger` is not implemented. + # So manually clear the triggers here. + self.triggers.clear() + + # `MemoryReactorClock` never clears the hooks made when `callWhenRunning` is + # called. So manually clear the hooks here. + self.whenRunningHooks.clear() + def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver: raise NotImplementedError() From 83be0161f491bb69613cf5b8cbccf68f50f04be4 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 09:00:02 -0600 Subject: [PATCH 094/181] Add reasoning for unregister_sighups being first --- synapse/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/server.py b/synapse/server.py index f14a88eb873..4a8a77851f9 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -377,6 +377,8 @@ async def shutdown(self) -> None: self.get_instance_id(), ) + # Unregister sighups first. If a shutdown was requested we shouldn't be responding + # to things like config changes. So it would be best to stop listening to these first. unregister_sighups(self._instance_id) # TODO: It would be desireable to be able to report an error if the HomeServer From 169bc53a7d241c68f5aa38502d2afc12dfe3834d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 12:47:47 -0600 Subject: [PATCH 095/181] Make docstring more verbose --- synapse/http/site.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/http/site.py b/synapse/http/site.py index 7a9d609db5a..063cf63307b 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -696,7 +696,9 @@ def connectionLost(self, reason: Failure) -> None: # type: ignore[override] """ Called when the connection is shut down. - Remove the connection from the factory's connection list, when it's lost. + Clear any circular references here, and any external references to this + Protocol. The connection has been closed. In our case, we need to remove the + connection from the factory's connection list, when it's lost. """ super().connectionLost(reason) self.factory.removeConnection(self) From e278d38e0191a0573410e5e6029dc9a262b4c2f9 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 20:40:54 +0000 Subject: [PATCH 096/181] Update scripts-dev/mypy_synapse_plugin.py Co-authored-by: Eric Eastwood --- scripts-dev/mypy_synapse_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index cff5987a6f4..ee0087daf88 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -222,7 +222,7 @@ def get_function_signature_hook( # callback, let's just pass it in while we have it. return lambda ctx: check_prometheus_metric_instantiation(ctx, fullname) - if "twisted.internet.task.LoopingCall" == fullname: + if fullname == "twisted.internet.task.LoopingCall": return check_looping_call return None From 3da4253f402053f6e7eeec566392682652d90875 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 14:41:25 -0600 Subject: [PATCH 097/181] Fix wording --- synapse/app/homeserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 7ec22d3a6f0..b6afccf48ea 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -322,7 +322,7 @@ def setup( Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. reactor: Optionally provide a reactor to use. Can be useful in different - scenarios that you want to control over the reactor, such as tests. + scenarios that you want control over the reactor, such as tests. freeze: Whether to freeze all objects in the garbage collector. May result in less work for the garbage collector since the `SynapseHomeServer` generally has a static lifetime. From 4023df93d138ef028e0e34a9d2d0a76860edcbf8 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 20:48:54 +0000 Subject: [PATCH 098/181] Update scripts-dev/mypy_synapse_plugin.py Co-authored-by: Eric Eastwood --- scripts-dev/mypy_synapse_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index ee0087daf88..6687f3ea2e7 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -244,7 +244,7 @@ def get_method_signature_hook( ): return check_is_cacheable_wrapper - if fullname.startswith(("twisted.internet.interfaces.IReactorTime.callLater",)): + if fullname == "twisted.internet.interfaces.IReactorTime.callLater": return check_call_later return None From fde361a7c34bb084e3cd060ed09fa31814d68bde Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 14:49:41 -0600 Subject: [PATCH 099/181] Revert log to debug --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8c2528bf029..c99f8fcbc8b 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -860,7 +860,7 @@ async def _send_request( ) delay_seconds *= random.uniform(0.8, 1.4) - logger.info( + logger.debug( "{%s} [%s] Waiting %ss before re-sending...", request.txn_id, request.destination, From 49e0df21b62f7b1ee580964dfae866b0778fa1cf Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 20:51:11 +0000 Subject: [PATCH 100/181] Update synapse/server.py Co-authored-by: Eric Eastwood --- synapse/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/server.py b/synapse/server.py index 4a8a77851f9..69a77a81537 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -185,7 +185,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: - # Old versions don't have `LiteralString` + # Old Python versions don't have `LiteralString` from txredisapi import ConnectionHandler from typing_extensions import LiteralString From 01c9de60eeafa190244615ba384282dea9c5cdbb Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:18:05 -0600 Subject: [PATCH 101/181] Remove unnecessary cleanup_metrics --- synapse/_scripts/generate_workers_map.py | 1 - synapse/server.py | 21 +++++++++------------ 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/synapse/_scripts/generate_workers_map.py b/synapse/_scripts/generate_workers_map.py index 58618d848c2..da8d1153724 100755 --- a/synapse/_scripts/generate_workers_map.py +++ b/synapse/_scripts/generate_workers_map.py @@ -157,7 +157,6 @@ def get_registered_paths_for_default( # TODO We only do this to avoid an error, but don't need the database etc hs.setup() registered_paths = get_registered_paths_for_hs(hs) - hs.cleanup_metrics() return registered_paths diff --git a/synapse/server.py b/synapse/server.py index 69a77a81537..b1fab55b0fe 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -411,7 +411,15 @@ async def shutdown(self) -> None: self.get_keyring().shutdown() - self.cleanup_metrics() + # Cleanup metrics associated with the homeserver + for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): + later_gauge.unregister_hooks_for_homeserver_instance_id( + self.get_instance_id() + ) + + CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver( + self.config.server.server_name + ) for db in self.get_datastores().databases: db.stop_background_updates() @@ -466,17 +474,6 @@ def register_async_shutdown_handler( ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, kwargs=kwargs) ) - def cleanup_metrics(self) -> None: - # Cleanup metrics associated with the homeserver - for later_gauge in all_later_gauges_to_clean_up_on_shutdown.values(): - later_gauge.unregister_hooks_for_homeserver_instance_id( - self.get_instance_id() - ) - - CACHE_METRIC_REGISTRY.unregister_hooks_for_homeserver( - self.config.server.server_name - ) - def register_sync_shutdown_handler( self, *, From 4d87c26250c99739d03869176fe99d1085916a79 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 21:22:25 +0000 Subject: [PATCH 102/181] Update synapse/server.py Co-authored-by: Eric Eastwood --- synapse/server.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index b1fab55b0fe..6415540eb99 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -400,9 +400,7 @@ async def shutdown(self) -> None: await port_shutdown self._listening_services.clear() - logger.info("Shutting down metrics listeners") for server, thread in self._metrics_listeners: - logger.info("Shutting down metrics listener") server.shutdown() thread.join() self._metrics_listeners.clear() From 996b924c838be04c3dae5f72052033bf5786e8c5 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:24:53 -0600 Subject: [PATCH 103/181] Rename variable --- synapse/handlers/message.py | 5 +---- synapse/util/__init__.py | 10 +++++----- tests/utils.py | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 71c60943364..164cee59f91 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -447,12 +447,9 @@ def _schedule_expiry_for_event(self, event_id: str, expiry_ts: int) -> None: logger.info("Scheduling expiry for event %s in %.3fs", event_id, delay) - track_for_shutdown = ( - True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False - ) self._scheduled_expiry = self.clock.call_later( delay, - track_for_shutdown, + True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "_expire_event", self.server_name, diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 5aeec18650b..a89f83d06c4 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -147,10 +147,10 @@ def shutdown(self) -> None: self.cancel_all_looping_calls() self.cancel_all_delayed_calls() - async def sleep(self, seconds: float, track_for_shutdown: bool = False) -> None: + async def sleep(self, seconds: float, cancel_on_shutdown: bool = False) -> None: d: defer.Deferred[float] = defer.Deferred() with context.PreserveLoggingContext(): - self.call_later(seconds, track_for_shutdown, d.callback, seconds) + self.call_later(seconds, cancel_on_shutdown, d.callback, seconds) await d def time(self) -> float: @@ -250,7 +250,7 @@ def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: def call_later( self, delay: float, - track_for_shutdown: bool, + cancel_on_shutdown: bool, callback: Callable, *args: Any, **kwargs: Any, @@ -262,7 +262,7 @@ def call_later( Args: delay: How long to wait in seconds. - track_for_shutdown: Whether this call should be tracked for cleanup during + cancel_on_shutdown: Whether this call should be tracked for cleanup during shutdown. Any call with a long delay, or that is created infrequently, should be tracked. Calls which are short or of 0 delay don't require tracking since the small delay after shutdown before they trigger is @@ -276,7 +276,7 @@ def call_later( if self._is_shutdown: raise Exception("Cannot start delayed call. Clock has been shutdown") - if track_for_shutdown: + if cancel_on_shutdown: call_id = self._delayed_call_id self._delayed_call_id = self._delayed_call_id + 1 diff --git a/tests/utils.py b/tests/utils.py index 5879f3cc236..abdc95a51bf 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -284,7 +284,7 @@ def time_msec(self) -> int: def call_later( self, delay: float, - _: bool, # track_for_shutdown Unused in tests + _: bool, # cancel_on_shutdown unused in tests callback: Callable[P, object], *args: P.args, **kwargs: P.kwargs, From 83680f3f6e2760912b8b7ddf179d045293b03eee Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 21:26:51 +0000 Subject: [PATCH 104/181] Update synapse/util/__init__.py Co-authored-by: Eric Eastwood --- synapse/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index a89f83d06c4..7ea3cd70c19 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -318,7 +318,7 @@ def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> N def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: """ - Stop all scheduled calls. + Stop all scheduled calls that were marked with `cancel_on_shutdown` when they were created. Args: ignore_errs: Whether to re-raise errors encountered when cancelling the From ad6cdf476fad9935dffbe548d7cee733965dce91 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:29:21 -0600 Subject: [PATCH 105/181] Rename variable --- synapse/app/_base.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 914c8981f63..ca7bbebc2f2 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -110,18 +110,21 @@ def register_sighup( - instance_id: str, func: Callable[P, None], *args: P.args, **kwargs: P.kwargs + homeserver_instance_id: str, + func: Callable[P, None], + *args: P.args, + **kwargs: P.kwargs, ) -> None: """ Register a function to be called when a SIGHUP occurs. Args: - instance_id: Unique ID for this Synapse process instance. + homeserver_instance_id: Unique ID for this Synapse process instance. func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ - _instance_id_to_sighup_callbacks_map.setdefault(instance_id, []).append( + _instance_id_to_sighup_callbacks_map.setdefault(homeserver_instance_id, []).append( (func, args, kwargs) ) From e3560245d695bd66a854a23f44e13fcedab35876 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:31:05 -0600 Subject: [PATCH 106/181] Flush out docstring --- synapse/app/_base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index ca7bbebc2f2..819a118420a 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -104,7 +104,10 @@ str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]] ] = {} """ -Mapping from homeserver instance_id to tuples of function, args list, kwargs dict +Map from homeserver instance_id to a list of callbacks. + +We use `instance_id` instead of `server_name` because it's possible to have multiple +workers running in the same process with the same `server_name`. """ P = ParamSpec("P") From e7431d05ac7817010923fed28bc2cc2d2041eea8 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:32:22 -0600 Subject: [PATCH 107/181] Indent docstring --- synapse/app/homeserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index b6afccf48ea..e8ebe079ea2 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -322,10 +322,10 @@ def setup( Args: config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. reactor: Optionally provide a reactor to use. Can be useful in different - scenarios that you want control over the reactor, such as tests. + scenarios that you want control over the reactor, such as tests. freeze: Whether to freeze all objects in the garbage collector. May result in - less work for the garbage collector since the `SynapseHomeServer` generally has - a static lifetime. + less work for the garbage collector since the `SynapseHomeServer` generally has + a static lifetime. Returns: A homeserver instance. From 5b6006486e00010ccf827601846752c122dc912c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:35:52 -0600 Subject: [PATCH 108/181] Add docstring for delay tracking threshold --- synapse/util/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 7ea3cd70c19..57a998e671f 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -110,6 +110,10 @@ def unwrapFirstError(failure: Failure) -> Failure: CALL_LATER_DELAY_TRACKING_THRESHOLD_S = 60 +""" +The length of time (in seconds) the `delay` in a call to `Clock.call_later` must be +before it is tracked for cancellation on shutdown. +""" class Clock: From a72ce6f5f7e878f98a358a93d1ea19b2e0d69728 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 21:36:37 +0000 Subject: [PATCH 109/181] Update changelog.d/18828.feature Co-authored-by: Eric Eastwood --- changelog.d/18828.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/18828.feature b/changelog.d/18828.feature index 8a61a97ea74..e7f3541de43 100644 --- a/changelog.d/18828.feature +++ b/changelog.d/18828.feature @@ -1 +1 @@ -Cleanly shutdown SynapseHomeServer object. +Cleanly shutdown `SynapseHomeServer` object. From 0c10de30ed1badfb91c92f28042016ba59e07440 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:41:16 -0600 Subject: [PATCH 110/181] Update docstring for freeze arg --- synapse/app/_base.py | 6 ++++-- synapse/app/homeserver.py | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 819a118420a..9f61dd1c1ff 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -548,8 +548,10 @@ async def start(hs: "HomeServer", freeze: bool = True) -> None: Args: hs: homeserver instance freeze: whether to freeze the homeserver base objects in the garbage collector. - May improve garbage collection performance by marking objects with an effectively - static lifetime as frozen so they don't need to be considered for cleanup. + May improve garbage collection performance by marking objects with an effectively + static lifetime as frozen so they don't need to be considered for cleanup. + If you ever want to `shutdown` the homeserver in memory, this needs to be + false otherwise the homeserver cannot be garbage collected after `shutdown`. """ server_name = hs.hostname reactor = hs.get_reactor() diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e8ebe079ea2..11c66f9822a 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -323,9 +323,11 @@ def setup( config_options_options: The options passed to Synapse. Usually `sys.argv[1:]`. reactor: Optionally provide a reactor to use. Can be useful in different scenarios that you want control over the reactor, such as tests. - freeze: Whether to freeze all objects in the garbage collector. May result in - less work for the garbage collector since the `SynapseHomeServer` generally has - a static lifetime. + freeze: whether to freeze the homeserver base objects in the garbage collector. + May improve garbage collection performance by marking objects with an effectively + static lifetime as frozen so they don't need to be considered for cleanup. + If you ever want to `shutdown` the homeserver in memory, this needs to be + false otherwise the homeserver cannot be garbage collected after `shutdown`. Returns: A homeserver instance. From 0337e1b6e9efbf7e872c42c832aebdaf0ee86289 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 15:45:42 -0600 Subject: [PATCH 111/181] Flush out freeze comments --- synapse/app/_base.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 9f61dd1c1ff..3ef37e0e12b 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -684,12 +684,17 @@ def run_sighup(*args: Any, **kwargs: Any) -> None: # everything currently allocated are things that will be used for the # rest of time. Doing so means less work each GC (hopefully). # + # Note that freezing the homeserver object means that it won't be able to be + # garbage collected in the case of attempting an in-memory `shutdown`. This only + # needs to be considered if such a case is desirable. Exiting the entire Python + # process will function expectedly either way. + # # PyPy does not (yet?) implement gc.freeze() if hasattr(gc, "freeze"): gc.collect() gc.freeze() - # Speed up shutdowns by freezing all allocated objects. This moves everything + # Speed up process exit by freezing all allocated objects. This moves everything # into the permanent generation and excludes them from the final GC. atexit.register(gc.freeze) From 3fe65ab88cadd8138384ad8b89d60a4f27792a4b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 16:22:42 -0600 Subject: [PATCH 112/181] Explain mypy ignores --- synapse/util/__init__.py | 5 +++++ tests/server.py | 15 +++++++++++++++ tests/util/caches/test_descriptors.py | 3 +++ 3 files changed, 23 insertions(+) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 57a998e671f..33704001fa4 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -297,12 +297,17 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: with context.PreserveLoggingContext(): # We can ignore the lint here since this is the one location callLater # should be called. + # mypy ignored here because: + # - this is the main location in code where using `callLater` is + # expected. call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] self._call_id_to_delayed_call[call_id] = call return call else: # We can ignore the lint here since this is the one location callLater should # be called. + # mypy ignored here because: + # - this is the main location in code where using `callLater` is expected. return self._reactor.callLater(delay, callback, *args, **kwargs) # type: ignore[call-later-not-tracked] def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: diff --git a/tests/server.py b/tests/server.py index f67b7ac3cc6..5b865e543f7 100644 --- a/tests/server.py +++ b/tests/server.py @@ -794,6 +794,9 @@ def _(res: Any) -> None: d: "Deferred[None]" = Deferred() d.addCallback(lambda x: function(*args, **kwargs)) d.addBoth(_) + # mypy ignored here because: + # - this is part of the test infrastructure where tracking these calls for + # shutdown isn't strictly necessary. self._reactor.callLater(0, d.callback, True) # type: ignore[call-later-not-tracked] return d @@ -912,9 +915,15 @@ def _produce() -> None: # some implementations of IProducer (for example, FileSender) # don't return a deferred. d = maybeDeferred(self.producer.resumeProducing) + # mypy ignored here because: + # - this is part of the test infrastructure where tracking these calls for + # shutdown isn't strictly necessary. d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] if not streaming: + # mypy ignored here because: + # - this is part of the test infrastructure where tracking these calls for + # shutdown isn't strictly necessary. self._reactor.callLater(0.0, _produce) # type: ignore[call-later-not-tracked] def write(self, byt: bytes) -> None: @@ -927,6 +936,9 @@ def write(self, byt: bytes) -> None: # TLSMemoryBIOProtocol) get very confused if a read comes back while they are # still doing a write. Doing a callLater here breaks the cycle. if self.autoflush: + # mypy ignored here because: + # - this is part of the test infrastructure where tracking these calls for + # shutdown isn't strictly necessary. self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] def writeSequence(self, seq: Iterable[bytes]) -> None: @@ -957,6 +969,9 @@ def flush(self, maxbytes: Optional[int] = None) -> None: self.buffer = self.buffer[len(to_write) :] if self.buffer and self.autoflush: + # mypy ignored here because: + # - this is part of the test infrastructure where tracking these calls for + # shutdown isn't strictly necessary. self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] if not self.buffer and self.disconnecting: diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py index fd9e4830981..45f92a87236 100644 --- a/tests/util/caches/test_descriptors.py +++ b/tests/util/caches/test_descriptors.py @@ -56,6 +56,9 @@ def run_on_reactor() -> "Deferred[int]": d: "Deferred[int]" = Deferred() + # mypy ignored here because: + # - this is part of the test infrastructure where tracking these calls for + # shutdown isn't strictly necessary. cast(IReactorTime, reactor).callLater(0, d.callback, 0) # type: ignore[call-later-not-tracked] return make_deferred_yieldable(d) From 6b643f2b09f5c96da3729a8553df93a8fcae706f Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 16:58:28 -0600 Subject: [PATCH 113/181] Remove unused optional arg --- synapse/util/__init__.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 33704001fa4..a8adb2a9800 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -151,10 +151,12 @@ def shutdown(self) -> None: self.cancel_all_looping_calls() self.cancel_all_delayed_calls() - async def sleep(self, seconds: float, cancel_on_shutdown: bool = False) -> None: + async def sleep(self, seconds: float) -> None: d: defer.Deferred[float] = defer.Deferred() with context.PreserveLoggingContext(): - self.call_later(seconds, cancel_on_shutdown, d.callback, seconds) + # We can ignore the lint here since this class is the one location callLater should + # be called. + self._reactor.callLater(seconds, d.callback) # type: ignore[call-later-not-tracked] await d def time(self) -> float: @@ -295,19 +297,14 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: self._call_id_to_delayed_call.pop(call_id) with context.PreserveLoggingContext(): - # We can ignore the lint here since this is the one location callLater + # We can ignore the lint here since this class is the one location callLater # should be called. - # mypy ignored here because: - # - this is the main location in code where using `callLater` is - # expected. call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] self._call_id_to_delayed_call[call_id] = call return call else: - # We can ignore the lint here since this is the one location callLater should + # We can ignore the lint here since this class is the one location callLater should # be called. - # mypy ignored here because: - # - this is the main location in code where using `callLater` is expected. return self._reactor.callLater(delay, callback, *args, **kwargs) # type: ignore[call-later-not-tracked] def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: From 2a4bae56c8e08e88301f3357e790519a2805448a Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 17:23:58 -0600 Subject: [PATCH 114/181] Add comments to call_later tracking choice --- synapse/app/phone_stats_home.py | 8 ++++++-- synapse/handlers/delayed_events.py | 6 +++++- synapse/handlers/presence.py | 8 ++++++-- synapse/handlers/room_member.py | 6 +++++- synapse/handlers/stats.py | 6 +++++- synapse/handlers/user_directory.py | 8 ++++++-- synapse/handlers/worker_lock.py | 7 ++++++- synapse/http/client.py | 6 +++++- synapse/module_api/__init__.py | 2 +- synapse/storage/database.py | 4 ++-- synapse/storage/databases/main/cache.py | 6 ++++-- synapse/storage/databases/main/registration.py | 2 +- synapse/storage/databases/main/roommember.py | 2 +- synapse/util/ratelimitutils.py | 6 +++++- synapse/util/task_scheduler.py | 6 +++++- 15 files changed, 63 insertions(+), 20 deletions(-) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 38ae8d85f0d..0b73f4c0025 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -285,13 +285,17 @@ async def _generate_monthly_active_users() -> None: # We need to defer this init for the cases that we daemonize # otherwise the process ID we get is that of the non-daemon process - clock.call_later(0, False, performance_stats_init) + clock.call_later( + 0, + False, # We don't track this call since it's short + performance_stats_init, + ) # We wait 5 minutes to send the first set of stats as the server can # be quite busy the first few minutes clock.call_later( INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, - True, + True, # We track this call since it would prevent shutdown for 5 minutes phone_stats_home, hs, stats, diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 58fffa17668..55b5eb1ad0f 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -91,7 +91,11 @@ async def _schedule_db_events() -> None: hs.get_notifier().add_replication_callback(self.notify_new_event) # Kick off again (without blocking) to catch any missed notifications # that may have fired before the callback was added. - self._clock.call_later(0, False, self.notify_new_event) + self._clock.call_later( + 0, + False, # We don't track this call since it's short + self.notify_new_event, + ) # Delayed events that are already marked as processed on startup might not have been # sent properly on the last run of the server, so unmark them to send them again. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index ff020a2b485..4086b7b188f 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -876,7 +876,11 @@ def __init__(self, hs: "HomeServer"): # The initial delay is to allow disconnected clients a chance to # reconnect before we treat them as offline. self.clock.call_later( - 30, False, self.clock.looping_call, self._handle_timeouts, 5000 + 30, + False, # We don't track this call since it's short + self.clock.looping_call, + self._handle_timeouts, + 5000, ) # Presence information is persisted, whether or not it is being tracked @@ -884,7 +888,7 @@ def __init__(self, hs: "HomeServer"): if self._presence_enabled: self.clock.call_later( 60, - False, + False, # We don't track this call since it's short self.clock.looping_call, self._persist_unpersisted_changes, 60 * 1000, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 2a95caca6a1..35d075936ce 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -2191,7 +2191,11 @@ def __init__(self, hs: "HomeServer"): self._notifier.add_replication_callback(self.notify_new_event) # We kick this off to pick up outstanding work from before the last restart. - self._clock.call_later(0, False, self.notify_new_event) + self._clock.call_later( + 0, + False, # We don't track this call since it's short + self.notify_new_event, + ) def notify_new_event(self) -> None: """Called when there may be more deltas to process""" diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index c9e39036648..fe185cf1fdc 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -75,7 +75,11 @@ def __init__(self, hs: "HomeServer"): # We kick this off so that we don't have to wait for a change before # we start populating stats - self.clock.call_later(0, False, self.notify_new_event) + self.clock.call_later( + 0, + False, # We don't track this call since it's short + self.notify_new_event, + ) def notify_new_event(self) -> None: """Called when there may be more deltas to process""" diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 56470ce833c..c71d069b741 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -138,7 +138,11 @@ def __init__(self, hs: "HomeServer"): # We kick this off so that we don't have to wait for a change before # we start populating the user directory - self.clock.call_later(0, False, self.notify_new_event) + self.clock.call_later( + 0, + False, # We don't track this call since it's short + self.notify_new_event, + ) # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( @@ -626,7 +630,7 @@ async def _unsafe_refresh_remote_profiles(self) -> None: # Come back later. self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, - False, + False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process, ) return diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index ecbde72ed1a..41c901281fe 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -197,7 +197,12 @@ def _wake_all_locks( if not deferred.called: deferred.callback(None) - self._clock.call_later(0, False, _wake_all_locks, locks) + self._clock.call_later( + 0, + False, # We don't track this call since it's short + _wake_all_locks, + locks, + ) @wrap_as_background_process("_cleanup_locks") async def _cleanup_locks(self) -> None: diff --git a/synapse/http/client.py b/synapse/http/client.py index 542dae7a7f2..8e387a038e8 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -171,7 +171,11 @@ def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCa """ def _scheduler(x: Callable[[], object]) -> IDelayedCall: - return clock.call_later(_EPSILON, False, x) + return clock.call_later( + _EPSILON, + False, # We don't track this call since it's short + x, + ) return _scheduler diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 814a0747360..6dc1a6b40e3 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -1448,7 +1448,7 @@ def delayed_background_call( return self._clock.call_later( # convert ms to seconds as needed by call_later. msec * 0.001, - False, + False, # We don't track calls in the module api since we don't know their purpose self.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), diff --git a/synapse/storage/database.py b/synapse/storage/database.py index d6d92362b9a..f87c6d03a01 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -638,7 +638,7 @@ def __init__( # background updates of tables that aren't safe to update. self._clock.call_later( 0.0, - False, + False, # We don't track this call since it's short run_as_background_process, "upsert_safety_check", self.server_name, @@ -688,7 +688,7 @@ async def _check_safe_to_upsert(self) -> None: if background_update_names: self._clock.call_later( 15.0, - False, + False, # We don't track this call since it's short run_as_background_process, "upsert_safety_check", self.server_name, diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 98871aba94a..83aad04dbe1 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -133,7 +133,7 @@ def __init__( ): self.hs.get_clock().call_later( CATCH_UP_CLEANUP_INTERVAL_MS / 1000, - True, + True, # We track this call to speedup shutdown self._clean_up_cache_invalidation_wrapper, ) @@ -797,7 +797,9 @@ async def _clean_up_cache_invalidation_wrapper(self) -> None: next_interval = REGULAR_CLEANUP_INTERVAL_MS self.hs.get_clock().call_later( - next_interval / 1000, True, self._clean_up_cache_invalidation_wrapper + next_interval / 1000, + True, # We track this call to speedup shutdown + self._clean_up_cache_invalidation_wrapper, ) async def _clean_up_batch_of_old_cache_invalidations( diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index db4e2a82fd3..b2aaa69bb48 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -214,7 +214,7 @@ def __init__( if hs.config.worker.run_background_tasks: self._clock.call_later( 0.0, - False, + False, # We don't track this call since it's short self._set_expiration_date_when_missing, ) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 46fc5d4cb94..c02721459ea 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -121,7 +121,7 @@ def __init__( ) self.hs.get_clock().call_later( 1, - False, + False, # We don't track this call since it's short self._count_known_servers, ) federation_known_servers_gauge.register_hook( diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 435c402db66..5bf6441eae8 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -419,4 +419,8 @@ def start_next_request() -> None: except KeyError: pass - self.clock.call_later(0.0, False, start_next_request) + self.clock.call_later( + 0.0, + False, # We don't track this call since it's short + start_next_request, + ) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 135ed7f5321..e3db742dd04 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -473,7 +473,11 @@ async def wrapper() -> None: occasional_status_call.stop() # Try launch a new task since we've finished with this one. - self._clock.call_later(0.1, False, self._launch_scheduled_tasks) + self._clock.call_later( + 0.1, + False, # We don't track this call since it's short + self._launch_scheduled_tasks, + ) if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: return From 18ed2d14a5b104322841292d77cd58455cd33a56 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 12 Sep 2025 17:27:11 -0600 Subject: [PATCH 115/181] Fix sleep call --- synapse/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index a8adb2a9800..0d9b7f76df4 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -156,7 +156,7 @@ async def sleep(self, seconds: float) -> None: with context.PreserveLoggingContext(): # We can ignore the lint here since this class is the one location callLater should # be called. - self._reactor.callLater(seconds, d.callback) # type: ignore[call-later-not-tracked] + self._reactor.callLater(seconds, d.callback, seconds) # type: ignore[call-later-not-tracked] await d def time(self) -> float: From f3d8c17dbdccfb3a3304b3b24f2c30ba056cd313 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 15 Sep 2025 19:10:45 -0600 Subject: [PATCH 116/181] Document calls to call_later --- synapse/appservice/scheduler.py | 5 +++-- synapse/handlers/delayed_events.py | 1 + synapse/handlers/message.py | 1 + synapse/handlers/user_directory.py | 11 +++++++---- synapse/push/emailpusher.py | 7 ++++++- synapse/push/httppusher.py | 8 +++++++- synapse/util/async_helpers.py | 2 ++ 7 files changed, 27 insertions(+), 8 deletions(-) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 405df1e6046..1c341c8783b 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -84,7 +84,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main import DataStore from synapse.types import DeviceListUpdates, JsonMapping -from synapse.util import Clock +from synapse.util import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock if TYPE_CHECKING: from synapse.server import HomeServer @@ -513,7 +513,8 @@ def recover(self) -> None: logger.info("Scheduling retries on %s in %fs", self.service.id, delay) self.scheduled_recovery = self.clock.call_later( delay, - True, + # Only track this call if it would delay shutdown by a substantial amount + True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "as-recoverer", self.server_name, diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 55b5eb1ad0f..6ae8be13084 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -458,6 +458,7 @@ def _schedule_next_at(self, next_send_ts: Timestamp) -> None: if self._next_delayed_event_call is None: self._next_delayed_event_call = self._clock.call_later( delay_sec, + # Only track this call if it would delay shutdown by a substantial amount True if delay_sec > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "_send_on_timeout", diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 164cee59f91..864b16f0fb9 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -449,6 +449,7 @@ def _schedule_expiry_for_event(self, event_id: str, expiry_ts: int) -> None: self._scheduled_expiry = self.clock.call_later( delay, + # Only track this call if it would delay shutdown by a substantial amount True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "_expire_event", diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index c71d069b741..84f010a8279 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -146,7 +146,9 @@ def __init__(self, hs: "HomeServer"): # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( - 10, False, self.kick_off_remote_profile_refresh_process + 10, + False, # We don't track this call since it's short + self.kick_off_remote_profile_refresh_process, ) async def search_users( @@ -563,14 +565,14 @@ async def _handle_possible_remote_profile_change( # the profile if the server only just sent us an event. self.clock.call_later( USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, - True, + False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process_for_remote_server, UserID.from_string(user_id).domain, ) # Schedule a wake-up to handle any backoffs that may occur in the future. self.clock.call_later( 2 * USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, - True, + False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process, ) return @@ -666,6 +668,7 @@ async def _unsafe_refresh_remote_profiles(self) -> None: delay = ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2 self._refresh_remote_profiles_call_later = self.clock.call_later( delay, + # Only track this call if it would delay shutdown by a substantial amount True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, self.kick_off_remote_profile_refresh_process, ) @@ -679,7 +682,7 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, - False, + False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process, ) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 9246518ef73..d1a3448fd43 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -30,6 +30,7 @@ from synapse.push.mailer import Mailer from synapse.push.push_types import EmailReason from synapse.storage.databases.main.event_push_actions import EmailPushAction +from synapse.util import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from synapse.util.threepids import validate_email if TYPE_CHECKING: @@ -228,8 +229,12 @@ async def _unsafe_process(self) -> None: self.timed_call = None if soonest_due_at is not None: + delay = self.seconds_until(soonest_due_at) self.timed_call = self.hs.get_clock().call_later( - self.seconds_until(soonest_due_at), True, self.on_timer + delay, + # Only track this call if it would delay shutdown substantially + True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, + self.on_timer, ) async def save_last_stream_ordering_and_success( diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index abad888cebb..ce475de7717 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -36,6 +36,7 @@ from synapse.push import Pusher, PusherConfig, PusherConfigException from synapse.storage.databases.main.event_push_actions import HttpPushAction from synapse.types import JsonDict, JsonMapping +from synapse.util import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from . import push_tools @@ -337,7 +338,12 @@ async def _unsafe_process(self) -> None: else: logger.info("Push failed: delaying for %ds", self.backoff_delay) self.timed_call = self.hs.get_clock().call_later( - self.backoff_delay, True, self.on_timer + self.backoff_delay, + # Only track backoffs if they would delay shutdown substantially + True + if self.backoff_delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, + self.on_timer, ) self.backoff_delay = min( self.backoff_delay * 2, self.MAX_BACKOFF_SEC diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 6bf8ad72253..942e089d4bd 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -969,6 +969,7 @@ async def sleep(self, name: str, delay_ms: int) -> None: sleep_deferred: "defer.Deferred[None]" = defer.Deferred() call = self._clock.call_later( delay_ms / 1000, + # Only track this call if it would delay shutdown by a substantial amount True if delay_ms / 1000 > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, sleep_deferred.callback, None, @@ -1029,6 +1030,7 @@ async def wait(self, timeout_seconds: float) -> bool: sleep_deferred: "defer.Deferred[None]" = defer.Deferred() call = self._clock.call_later( timeout_seconds, + # Only track this call if it would delay shutdown by a substantial amount True if timeout_seconds > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, sleep_deferred.callback, None, From c2d47d823eafc88157e75baf3f895c366fbc32d2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 15 Sep 2025 19:11:01 -0600 Subject: [PATCH 117/181] Fix call_later call cancellation --- synapse/util/__init__.py | 47 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 0d9b7f76df4..f26ecb502ba 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -34,12 +34,14 @@ Sequence, Set, TypeVar, + cast, ) import attr from immutabledict import immutabledict from matrix_common.versionstring import get_distribution_version_string from typing_extensions import ParamSpec +from zope.interface import implementer from twisted.internet import defer, task from twisted.internet.interfaces import IDelayedCall, IReactorTime @@ -300,6 +302,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: # We can ignore the lint here since this class is the one location callLater # should be called. call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] + call = cast(IDelayedCall, DelayedCallWrapper(call, call_id, self)) self._call_id_to_delayed_call[call_id] = call return call else: @@ -330,7 +333,9 @@ def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: ignore_errs: Whether to re-raise errors encountered when cancelling the scheduled call. """ - for call in self._call_id_to_delayed_call.values(): + # We wrap the dict in a list here since calling cancel on a delayed_call + # will result in the call removing itself from the map mid-iteration. + for call in list(self._call_id_to_delayed_call.values()): try: call.cancel() except Exception: @@ -339,6 +344,46 @@ def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: self._call_id_to_delayed_call.clear() +@implementer(IDelayedCall) +class DelayedCallWrapper: + """Wraps an `IDelayedCall` so that we can intercept the call to `cancel()` and + properly cleanup the delayed call from the tracking map of the `Clock`. + + args: + delayed_call: The actual `IDelayedCall` + call_id: Unique identifier for this delayed call + clock: The clock instance tracking this call + """ + + def __init__(self, delayed_call: IDelayedCall, call_id: int, clock: Clock): + self.delayed_call = delayed_call + self.call_id = call_id + self.clock = clock + + def cancel(self) -> None: + """Remove the call from the tracking map and propagate the call to the + underlying delayed_call. + """ + self.delayed_call.cancel() + self.clock._call_id_to_delayed_call.pop(self.call_id) + + def getTime(self) -> float: + """Propagate the call to the underlying delayed_call.""" + return self.delayed_call.getTime() + + def delay(self, secondsLater: float) -> None: + """Propagate the call to the underlying delayed_call.""" + self.delayed_call.delay(secondsLater) + + def reset(self, secondsFromNow: float) -> None: + """Propagate the call to the underlying delayed_call.""" + self.delayed_call.reset(secondsFromNow) + + def active(self) -> bool: + """Propagate the call to the underlying delayed_call.""" + return self.delayed_call.active() + + def log_failure( failure: Failure, msg: str, consumeErrors: bool = True ) -> Optional[Failure]: From 7d5f6e05515a6c70317d846c9a24694a86c80a0e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 16 Sep 2025 08:58:38 -0600 Subject: [PATCH 118/181] Add comment explaining lack of tracking for response cache --- synapse/util/caches/response_cache.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index f14e32e3a9d..01ab9c03f20 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -198,6 +198,11 @@ def on_complete(r: RV) -> RV: # the should_cache bit, we leave it in the cache for now and schedule # its removal later. if self.timeout_sec and context.should_cache: + # We don't need to track these calls since they don't hold and strong + # references which would keep the `HomeServer` in memory after shutdown. + # We don't want to track these because they can get cancelled really + # quickly and thrash the tracking mechanism, ie. during repeated calls + # to /sync. self.clock.call_later(self.timeout_sec, False, self._entry_timeout, key) else: # otherwise, remove the result immediately. From 4636d1f21981ca4acd3d8ca5449b76786e334a68 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 16 Sep 2025 10:19:33 -0600 Subject: [PATCH 119/181] Propagate cancel_on_shutdown up to timeout_deferred --- synapse/handlers/worker_lock.py | 16 +++++++++++++--- synapse/http/client.py | 9 ++++++++- synapse/http/matrixfederationclient.py | 9 ++++++++- synapse/http/proxy.py | 1 + synapse/notifier.py | 7 +++++++ synapse/replication/tcp/client.py | 5 ++++- synapse/util/async_helpers.py | 4 +++- tests/util/test_async_helpers.py | 6 +++--- 8 files changed, 47 insertions(+), 10 deletions(-) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 41c901281fe..729fd25e291 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -42,7 +42,7 @@ from synapse.logging.opentracing import start_active_span from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.databases.main.lock import Lock, LockStore -from synapse.util import Clock +from synapse.util import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock from synapse.util.async_helpers import timeout_deferred from synapse.util.constants import ONE_MINUTE_SECONDS @@ -251,9 +251,14 @@ async def __aenter__(self) -> None: # periodically wake up in case the lock was released but we # weren't notified. with PreserveLoggingContext(): + timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=self._get_next_retry_interval(), + timeout=timeout, + # Only track this call if it would delay shutdown substantially + cancel_on_shutdown=True + if timeout > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, clock=self.clock, ) except Exception: @@ -328,9 +333,14 @@ async def __aenter__(self) -> None: # periodically wake up in case the lock was released but we # weren't notified. with PreserveLoggingContext(): + timeout = self._get_next_retry_interval() await timeout_deferred( deferred=self.deferred, - timeout=self._get_next_retry_interval(), + timeout=timeout, + # Only track this call if it would delay shutdown substantially + cancel_on_shutdown=True + if timeout > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, clock=self.clock, ) except Exception: diff --git a/synapse/http/client.py b/synapse/http/client.py index 8e387a038e8..d627a9ecf8d 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -439,6 +439,7 @@ async def request( request_deferred = timeout_deferred( request_deferred, 60, + False, # We don't track this call since it's short self.hs.get_clock(), ) @@ -764,7 +765,12 @@ async def get_file( d = read_body_with_max_size(response, output_stream, max_size) # Ensure that the body is not read forever. - d = timeout_deferred(d, 30, self.hs.get_clock()) + d = timeout_deferred( + d, + 30, + False, # We don't track this call since it's short + self.hs.get_clock(), + ) length = await make_deferred_yieldable(d) except BodyExceededMaxSize: @@ -960,6 +966,7 @@ async def request( request_deferred = timeout_deferred( request_deferred, 60, + False, # We don't track this call since it's short self.hs.get_clock(), ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c99f8fcbc8b..3fee384308c 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -300,7 +300,12 @@ async def _handle_response( check_content_type_is(response.headers, parser.CONTENT_TYPE) d = read_body_with_max_size(response, parser, max_response_size) - d = timeout_deferred(d, timeout=timeout_sec, clock=clock) + d = timeout_deferred( + d, + timeout=timeout_sec, + cancel_on_shutdown=False, # We don't track this call since it's short + clock=clock, + ) length = await make_deferred_yieldable(d) @@ -738,6 +743,7 @@ async def _send_request( request_deferred = timeout_deferred( request_deferred, timeout=_sec_timeout, + cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self.clock, ) @@ -798,6 +804,7 @@ async def _send_request( d = timeout_deferred( d, timeout=_sec_timeout, + cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self.clock, ) diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index 4ae48e56619..5406d42dc97 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -166,6 +166,7 @@ async def _async_render(self, request: "SynapseRequest") -> Tuple[int, Any]: # so that it has enough time to complete and pass us the data before we give # up. timeout=90, + cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self._clock, ) diff --git a/synapse/notifier.py b/synapse/notifier.py index 838026be07e..66ad2e66cef 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -677,6 +677,13 @@ async def wait_for_events( listener = timeout_deferred( listener, (end_time - now) / 1000.0, + # We don't track these calls since they are constantly being + # overridden by new calls to /sync and they don't hold the + # `HomeServer` in memory on shutdown. It is safe to let them + # timeout of their own accord after shutting down since it + # won't delay shutdown and there won't be any adverse + # behvaviour. + False, self.hs.get_clock(), ) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 786703a7662..bd40d728b99 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -337,7 +337,10 @@ async def wait_for_stream_position( # to wedge here forever. deferred: "Deferred[None]" = Deferred() deferred = timeout_deferred( - deferred, _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, self._clock + deferred, + _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, + False, # We don't track this call since it's short + self._clock, ) waiting_list = self._streams_to_waiters.setdefault( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 942e089d4bd..26487546556 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -762,6 +762,7 @@ async def _ctx_manager() -> AsyncIterator[None]: def timeout_deferred( deferred: "defer.Deferred[_T]", timeout: float, + cancel_on_shutdown: bool, clock: Clock, ) -> "defer.Deferred[_T]": """The in built twisted `Deferred.addTimeout` fails to time out deferreds @@ -804,7 +805,8 @@ def time_it_out() -> None: if not new_d.called: new_d.errback(defer.TimeoutError("Timed out after %gs" % (timeout,))) - delayed_call = clock.call_later(timeout, False, time_it_out) + # We don't track these calls since they are short. + delayed_call = clock.call_later(timeout, cancel_on_shutdown, time_it_out) def convert_cancelled(value: Failure) -> Failure: # if the original deferred was cancelled, and our timeout has fired, then diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index 98184f795a5..f2ec4bf71d2 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -161,7 +161,7 @@ def canceller(_d: Deferred) -> None: cancelled = True non_completing_d: Deferred = Deferred(canceller) - timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock) + timing_out_d = timeout_deferred(non_completing_d, 1.0, False, self.clock) self.assertNoResult(timing_out_d) self.assertFalse(cancelled, "deferred was cancelled prematurely") @@ -179,7 +179,7 @@ def canceller(_d: Deferred) -> None: raise Exception("can't cancel this deferred") non_completing_d: Deferred = Deferred(canceller) - timing_out_d = timeout_deferred(non_completing_d, 1.0, self.clock) + timing_out_d = timeout_deferred(non_completing_d, 1.0, False, self.clock) self.assertNoResult(timing_out_d) @@ -215,7 +215,7 @@ def errback(res: Failure, deferred_name: str) -> Failure: original_deferred = blocking() original_deferred.addErrback(errback, "orig") - timing_out_d = timeout_deferred(original_deferred, 1.0, self.clock) + timing_out_d = timeout_deferred(original_deferred, 1.0, False, self.clock) self.assertNoResult(timing_out_d) self.assertIs(current_context(), SENTINEL_CONTEXT) timing_out_d.addErrback(errback, "timingout") From 83c84a05680e2e272a05e912d55e6d9411dd4048 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 16 Sep 2025 10:24:05 -0600 Subject: [PATCH 120/181] Make timeout_deferred use kwargs --- synapse/http/client.py | 24 ++++++++++++------------ synapse/http/matrixfederationclient.py | 6 +++--- synapse/http/proxy.py | 2 +- synapse/notifier.py | 8 ++++---- synapse/replication/tcp/client.py | 8 ++++---- synapse/util/async_helpers.py | 1 + tests/util/test_async_helpers.py | 21 ++++++++++++++++++--- 7 files changed, 43 insertions(+), 27 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index d627a9ecf8d..10c3b953c39 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -437,10 +437,10 @@ async def request( # we use our own timeout mechanism rather than treq's as a workaround # for https://twistedmatrix.com/trac/ticket/9534. request_deferred = timeout_deferred( - request_deferred, - 60, - False, # We don't track this call since it's short - self.hs.get_clock(), + deferred=request_deferred, + timeout=60, + cancel_on_shutdown=False, # We don't track this call since it's short + clock=self.hs.get_clock(), ) # turn timeouts into RequestTimedOutErrors @@ -766,10 +766,10 @@ async def get_file( # Ensure that the body is not read forever. d = timeout_deferred( - d, - 30, - False, # We don't track this call since it's short - self.hs.get_clock(), + deferred=d, + timeout=30, + cancel_on_shutdown=False, # We don't track this call since it's short + clock=self.hs.get_clock(), ) length = await make_deferred_yieldable(d) @@ -964,10 +964,10 @@ async def request( # for https://twistedmatrix.com/trac/ticket/9534. # (Updated url https://github.com/twisted/twisted/issues/9534) request_deferred = timeout_deferred( - request_deferred, - 60, - False, # We don't track this call since it's short - self.hs.get_clock(), + deferred=request_deferred, + timeout=60, + cancel_on_shutdown=False, # We don't track this call since it's short + clock=self.hs.get_clock(), ) # turn timeouts into RequestTimedOutErrors diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 3fee384308c..ab9d6cb4ec9 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -301,7 +301,7 @@ async def _handle_response( d = read_body_with_max_size(response, parser, max_response_size) d = timeout_deferred( - d, + deferred=d, timeout=timeout_sec, cancel_on_shutdown=False, # We don't track this call since it's short clock=clock, @@ -741,7 +741,7 @@ async def _send_request( bodyProducer=producer, ) request_deferred = timeout_deferred( - request_deferred, + deferred=request_deferred, timeout=_sec_timeout, cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self.clock, @@ -802,7 +802,7 @@ async def _send_request( # Update transactions table? d = treq.content(response) d = timeout_deferred( - d, + deferred=d, timeout=_sec_timeout, cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self.clock, diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index 5406d42dc97..c8e6d414421 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -161,7 +161,7 @@ async def _async_render(self, request: "SynapseRequest") -> Tuple[int, Any]: bodyProducer=QuieterFileBodyProducer(request.content), ) request_deferred = timeout_deferred( - request_deferred, + deferred=request_deferred, # This should be set longer than the timeout in `MatrixFederationHttpClient` # so that it has enough time to complete and pass us the data before we give # up. diff --git a/synapse/notifier.py b/synapse/notifier.py index 66ad2e66cef..ecb4636817e 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -675,16 +675,16 @@ async def wait_for_events( # is a new token. listener = user_stream.new_listener(prev_token) listener = timeout_deferred( - listener, - (end_time - now) / 1000.0, + deferred=listener, + timeout=(end_time - now) / 1000.0, # We don't track these calls since they are constantly being # overridden by new calls to /sync and they don't hold the # `HomeServer` in memory on shutdown. It is safe to let them # timeout of their own accord after shutting down since it # won't delay shutdown and there won't be any adverse # behvaviour. - False, - self.hs.get_clock(), + cancel_on_shutdown=False, + clock=self.hs.get_clock(), ) log_kv( diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index bd40d728b99..71feb65782c 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -337,10 +337,10 @@ async def wait_for_stream_position( # to wedge here forever. deferred: "Deferred[None]" = Deferred() deferred = timeout_deferred( - deferred, - _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, - False, # We don't track this call since it's short - self._clock, + deferred=deferred, + timeout=_WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, + cancel_on_shutdown=False, # We don't track this call since it's short + clock=self._clock, ) waiting_list = self._streams_to_waiters.setdefault( diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 26487546556..be363fa07b2 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -760,6 +760,7 @@ async def _ctx_manager() -> AsyncIterator[None]: def timeout_deferred( + *, deferred: "defer.Deferred[_T]", timeout: float, cancel_on_shutdown: bool, diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index f2ec4bf71d2..d8d04edebbd 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -161,7 +161,12 @@ def canceller(_d: Deferred) -> None: cancelled = True non_completing_d: Deferred = Deferred(canceller) - timing_out_d = timeout_deferred(non_completing_d, 1.0, False, self.clock) + timing_out_d = timeout_deferred( + deferred=non_completing_d, + timeout=1.0, + cancel_on_shutdown=False, + clock=self.clock, + ) self.assertNoResult(timing_out_d) self.assertFalse(cancelled, "deferred was cancelled prematurely") @@ -179,7 +184,12 @@ def canceller(_d: Deferred) -> None: raise Exception("can't cancel this deferred") non_completing_d: Deferred = Deferred(canceller) - timing_out_d = timeout_deferred(non_completing_d, 1.0, False, self.clock) + timing_out_d = timeout_deferred( + deferred=non_completing_d, + timeout=1.0, + cancel_on_shutdown=False, + clock=self.clock, + ) self.assertNoResult(timing_out_d) @@ -215,7 +225,12 @@ def errback(res: Failure, deferred_name: str) -> Failure: original_deferred = blocking() original_deferred.addErrback(errback, "orig") - timing_out_d = timeout_deferred(original_deferred, 1.0, False, self.clock) + timing_out_d = timeout_deferred( + deferred=original_deferred, + timeout=1.0, + cancel_on_shutdown=False, + clock=self.clock, + ) self.assertNoResult(timing_out_d) self.assertIs(current_context(), SENTINEL_CONTEXT) timing_out_d.addErrback(errback, "timingout") From 37f971373f3067a71a8f4935a69b1cb6e6d0a800 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 08:03:12 -0600 Subject: [PATCH 121/181] Extend docstring for homeserver_instance_id arg --- synapse/app/_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 3ef37e0e12b..e5c228f8334 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -122,7 +122,8 @@ def register_sighup( Register a function to be called when a SIGHUP occurs. Args: - homeserver_instance_id: Unique ID for this Synapse process instance. + homeserver_instance_id: Unique ID for this Synapse process instance to register + sighups for (`hs.get_instance_id()`). func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ From 2465c2f8720bb5f07342a325eed8e415b30c7b4c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 09:01:20 -0600 Subject: [PATCH 122/181] Change call_later to set default cancel arg --- synapse/app/phone_stats_home.py | 3 +-- synapse/appservice/scheduler.py | 6 ++++-- synapse/handlers/delayed_events.py | 7 ++++--- synapse/handlers/message.py | 6 ++++-- synapse/handlers/presence.py | 2 -- synapse/handlers/room_member.py | 1 - synapse/handlers/stats.py | 1 - synapse/handlers/user_directory.py | 12 ++++-------- synapse/handlers/worker_lock.py | 1 - synapse/http/client.py | 1 - synapse/module_api/__init__.py | 1 - synapse/push/emailpusher.py | 6 ++++-- synapse/push/httppusher.py | 4 ++-- synapse/storage/database.py | 2 -- synapse/storage/databases/main/cache.py | 4 ++-- synapse/storage/databases/main/registration.py | 1 - synapse/storage/databases/main/roommember.py | 1 - synapse/util/__init__.py | 4 ++-- synapse/util/async_helpers.py | 16 +++++++++++----- synapse/util/caches/response_cache.py | 2 +- synapse/util/ratelimitutils.py | 1 - synapse/util/task_scheduler.py | 1 - tests/test_test_utils.py | 8 ++++---- tests/utils.py | 6 ++++-- 24 files changed, 47 insertions(+), 50 deletions(-) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 0b73f4c0025..c5cfcad0751 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -287,7 +287,6 @@ async def _generate_monthly_active_users() -> None: # otherwise the process ID we get is that of the non-daemon process clock.call_later( 0, - False, # We don't track this call since it's short performance_stats_init, ) @@ -295,8 +294,8 @@ async def _generate_monthly_active_users() -> None: # be quite busy the first few minutes clock.call_later( INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, - True, # We track this call since it would prevent shutdown for 5 minutes phone_stats_home, hs, stats, + call_later_cancel_on_shutdown=True, # We track this call since it would prevent shutdown for 5 minutes ) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 1c341c8783b..2ef42741d4e 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -513,12 +513,14 @@ def recover(self) -> None: logger.info("Scheduling retries on %s in %fs", self.service.id, delay) self.scheduled_recovery = self.clock.call_later( delay, - # Only track this call if it would delay shutdown by a substantial amount - True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "as-recoverer", self.server_name, self.retry, + # Only track this call if it would delay shutdown by a substantial amount + call_later_cancel_on_shutdown=True + if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, ) def _backoff(self) -> None: diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 6ae8be13084..6263f3ce4bd 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -93,7 +93,6 @@ async def _schedule_db_events() -> None: # that may have fired before the callback was added. self._clock.call_later( 0, - False, # We don't track this call since it's short self.notify_new_event, ) @@ -458,12 +457,14 @@ def _schedule_next_at(self, next_send_ts: Timestamp) -> None: if self._next_delayed_event_call is None: self._next_delayed_event_call = self._clock.call_later( delay_sec, - # Only track this call if it would delay shutdown by a substantial amount - True if delay_sec > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "_send_on_timeout", self.server_name, self._send_on_timeout, + # Only track this call if it would delay shutdown by a substantial amount + call_later_cancel_on_shutdown=True + if delay_sec > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, ) else: self._next_delayed_event_call.reset(delay_sec) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 864b16f0fb9..778368f4869 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -449,13 +449,15 @@ def _schedule_expiry_for_event(self, event_id: str, expiry_ts: int) -> None: self._scheduled_expiry = self.clock.call_later( delay, - # Only track this call if it would delay shutdown by a substantial amount - True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, run_as_background_process, "_expire_event", self.server_name, self._expire_event, event_id, + # Only track this call if it would delay shutdown by a substantial amount + call_later_cancel_on_shutdown=True + if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, ) async def _expire_event(self, event_id: str) -> None: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 4086b7b188f..1ae0638213e 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -877,7 +877,6 @@ def __init__(self, hs: "HomeServer"): # reconnect before we treat them as offline. self.clock.call_later( 30, - False, # We don't track this call since it's short self.clock.looping_call, self._handle_timeouts, 5000, @@ -888,7 +887,6 @@ def __init__(self, hs: "HomeServer"): if self._presence_enabled: self.clock.call_later( 60, - False, # We don't track this call since it's short self.clock.looping_call, self._persist_unpersisted_changes, 60 * 1000, diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 35d075936ce..4c50feb4442 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -2193,7 +2193,6 @@ def __init__(self, hs: "HomeServer"): # We kick this off to pick up outstanding work from before the last restart. self._clock.call_later( 0, - False, # We don't track this call since it's short self.notify_new_event, ) diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index fe185cf1fdc..6895287beba 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -77,7 +77,6 @@ def __init__(self, hs: "HomeServer"): # we start populating stats self.clock.call_later( 0, - False, # We don't track this call since it's short self.notify_new_event, ) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 84f010a8279..749dbf39c89 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -140,14 +140,12 @@ def __init__(self, hs: "HomeServer"): # we start populating the user directory self.clock.call_later( 0, - False, # We don't track this call since it's short self.notify_new_event, ) # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( 10, - False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process, ) @@ -565,14 +563,12 @@ async def _handle_possible_remote_profile_change( # the profile if the server only just sent us an event. self.clock.call_later( USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, - False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process_for_remote_server, UserID.from_string(user_id).domain, ) # Schedule a wake-up to handle any backoffs that may occur in the future. self.clock.call_later( 2 * USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, - False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process, ) return @@ -632,7 +628,6 @@ async def _unsafe_refresh_remote_profiles(self) -> None: # Come back later. self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, - False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process, ) return @@ -668,9 +663,11 @@ async def _unsafe_refresh_remote_profiles(self) -> None: delay = ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2 self._refresh_remote_profiles_call_later = self.clock.call_later( delay, - # Only track this call if it would delay shutdown by a substantial amount - True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, self.kick_off_remote_profile_refresh_process, + # Only track this call if it would delay shutdown by a substantial amount + call_later_cancel_on_shutdown=True + if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, ) return @@ -682,7 +679,6 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, - False, # We don't track this call since it's short self.kick_off_remote_profile_refresh_process, ) diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 729fd25e291..df01a1ebc5e 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -199,7 +199,6 @@ def _wake_all_locks( self._clock.call_later( 0, - False, # We don't track this call since it's short _wake_all_locks, locks, ) diff --git a/synapse/http/client.py b/synapse/http/client.py index 10c3b953c39..7ca280832a7 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -173,7 +173,6 @@ def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCa def _scheduler(x: Callable[[], object]) -> IDelayedCall: return clock.call_later( _EPSILON, - False, # We don't track this call since it's short x, ) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 6dc1a6b40e3..9309aa93942 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -1448,7 +1448,6 @@ def delayed_background_call( return self._clock.call_later( # convert ms to seconds as needed by call_later. msec * 0.001, - False, # We don't track calls in the module api since we don't know their purpose self.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index d1a3448fd43..1007a5ff318 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -232,9 +232,11 @@ async def _unsafe_process(self) -> None: delay = self.seconds_until(soonest_due_at) self.timed_call = self.hs.get_clock().call_later( delay, - # Only track this call if it would delay shutdown substantially - True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, self.on_timer, + # Only track this call if it would delay shutdown substantially + call_later_cancel_on_shutdown=True + if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, ) async def save_last_stream_ordering_and_success( diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index ce475de7717..08b1ddc7439 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -339,11 +339,11 @@ async def _unsafe_process(self) -> None: logger.info("Push failed: delaying for %ds", self.backoff_delay) self.timed_call = self.hs.get_clock().call_later( self.backoff_delay, + self.on_timer, # Only track backoffs if they would delay shutdown substantially - True + call_later_cancel_on_shutdown=True if self.backoff_delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, - self.on_timer, ) self.backoff_delay = min( self.backoff_delay * 2, self.MAX_BACKOFF_SEC diff --git a/synapse/storage/database.py b/synapse/storage/database.py index f87c6d03a01..47422eba3d9 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -638,7 +638,6 @@ def __init__( # background updates of tables that aren't safe to update. self._clock.call_later( 0.0, - False, # We don't track this call since it's short run_as_background_process, "upsert_safety_check", self.server_name, @@ -688,7 +687,6 @@ async def _check_safe_to_upsert(self) -> None: if background_update_names: self._clock.call_later( 15.0, - False, # We don't track this call since it's short run_as_background_process, "upsert_safety_check", self.server_name, diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index 83aad04dbe1..c7927c24c09 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -133,8 +133,8 @@ def __init__( ): self.hs.get_clock().call_later( CATCH_UP_CLEANUP_INTERVAL_MS / 1000, - True, # We track this call to speedup shutdown self._clean_up_cache_invalidation_wrapper, + call_later_cancel_on_shutdown=True, # We track this call to speedup shutdown ) async def get_all_updated_caches( @@ -798,8 +798,8 @@ async def _clean_up_cache_invalidation_wrapper(self) -> None: self.hs.get_clock().call_later( next_interval / 1000, - True, # We track this call to speedup shutdown self._clean_up_cache_invalidation_wrapper, + call_later_cancel_on_shutdown=True, # We track this call to speedup shutdown ) async def _clean_up_batch_of_old_cache_invalidations( diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index b2aaa69bb48..117444e7b75 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -214,7 +214,6 @@ def __init__( if hs.config.worker.run_background_tasks: self._clock.call_later( 0.0, - False, # We don't track this call since it's short self._set_expiration_date_when_missing, ) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index c02721459ea..9db2e14a06f 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -121,7 +121,6 @@ def __init__( ) self.hs.get_clock().call_later( 1, - False, # We don't track this call since it's short self._count_known_servers, ) federation_known_servers_gauge.register_hook( diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index f26ecb502ba..2bb8e719a90 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -258,9 +258,9 @@ def cancel_all_looping_calls(self, consumeErrors: bool = True) -> None: def call_later( self, delay: float, - cancel_on_shutdown: bool, callback: Callable, *args: Any, + call_later_cancel_on_shutdown: bool = False, **kwargs: Any, ) -> IDelayedCall: """Call something later @@ -284,7 +284,7 @@ def call_later( if self._is_shutdown: raise Exception("Cannot start delayed call. Clock has been shutdown") - if cancel_on_shutdown: + if call_later_cancel_on_shutdown: call_id = self._delayed_call_id self._delayed_call_id = self._delayed_call_id + 1 diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index be363fa07b2..ea47ac7f820 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -807,7 +807,9 @@ def time_it_out() -> None: new_d.errback(defer.TimeoutError("Timed out after %gs" % (timeout,))) # We don't track these calls since they are short. - delayed_call = clock.call_later(timeout, cancel_on_shutdown, time_it_out) + delayed_call = clock.call_later( + timeout, time_it_out, call_later_cancel_on_shutdown=cancel_on_shutdown + ) def convert_cancelled(value: Failure) -> Failure: # if the original deferred was cancelled, and our timeout has fired, then @@ -972,10 +974,12 @@ async def sleep(self, name: str, delay_ms: int) -> None: sleep_deferred: "defer.Deferred[None]" = defer.Deferred() call = self._clock.call_later( delay_ms / 1000, - # Only track this call if it would delay shutdown by a substantial amount - True if delay_ms / 1000 > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, sleep_deferred.callback, None, + # Only track this call if it would delay shutdown by a substantial amount + call_later_cancel_on_shutdown=True + if delay_ms / 1000 > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, ) # Create a deferred that will get called if `wake` is called with @@ -1033,10 +1037,12 @@ async def wait(self, timeout_seconds: float) -> bool: sleep_deferred: "defer.Deferred[None]" = defer.Deferred() call = self._clock.call_later( timeout_seconds, - # Only track this call if it would delay shutdown by a substantial amount - True if timeout_seconds > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False, sleep_deferred.callback, None, + # Only track this call if it would delay shutdown by a substantial amount + call_later_cancel_on_shutdown=True + if timeout_seconds > CALL_LATER_DELAY_TRACKING_THRESHOLD_S + else False, ) try: diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 01ab9c03f20..dac14e6e587 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -203,7 +203,7 @@ def on_complete(r: RV) -> RV: # We don't want to track these because they can get cancelled really # quickly and thrash the tracking mechanism, ie. during repeated calls # to /sync. - self.clock.call_later(self.timeout_sec, False, self._entry_timeout, key) + self.clock.call_later(self.timeout_sec, self._entry_timeout, key) else: # otherwise, remove the result immediately. self.unset(key) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 5bf6441eae8..408b93645cf 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -421,6 +421,5 @@ def start_next_request() -> None: self.clock.call_later( 0.0, - False, # We don't track this call since it's short start_next_request, ) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index e3db742dd04..69c8846d1df 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -475,7 +475,6 @@ async def wrapper() -> None: # Try launch a new task since we've finished with this one. self._clock.call_later( 0.1, - False, # We don't track this call since it's short self._launch_scheduled_tasks, ) diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py index 97fabb35b67..c52f963a7ee 100644 --- a/tests/test_test_utils.py +++ b/tests/test_test_utils.py @@ -40,12 +40,12 @@ def test_later(self) -> None: def _cb0() -> None: invoked[0] = 1 - self.clock.call_later(10, False, _cb0) + self.clock.call_later(10, _cb0) def _cb1() -> None: invoked[1] = 1 - self.clock.call_later(20, False, _cb1) + self.clock.call_later(20, _cb1) self.assertFalse(invoked[0]) @@ -64,12 +64,12 @@ def test_cancel_later(self) -> None: def _cb0() -> None: invoked[0] = 1 - t0 = self.clock.call_later(10, False, _cb0) + t0 = self.clock.call_later(10, _cb0) def _cb1() -> None: invoked[1] = 1 - self.clock.call_later(20, False, _cb1) + self.clock.call_later(20, _cb1) self.clock.cancel_call_later(t0) diff --git a/tests/utils.py b/tests/utils.py index abdc95a51bf..afe185b9214 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -281,12 +281,14 @@ def time(self) -> float: def time_msec(self) -> int: return int(self.time() * 1000) - def call_later( + # Allow the `call_later_cancel_on_shutdown` arg after `*args` to mimic the API of + # the actual `Clock`. + def call_later( # type: ignore[valid-type] self, delay: float, - _: bool, # cancel_on_shutdown unused in tests callback: Callable[P, object], *args: P.args, + call_later_cancel_on_shutdown: bool = False, **kwargs: P.kwargs, ) -> Timer: ctx = current_context() From fe3491b2ef379bf8444ce80b44061b96c5febb92 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 09:20:47 -0600 Subject: [PATCH 123/181] Switch arg default to True --- synapse/app/phone_stats_home.py | 2 +- synapse/handlers/delayed_events.py | 1 + synapse/handlers/presence.py | 2 ++ synapse/handlers/room_member.py | 1 + synapse/handlers/stats.py | 1 + synapse/handlers/user_directory.py | 6 ++++++ synapse/handlers/worker_lock.py | 1 + synapse/http/client.py | 1 + synapse/module_api/__init__.py | 1 + synapse/storage/database.py | 2 ++ synapse/storage/databases/main/cache.py | 2 -- synapse/storage/databases/main/registration.py | 1 + synapse/storage/databases/main/roommember.py | 1 + synapse/util/__init__.py | 2 +- synapse/util/caches/response_cache.py | 7 ++++++- synapse/util/ratelimitutils.py | 1 + synapse/util/task_scheduler.py | 1 + 17 files changed, 28 insertions(+), 5 deletions(-) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index c5cfcad0751..faff293d0bc 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -288,6 +288,7 @@ async def _generate_monthly_active_users() -> None: clock.call_later( 0, performance_stats_init, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # We wait 5 minutes to send the first set of stats as the server can @@ -297,5 +298,4 @@ async def _generate_monthly_active_users() -> None: phone_stats_home, hs, stats, - call_later_cancel_on_shutdown=True, # We track this call since it would prevent shutdown for 5 minutes ) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 6263f3ce4bd..a198f01025e 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -94,6 +94,7 @@ async def _schedule_db_events() -> None: self._clock.call_later( 0, self.notify_new_event, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Delayed events that are already marked as processed on startup might not have been diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 1ae0638213e..7c2c72d00e4 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -880,6 +880,7 @@ def __init__(self, hs: "HomeServer"): self.clock.looping_call, self._handle_timeouts, 5000, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Presence information is persisted, whether or not it is being tracked @@ -890,6 +891,7 @@ def __init__(self, hs: "HomeServer"): self.clock.looping_call, self._persist_unpersisted_changes, 60 * 1000, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) presence_wheel_timer_size_gauge.register_hook( diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 4c50feb4442..ca39d432e19 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -2194,6 +2194,7 @@ def __init__(self, hs: "HomeServer"): self._clock.call_later( 0, self.notify_new_event, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def notify_new_event(self) -> None: diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 6895287beba..bc9c24a1b0f 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -78,6 +78,7 @@ def __init__(self, hs: "HomeServer"): self.clock.call_later( 0, self.notify_new_event, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def notify_new_event(self) -> None: diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 749dbf39c89..331c56be9b8 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -141,12 +141,14 @@ def __init__(self, hs: "HomeServer"): self.clock.call_later( 0, self.notify_new_event, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( 10, self.kick_off_remote_profile_refresh_process, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) async def search_users( @@ -565,11 +567,13 @@ async def _handle_possible_remote_profile_change( USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, self.kick_off_remote_profile_refresh_process_for_remote_server, UserID.from_string(user_id).domain, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Schedule a wake-up to handle any backoffs that may occur in the future. self.clock.call_later( 2 * USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, self.kick_off_remote_profile_refresh_process, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) return @@ -629,6 +633,7 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, self.kick_off_remote_profile_refresh_process, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) return @@ -680,6 +685,7 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, self.kick_off_remote_profile_refresh_process, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def kick_off_remote_profile_refresh_process_for_remote_server( diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index df01a1ebc5e..240dcb2128e 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -201,6 +201,7 @@ def _wake_all_locks( 0, _wake_all_locks, locks, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) @wrap_as_background_process("_cleanup_locks") diff --git a/synapse/http/client.py b/synapse/http/client.py index 7ca280832a7..8484be95f1a 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -174,6 +174,7 @@ def _scheduler(x: Callable[[], object]) -> IDelayedCall: return clock.call_later( _EPSILON, x, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) return _scheduler diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 9309aa93942..4ec65cb6873 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -1451,6 +1451,7 @@ def delayed_background_call( self.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), + call_later_cancel_on_shutdown=False, # We don't track calls in the module api since we don't know their purpose ) async def sleep(self, seconds: float) -> None: diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 47422eba3d9..bd1a66d98d2 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -642,6 +642,7 @@ def __init__( "upsert_safety_check", self.server_name, self._check_safe_to_upsert, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def stop_background_updates(self) -> None: @@ -691,6 +692,7 @@ async def _check_safe_to_upsert(self) -> None: "upsert_safety_check", self.server_name, self._check_safe_to_upsert, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def start_profiling(self) -> None: diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index c7927c24c09..b99ca250504 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -134,7 +134,6 @@ def __init__( self.hs.get_clock().call_later( CATCH_UP_CLEANUP_INTERVAL_MS / 1000, self._clean_up_cache_invalidation_wrapper, - call_later_cancel_on_shutdown=True, # We track this call to speedup shutdown ) async def get_all_updated_caches( @@ -799,7 +798,6 @@ async def _clean_up_cache_invalidation_wrapper(self) -> None: self.hs.get_clock().call_later( next_interval / 1000, self._clean_up_cache_invalidation_wrapper, - call_later_cancel_on_shutdown=True, # We track this call to speedup shutdown ) async def _clean_up_batch_of_old_cache_invalidations( diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 117444e7b75..a0016b82595 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -215,6 +215,7 @@ def __init__( self._clock.call_later( 0.0, self._set_expiration_date_when_missing, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # If support for MSC3866 is enabled and configured to require approval for new diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9db2e14a06f..c57e3a1ed7e 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -122,6 +122,7 @@ def __init__( self.hs.get_clock().call_later( 1, self._count_known_servers, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) federation_known_servers_gauge.register_hook( homeserver_instance_id=hs.get_instance_id(), diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 2bb8e719a90..faa54cf4727 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -260,7 +260,7 @@ def call_later( delay: float, callback: Callable, *args: Any, - call_later_cancel_on_shutdown: bool = False, + call_later_cancel_on_shutdown: bool = True, **kwargs: Any, ) -> IDelayedCall: """Call something later diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index dac14e6e587..a44504f3fd7 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -203,7 +203,12 @@ def on_complete(r: RV) -> RV: # We don't want to track these because they can get cancelled really # quickly and thrash the tracking mechanism, ie. during repeated calls # to /sync. - self.clock.call_later(self.timeout_sec, self._entry_timeout, key) + self.clock.call_later( + self.timeout_sec, + self._entry_timeout, + key, + call_later_cancel_on_shutdown=False, + ) else: # otherwise, remove the result immediately. self.unset(key) diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 408b93645cf..8910cb52c94 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -422,4 +422,5 @@ def start_next_request() -> None: self.clock.call_later( 0.0, start_next_request, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 69c8846d1df..e290c4ddc90 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -476,6 +476,7 @@ async def wrapper() -> None: self._clock.call_later( 0.1, self._launch_scheduled_tasks, + call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: From 5f167ff6e2984ceeba75fab72ba0539883a21c3d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 17:07:26 +0000 Subject: [PATCH 124/181] Update synapse/notifier.py Co-authored-by: Eric Eastwood --- synapse/notifier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/notifier.py b/synapse/notifier.py index ecb4636817e..6bf2fa10404 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -682,7 +682,7 @@ async def wait_for_events( # `HomeServer` in memory on shutdown. It is safe to let them # timeout of their own accord after shutting down since it # won't delay shutdown and there won't be any adverse - # behvaviour. + # behaviour. cancel_on_shutdown=False, clock=self.hs.get_clock(), ) From 8f57498f5fb51b23e6ad6dd6309e969444899b8e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 11:12:06 -0600 Subject: [PATCH 125/181] Refactor sighup callbacks --- synapse/app/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index e5c228f8334..7296e8ec4d7 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -583,8 +583,8 @@ async def _handle_sighup(*args: Any, **kwargs: Any) -> None: # we're not using systemd. sdnotify(b"RELOADING=1") - for _, v in _instance_id_to_sighup_callbacks_map.items(): - for func, args, kwargs in v: + for sighup_callbacks in _instance_id_to_sighup_callbacks_map.values(): + for func, args, kwargs in sighup_callbacks: func(*args, **kwargs) sdnotify(b"READY=1") From ecb2608880d9e56ad4bfb02d6e3f3d19d29982cd Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 11:15:53 -0600 Subject: [PATCH 126/181] Add comment to clock about lints --- synapse/util/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index faa54cf4727..1555a6f1f1d 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -127,6 +127,9 @@ class Clock: ensure the calls made by this `HomeServer` instance are tracked and can be cleaned up during `HomeServer.shutdown()`. + We enforce usage of this clock instead of using the reactor directly via lints in + `scripts-dev/mypy_synapse_plugin.py`. + Args: reactor: The Twisted reactor to use. """ From 0bb0e6ef8d91dc69fad453cc2258a39d7b600e71 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 11:17:57 -0600 Subject: [PATCH 127/181] Update docstring --- synapse/app/_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 7296e8ec4d7..27086e3cb28 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -122,8 +122,8 @@ def register_sighup( Register a function to be called when a SIGHUP occurs. Args: - homeserver_instance_id: Unique ID for this Synapse process instance to register - sighups for (`hs.get_instance_id()`). + homeserver_instance_id: Unique ID for this Synapse process instance + (`hs.get_instance_id()`) that this hook is associated with. func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. """ From 9123ec710b911e48d8697ec0f7f1831af0896c79 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 11:18:29 -0600 Subject: [PATCH 128/181] Update docstring --- synapse/app/_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 27086e3cb28..baf8bb7ec21 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -122,7 +122,7 @@ def register_sighup( Register a function to be called when a SIGHUP occurs. Args: - homeserver_instance_id: Unique ID for this Synapse process instance + homeserver_instance_id: The unique ID for this Synapse process instance (`hs.get_instance_id()`) that this hook is associated with. func: Function to be called when sent a SIGHUP signal. *args, **kwargs: args and kwargs to be passed to the target function. From 9fe95a4e99efa1061f73f907ed54b37c519525c3 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 11:22:53 -0600 Subject: [PATCH 129/181] Add comment explaining lack of shutdown --- synapse/_scripts/generate_workers_map.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/_scripts/generate_workers_map.py b/synapse/_scripts/generate_workers_map.py index da8d1153724..f66c01040cc 100755 --- a/synapse/_scripts/generate_workers_map.py +++ b/synapse/_scripts/generate_workers_map.py @@ -157,6 +157,12 @@ def get_registered_paths_for_default( # TODO We only do this to avoid an error, but don't need the database etc hs.setup() registered_paths = get_registered_paths_for_hs(hs) + # NOTE: a more robust implementation would properly shutdown/cleanup each server + # to avoid resource buildup. + # However, the call to `shutdown` is `async` so it would require additional complexity here. + # We are intentionally skipping this cleanup because this is a short-lived, one-off + # utility script where the simpler approach is sufficient and we shouldn't run into + # any resource buildup issues. return registered_paths From 17d012d7328e03d6502b2f8f15b02186ffb7b58c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 11:55:05 -0600 Subject: [PATCH 130/181] Remove cast --- synapse/util/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 1555a6f1f1d..be356db3ffd 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -34,7 +34,6 @@ Sequence, Set, TypeVar, - cast, ) import attr @@ -305,7 +304,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: # We can ignore the lint here since this class is the one location callLater # should be called. call = self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] - call = cast(IDelayedCall, DelayedCallWrapper(call, call_id, self)) + call = DelayedCallWrapper(call, call_id, self) self._call_id_to_delayed_call[call_id] = call return call else: From daa6c3eecc6e35a9ce1b08bf8655b00332c1093d Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 12:02:00 -0600 Subject: [PATCH 131/181] Update mypy ignore comments in tests --- tests/server.py | 20 ++++++++++---------- tests/util/caches/test_descriptors.py | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/server.py b/tests/server.py index 5b865e543f7..ea306926c6e 100644 --- a/tests/server.py +++ b/tests/server.py @@ -795,8 +795,8 @@ def _(res: Any) -> None: d.addCallback(lambda x: function(*args, **kwargs)) d.addBoth(_) # mypy ignored here because: - # - this is part of the test infrastructure where tracking these calls for - # shutdown isn't strictly necessary. + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. self._reactor.callLater(0, d.callback, True) # type: ignore[call-later-not-tracked] return d @@ -916,14 +916,14 @@ def _produce() -> None: # don't return a deferred. d = maybeDeferred(self.producer.resumeProducing) # mypy ignored here because: - # - this is part of the test infrastructure where tracking these calls for - # shutdown isn't strictly necessary. + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. d.addCallback(lambda x: self._reactor.callLater(0.1, _produce)) # type: ignore[call-later-not-tracked,call-overload] if not streaming: # mypy ignored here because: - # - this is part of the test infrastructure where tracking these calls for - # shutdown isn't strictly necessary. + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. self._reactor.callLater(0.0, _produce) # type: ignore[call-later-not-tracked] def write(self, byt: bytes) -> None: @@ -937,8 +937,8 @@ def write(self, byt: bytes) -> None: # still doing a write. Doing a callLater here breaks the cycle. if self.autoflush: # mypy ignored here because: - # - this is part of the test infrastructure where tracking these calls for - # shutdown isn't strictly necessary. + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] def writeSequence(self, seq: Iterable[bytes]) -> None: @@ -970,8 +970,8 @@ def flush(self, maxbytes: Optional[int] = None) -> None: self.buffer = self.buffer[len(to_write) :] if self.buffer and self.autoflush: # mypy ignored here because: - # - this is part of the test infrastructure where tracking these calls for - # shutdown isn't strictly necessary. + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. self._reactor.callLater(0.0, self.flush) # type: ignore[call-later-not-tracked] if not self.buffer and self.disconnecting: diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py index 45f92a87236..3dc0224adf1 100644 --- a/tests/util/caches/test_descriptors.py +++ b/tests/util/caches/test_descriptors.py @@ -57,8 +57,8 @@ def run_on_reactor() -> "Deferred[int]": d: "Deferred[int]" = Deferred() # mypy ignored here because: - # - this is part of the test infrastructure where tracking these calls for - # shutdown isn't strictly necessary. + # - this is part of the test infrastructure (outside of Synapse) so tracking + # these calls for for homeserver shutdown doesn't make sense. cast(IReactorTime, reactor).callLater(0, d.callback, 0) # type: ignore[call-later-not-tracked] return make_deferred_yieldable(d) From da4bdfec44c89bd6eace395df3a0ee43360a5309 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 14:40:33 -0600 Subject: [PATCH 132/181] Change wording of freeze docstring --- synapse/app/_base.py | 4 ++-- synapse/app/homeserver.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index baf8bb7ec21..7a9d687bbce 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -551,8 +551,8 @@ async def start(hs: "HomeServer", freeze: bool = True) -> None: freeze: whether to freeze the homeserver base objects in the garbage collector. May improve garbage collection performance by marking objects with an effectively static lifetime as frozen so they don't need to be considered for cleanup. - If you ever want to `shutdown` the homeserver in memory, this needs to be - false otherwise the homeserver cannot be garbage collected after `shutdown`. + If you ever want to `shutdown` the homeserver, this needs to be + False otherwise the homeserver cannot be garbage collected after `shutdown`. """ server_name = hs.hostname reactor = hs.get_reactor() diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 11c66f9822a..c27268a9355 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -326,8 +326,8 @@ def setup( freeze: whether to freeze the homeserver base objects in the garbage collector. May improve garbage collection performance by marking objects with an effectively static lifetime as frozen so they don't need to be considered for cleanup. - If you ever want to `shutdown` the homeserver in memory, this needs to be - false otherwise the homeserver cannot be garbage collected after `shutdown`. + If you ever want to `shutdown` the homeserver, this needs to be + False otherwise the homeserver cannot be garbage collected after `shutdown`. Returns: A homeserver instance. From a1e84145aa87703f2ff724f64c7919c18aa89681 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 18 Sep 2025 15:04:17 -0600 Subject: [PATCH 133/181] Update docstrings around using internal Clock --- scripts-dev/mypy_synapse_plugin.py | 20 +++++++++++--------- synapse/util/__init__.py | 10 ++++++---- synapse/util/async_helpers.py | 8 +++++++- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 6687f3ea2e7..30611781296 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -252,13 +252,15 @@ def get_method_signature_hook( def check_call_later(ctx: MethodSigContext) -> CallableType: """ - Ensure that the `reactor.callLater` callsites are used intentionally. - - Using `synapse.util.Clock.call_later` should be preferred. This is because the - `synapse.util.Clock` tracks delayed calls in order to cancel any outstanding calls - during server shutdown. Delayed calls which are either short lived (<~60s) or - frequently called and can be tracked via other means could be candidates for using - `reactor.callLater` directly. In those cases, use a type ignore comment to disable the + Ensure that the `reactor.callLater` callsites aren't used. + + `synapse.util.Clock.call_later` should always be used instead of `reactor.callLater`. + This is because the `synapse.util.Clock` tracks delayed calls in order to cancel any + outstanding calls during server shutdown. Delayed calls which are either short lived + (<~60s) or frequently called and can be tracked via other means could be candidates for + using `synapse.util.Clock.call_later` with `call_later_cancel_on_shutdown` set to + `False`. There shouldn't be a need to use `reactor.callLater` outside of tests or the + `Clock` class itself. If a need arises, you can use a type ignore comment to disable the check, e.g. `# type: ignore[call-later-not-tracked]`. Args: @@ -278,9 +280,9 @@ def check_call_later(ctx: MethodSigContext) -> CallableType: def check_looping_call(ctx: FunctionSigContext) -> CallableType: """ - Ensure that the `task.LoopingCall` callsites are used intentionally. + Ensure that the `task.LoopingCall` callsites aren't used. - Using `synapse.util.Clock.looping_call` should be preferred. This is because the + `synapse.util.Clock.looping_call` should always be used instead of `task.LoopingCall`. `synapse.util.Clock` tracks looping calls in order to cancel any outstanding calls during server shutdown. diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index be356db3ffd..c268b765152 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -272,14 +272,16 @@ def call_later( Args: delay: How long to wait in seconds. - cancel_on_shutdown: Whether this call should be tracked for cleanup during + callback: Function to call + *args: Postional arguments to pass to function. + call_cater_cancel_on_shutdown: Whether this call should be tracked for cleanup during shutdown. Any call with a long delay, or that is created infrequently, should be tracked. Calls which are short or of 0 delay don't require tracking since the small delay after shutdown before they trigger is immaterial. It's not worth the overhead to track those calls as it blows up - the tracking map on large server instances. - callback: Function to call - *args: Postional arguments to pass to function. + the tracking collection on large server instances. + Placed in between `*args` and `**kwargs` in order to be able to set a + default value. **kwargs: Key arguments to pass to function. """ diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index ea47ac7f820..0dfca551cea 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -782,7 +782,13 @@ def timeout_deferred( Args: deferred: The Deferred to potentially timeout. timeout: Timeout in seconds - reactor: The twisted reactor to use + cancel_on_shutdown: Whether this call should be tracked for cleanup during + shutdown. Any call with a long delay, or that is created infrequently, + should be tracked. Calls which are short or of 0 delay don't require + tracking since the small delay after shutdown before they trigger is + immaterial. It's not worth the overhead to track those calls as it blows up + the tracking collection on large server instances. + clock: The `Clock` instance used to track delayed calls. Returns: From e383758eb201544cb9b5bb537e1b6a58547c66c2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 13:25:00 -0600 Subject: [PATCH 134/181] Remove args that aren't args anymore --- scripts-dev/mypy_synapse_plugin.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 30611781296..3962a130f8f 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -265,8 +265,6 @@ def check_call_later(ctx: MethodSigContext) -> CallableType: Args: ctx: The `FunctionSigContext` from mypy. - fullname: The fully qualified name of the function being called, - e.g. `"twisted.internet.interfaces.IReactorTime.callLater"` """ signature: CallableType = ctx.default_signature ctx.api.fail( @@ -288,8 +286,6 @@ def check_looping_call(ctx: FunctionSigContext) -> CallableType: Args: ctx: The `FunctionSigContext` from mypy. - fullname: The fully qualified name of the function being called, - e.g. `"twisted.internet.task.LoopingCall"` """ signature: CallableType = ctx.default_signature ctx.api.fail( From 0ec5803364e77c34e863b0f47ac84cf4a37474a9 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 13:26:47 -0600 Subject: [PATCH 135/181] Line wrap lint errors --- scripts-dev/mypy_synapse_plugin.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 3962a130f8f..98cf05479d9 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -268,7 +268,9 @@ def check_call_later(ctx: MethodSigContext) -> CallableType: """ signature: CallableType = ctx.default_signature ctx.api.fail( - "Expected all `reactor.callLater` calls to use `synapse.util.Clock.call_later` instead. This is so that long lived calls can be tracked for cancellation during server shutdown", + "Expected all `reactor.callLater` calls to use `synapse.util.Clock.call_later` " + "instead. This is so that long lived calls can be tracked for cancellation during " + "server shutdown", ctx.context, code=INTERNAL_CLOCK_CALL_LATER_NOT_USED, ) @@ -289,7 +291,9 @@ def check_looping_call(ctx: FunctionSigContext) -> CallableType: """ signature: CallableType = ctx.default_signature ctx.api.fail( - "Expected all `task.LoopingCall` instances to use `synapse.util.Clock.looping_call` instead. This is so that long lived calls can be tracked for cancellation during server shutdown", + "Expected all `task.LoopingCall` instances to use `synapse.util.Clock.looping_call` " + "instead. This is so that long lived calls can be tracked for cancellation during " + "server shutdown", ctx.context, code=INTERNAL_CLOCK_LOOPING_CALL_NOT_USED, ) From c02f0bdb06bca3f9f41a13a3737e42f2673251b2 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 13:27:50 -0600 Subject: [PATCH 136/181] Better lint categories --- scripts-dev/mypy_synapse_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 98cf05479d9..4a534223184 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -71,13 +71,13 @@ INTERNAL_CLOCK_CALL_LATER_NOT_USED = ErrorCode( "call-later-not-tracked", "`synapse.util.Clock.call_later` should be used instead of `reactor.callLater`", - category="delayed-call-tracking", + category="synapse-reactor-clock", ) INTERNAL_CLOCK_LOOPING_CALL_NOT_USED = ErrorCode( "looping-call-not-tracked", "`synapse.util.Clock.looping_call` should be used instead of `task.LoopingCall`", - category="delayed-call-tracking", + category="synapse-reactor-clock", ) From 075ef101f339fe2296740600252703aa2a1d4b89 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 14:08:45 -0600 Subject: [PATCH 137/181] Rename lints for clarity --- scripts-dev/mypy_synapse_plugin.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 4a534223184..faec0abd1ec 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -74,8 +74,8 @@ category="synapse-reactor-clock", ) -INTERNAL_CLOCK_LOOPING_CALL_NOT_USED = ErrorCode( - "looping-call-not-tracked", +PREFER_SYNAPSE_CLOCK_LOOPING_CALL = ErrorCode( + "prefer-synapse-clock-looping-call", "`synapse.util.Clock.looping_call` should be used instead of `task.LoopingCall`", category="synapse-reactor-clock", ) @@ -295,7 +295,7 @@ def check_looping_call(ctx: FunctionSigContext) -> CallableType: "instead. This is so that long lived calls can be tracked for cancellation during " "server shutdown", ctx.context, - code=INTERNAL_CLOCK_LOOPING_CALL_NOT_USED, + code=PREFER_SYNAPSE_CLOCK_LOOPING_CALL, ) return signature From 6cc2c2f9df26b466a00988f8ebf2de0bafeebbc1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 20:09:20 +0000 Subject: [PATCH 138/181] Update synapse/util/__init__.py Co-authored-by: Eric Eastwood --- synapse/util/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index c268b765152..5b68e920df6 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -337,7 +337,7 @@ def cancel_all_delayed_calls(self, ignore_errs: bool = True) -> None: ignore_errs: Whether to re-raise errors encountered when cancelling the scheduled call. """ - # We wrap the dict in a list here since calling cancel on a delayed_call + # We make a copy here since calling `cancel()` on a delayed_call # will result in the call removing itself from the map mid-iteration. for call in list(self._call_id_to_delayed_call.values()): try: From 3e37d95dc823f4f80f47507d94592d11fceaa571 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 20:09:34 +0000 Subject: [PATCH 139/181] Update synapse/util/async_helpers.py Co-authored-by: Eric Eastwood --- synapse/util/async_helpers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 8bb2535732a..4fba8534312 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -801,11 +801,11 @@ def timeout_deferred( deferred: The Deferred to potentially timeout. timeout: Timeout in seconds cancel_on_shutdown: Whether this call should be tracked for cleanup during - shutdown. Any call with a long delay, or that is created infrequently, - should be tracked. Calls which are short or of 0 delay don't require - tracking since the small delay after shutdown before they trigger is - immaterial. It's not worth the overhead to track those calls as it blows up - the tracking collection on large server instances. + shutdown. Any call with a long delay, or that is created infrequently, + should be tracked. Calls which are short or of 0 delay don't require + tracking since the small delay after shutdown before they trigger is + immaterial. It's not worth the overhead to track those calls as it blows up + the tracking collection on large server instances. clock: The `Clock` instance used to track delayed calls. From 1c2a229655f0aab0d884ca0d6924a27a2aeabdff Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 20:10:05 +0000 Subject: [PATCH 140/181] Update tests/app/test_homeserver_shutdown.py Co-authored-by: Eric Eastwood --- tests/app/test_homeserver_shutdown.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index a282d6d82ab..bfd3671b6ac 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -51,6 +51,7 @@ def test_clean_homeserver_shutdown(self) -> None: self._runCleanups(TestResult()) self.get_success(self.reactor.shutdown()) + # Cleanup the internal reference in our test case del self.hs # Advance the reactor to allow for any outstanding calls to be run. From 63e096cb438989f03b690de94b87cbe19514b869 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 14:15:36 -0600 Subject: [PATCH 141/181] Move comment next to applicable arg --- synapse/util/caches/response_cache.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index a44504f3fd7..4a1fd480843 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -198,15 +198,15 @@ def on_complete(r: RV) -> RV: # the should_cache bit, we leave it in the cache for now and schedule # its removal later. if self.timeout_sec and context.should_cache: - # We don't need to track these calls since they don't hold and strong - # references which would keep the `HomeServer` in memory after shutdown. - # We don't want to track these because they can get cancelled really - # quickly and thrash the tracking mechanism, ie. during repeated calls - # to /sync. self.clock.call_later( self.timeout_sec, self._entry_timeout, key, + # We don't need to track these calls since they don't hold any strong + # references which would keep the `HomeServer` in memory after shutdown. + # We don't want to track these because they can get cancelled really + # quickly and thrash the tracking mechanism, ie. during repeated calls + # to /sync. call_later_cancel_on_shutdown=False, ) else: From c36aaa1518433fc7d8c331961377c4259d50865e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 14:25:23 -0600 Subject: [PATCH 142/181] Fix lint ignore renames --- synapse/metrics/_gc.py | 2 +- synapse/util/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/metrics/_gc.py b/synapse/metrics/_gc.py index 820dbca75ba..1da871f18ff 100644 --- a/synapse/metrics/_gc.py +++ b/synapse/metrics/_gc.py @@ -140,7 +140,7 @@ def _maybe_gc() -> None: # We can ignore the lint here since this looping call does not hold a `HomeServer` # reference so can be cleaned up by other means on shutdown. - gc_task = task.LoopingCall(_maybe_gc) # type: ignore[looping-call-not-tracked] + gc_task = task.LoopingCall(_maybe_gc) # type: ignore[prefer-synapse-clock-looping-call] gc_task.start(0.1) diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 5b68e920df6..44b988c23ec 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -234,7 +234,7 @@ def _looping_call_common( raise Exception("Cannot start looping call. Clock has been shutdown") # We can ignore the lint here since this is the one location LoopingCall's # should be created. - call = task.LoopingCall(f, *args, **kwargs) # type: ignore[looping-call-not-tracked] + call = task.LoopingCall(f, *args, **kwargs) # type: ignore[prefer-synapse-clock-looping-call] call.clock = self._reactor d = call.start(msec / 1000.0, now=now) d.addErrback(log_failure, "Looping call died", consumeErrors=False) From 490195f1ef7a01d4c3f7d7c622f88582475837d1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 15:02:54 -0600 Subject: [PATCH 143/181] Add comment about HTTP federation test --- tests/app/test_homeserver_shutdown.py | 10 +++++++++- tests/server.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index bfd3671b6ac..cab6a72afce 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -32,6 +32,14 @@ class HomeserverCleanShutdownTestCase(HomeserverTestCase): def setUp(self) -> None: pass + # NOTE: ideally we'd have another test to ensure we properly shutdown with + # real in-flight HTTP requests since those result in additional resources being + # setup that hold strong references to the homeserver. + # Mainly, the HTTP channel created by a real TCP connection from client to server + # is held open between requests and care needs to be taken in Twisted to ensure it is properly + # closed in a timely manner during shutdown. Simulating this behaviour in a unit test + # won't be as good as a proper integration test in complement. + def test_clean_homeserver_shutdown(self) -> None: """Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected""" self.reactor, self.clock = get_clock() @@ -49,7 +57,7 @@ def test_clean_homeserver_shutdown(self) -> None: # This works since we register `hs.shutdown()` as a cleanup function in # `setup_test_homeserver`. self._runCleanups(TestResult()) - self.get_success(self.reactor.shutdown()) + self.reactor.shutdown() # Cleanup the internal reference in our test case del self.hs diff --git a/tests/server.py b/tests/server.py index ea306926c6e..2ee51b95c0a 100644 --- a/tests/server.py +++ b/tests/server.py @@ -525,7 +525,7 @@ def getHostByName( # overwrite it again. self.nameResolver = SimpleResolverComplexifier(FakeResolver()) - async def shutdown(self) -> None: + def shutdown(self) -> None: """Cleanup any outstanding resources referenced by this reactor. Useful when trying to remove any references to the `HomeServer` that may have been registered with this fake reactor""" From 34140b36000a8b5547d62c677660beaa6c1d8d64 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 15:09:13 -0600 Subject: [PATCH 144/181] Add arg to setup_test_homeserver for cleanup --- tests/app/test_homeserver_shutdown.py | 5 ++--- tests/server.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index cab6a72afce..ee90055cc0e 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -48,15 +48,14 @@ def test_clean_homeserver_shutdown(self) -> None: reactor=self.reactor, homeserver_to_use=SynapseHomeServer, clock=self.clock, + shutdown_homeserver_on_cleanup=False, ) self.wait_for_background_updates() hs_ref = weakref.ref(self.hs) # Cleanup the homeserver. - # This works since we register `hs.shutdown()` as a cleanup function in - # `setup_test_homeserver`. - self._runCleanups(TestResult()) + self.get_success(self.hs.shutdown()) self.reactor.shutdown() # Cleanup the internal reference in our test case diff --git a/tests/server.py b/tests/server.py index 2ee51b95c0a..097e04c1f41 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1053,6 +1053,7 @@ def setup_test_homeserver( config: Optional[HomeServerConfig] = None, reactor: Optional[ISynapseReactor] = None, homeserver_to_use: Type[HomeServer] = TestHomeServer, + shutdown_homeserver_on_cleanup: bool = True, **kwargs: Any, ) -> HomeServer: """ @@ -1064,6 +1065,9 @@ def setup_test_homeserver( Args: cleanup_func : The function used to register a cleanup routine for after the test. + shutdown_homeserver_on_cleanup: Whether to include `hs.shutdown()` in the + cleanup funcs. This should always be `True` unless the test is manually + testing `hs.shutdown()`. Calling this method directly is deprecated: you should instead derive from HomeserverTestCase. @@ -1173,10 +1177,11 @@ def setup_test_homeserver( reactor=reactor, ) - # Register the cleanup hook for the homeserver. - # A full `hs.shutdown()` is necessary otherwise CI tests will fail while exhibiting - # strange behaviours. - cleanup_func(lambda: (defer.ensureDeferred(hs.shutdown()), None)[1]) + if shutdown_homeserver_on_cleanup: + # Register the cleanup hook for the homeserver. + # A full `hs.shutdown()` is necessary otherwise CI tests will fail while exhibiting + # strange behaviours. + cleanup_func(lambda: (defer.ensureDeferred(hs.shutdown()), None)[1]) # Install @cache_in_self attributes for key, val in kwargs.items(): From 9fd10db1f6b5a52a4e4b6ec90f678b2c04e58406 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 15:15:49 -0600 Subject: [PATCH 145/181] Remove unused import --- tests/app/test_homeserver_shutdown.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index ee90055cc0e..da60b868681 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -20,7 +20,6 @@ import gc import weakref -from unittest import TestResult from synapse.app.homeserver import SynapseHomeServer From 0bd1706837f60162b5b5cb7750432016a82374fa Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 19 Sep 2025 15:25:16 -0600 Subject: [PATCH 146/181] Switch test hs cleanup to use weakref --- tests/app/test_homeserver_shutdown.py | 1 - tests/server.py | 23 ++++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index da60b868681..9b20e2df779 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -47,7 +47,6 @@ def test_clean_homeserver_shutdown(self) -> None: reactor=self.reactor, homeserver_to_use=SynapseHomeServer, clock=self.clock, - shutdown_homeserver_on_cleanup=False, ) self.wait_for_background_updates() diff --git a/tests/server.py b/tests/server.py index 097e04c1f41..dedb920bb52 100644 --- a/tests/server.py +++ b/tests/server.py @@ -28,6 +28,7 @@ import time import uuid import warnings +import weakref from collections import deque from io import SEEK_END, BytesIO from typing import ( @@ -1053,7 +1054,6 @@ def setup_test_homeserver( config: Optional[HomeServerConfig] = None, reactor: Optional[ISynapseReactor] = None, homeserver_to_use: Type[HomeServer] = TestHomeServer, - shutdown_homeserver_on_cleanup: bool = True, **kwargs: Any, ) -> HomeServer: """ @@ -1065,9 +1065,6 @@ def setup_test_homeserver( Args: cleanup_func : The function used to register a cleanup routine for after the test. - shutdown_homeserver_on_cleanup: Whether to include `hs.shutdown()` in the - cleanup funcs. This should always be `True` unless the test is manually - testing `hs.shutdown()`. Calling this method directly is deprecated: you should instead derive from HomeserverTestCase. @@ -1177,11 +1174,19 @@ def setup_test_homeserver( reactor=reactor, ) - if shutdown_homeserver_on_cleanup: - # Register the cleanup hook for the homeserver. - # A full `hs.shutdown()` is necessary otherwise CI tests will fail while exhibiting - # strange behaviours. - cleanup_func(lambda: (defer.ensureDeferred(hs.shutdown()), None)[1]) + # Capture the `hs` as a `weakref` here to ensure there is no scenario where uncalled + # cleanup functions result in holding the `hs` in memory. + cleanup_hs_ref = weakref.ref(hs) + + def shutdown_hs_on_cleanup() -> None: + cleanup_hs = cleanup_hs_ref() + if cleanup_hs is not None: + defer.ensureDeferred(cleanup_hs.shutdown()) + + # Register the cleanup hook for the homeserver. + # A full `hs.shutdown()` is necessary otherwise CI tests will fail while exhibiting + # strange behaviours. + cleanup_func(shutdown_hs_on_cleanup) # Install @cache_in_self attributes for key, val in kwargs.items(): From 8b086c6a84c695fbb191f0854e7daa2d25f55ea4 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 24 Sep 2025 15:31:27 -0600 Subject: [PATCH 147/181] Return deferred from cleanup_func --- tests/server.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/server.py b/tests/server.py index dedb920bb52..31206da52fd 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1049,7 +1049,7 @@ class TestHomeServer(HomeServer): def setup_test_homeserver( - cleanup_func: Callable[[Callable[[], None]], None], + cleanup_func: Callable[[Callable[[], Optional[Deferred[None]]]], None], name: str = "test", config: Optional[HomeServerConfig] = None, reactor: Optional[ISynapseReactor] = None, @@ -1064,7 +1064,9 @@ def setup_test_homeserver( Args: cleanup_func : The function used to register a cleanup routine for - after the test. + after the test. If the function returns a Deferred, the + test case will wait until the Deferred has fired before + proceeding to the next cleanup function. Calling this method directly is deprecated: you should instead derive from HomeserverTestCase. @@ -1178,10 +1180,12 @@ def setup_test_homeserver( # cleanup functions result in holding the `hs` in memory. cleanup_hs_ref = weakref.ref(hs) - def shutdown_hs_on_cleanup() -> None: + def shutdown_hs_on_cleanup() -> Deferred[None]: cleanup_hs = cleanup_hs_ref() + deferred: Deferred[None] = defer.succeed(None) if cleanup_hs is not None: - defer.ensureDeferred(cleanup_hs.shutdown()) + deferred = defer.ensureDeferred(cleanup_hs.shutdown()) + return deferred # Register the cleanup hook for the homeserver. # A full `hs.shutdown()` is necessary otherwise CI tests will fail while exhibiting From eb16eae2648c3108a0982b3d26ee9a2aabbc1df1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 24 Sep 2025 15:45:39 -0600 Subject: [PATCH 148/181] Add debugging info for shutdown test --- tests/app/test_homeserver_shutdown.py | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 9b20e2df779..61331dde221 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -70,3 +70,40 @@ def test_clean_homeserver_shutdown(self) -> None: # weakref to it. if hs_ref() is not None: self.fail("HomeServer reference should not be valid at this point") + + # To help debug this test when it fails, it is useful to leverage the + # `objgraph` module. + # The following code serves as an example of what I have found to be useful + # when tracking down references holding the `SynapseHomeServer` in memory: + # + # all_objects = gc.get_objects() + # for obj in all_objects: + # try: + # # These are a subset of types that are typically involved with + # # holding the `HomeServer` in memory. You may want to inspect + # # other types as well. + # if isinstance(obj, DataStore): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # db_obj = obj + # if isinstance(obj, SynapseHomeServer): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # synapse_hs = obj + # if isinstance(obj, SynapseSite): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # sysite = obj + # if isinstance(obj, DatabasePool): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # dbpool = obj + # except Exception: + # pass + # + # print(sys.getrefcount(hs_ref()), "refs to", hs_ref()) + # + # # The following values for `max_depth` and `too_many` have been found to + # # render a useful amount of information without taking an overly long time + # # to generate the result. + # objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10) From 8855f4a313574b4028e94492734e9f01642a7ab5 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Wed, 24 Sep 2025 16:40:32 -0600 Subject: [PATCH 149/181] Override MemoryReactor in tests for clean shutdown --- tests/app/test_homeserver_shutdown.py | 4 +- tests/server.py | 58 ++++++++++++++++++++++----- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 61331dde221..ffa3a98e3ff 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -54,7 +54,9 @@ def test_clean_homeserver_shutdown(self) -> None: # Cleanup the homeserver. self.get_success(self.hs.shutdown()) - self.reactor.shutdown() + + # Run the reactor so any `callWhenRunning` functions can be cleared out. + self.reactor.run() # Cleanup the internal reference in our test case del self.hs diff --git a/tests/server.py b/tests/server.py index 31206da52fd..bba2813a538 100644 --- a/tests/server.py +++ b/tests/server.py @@ -526,17 +526,57 @@ def getHostByName( # overwrite it again. self.nameResolver = SimpleResolverComplexifier(FakeResolver()) - def shutdown(self) -> None: - """Cleanup any outstanding resources referenced by this reactor. Useful when - trying to remove any references to the `HomeServer` that may have been - registered with this fake reactor""" + # type-ignore: `MemoryReactor` returns `None` whereas the + # `twisted.internet.interfaces.IReactorCore` interface allows a return type + # of `Any`, so long as the ID type matches that used by `removeSystemEventTrigger`. + def addSystemEventTrigger( # type: ignore[override] + self, + phase: str, + eventType: str, + callable: Callable[P, object], + *args: P.args, + **kw: P.kwargs, + ) -> int: + """ + Override the call from `MemoryReactorClock` in order to track registered event + triggers by `triggerID`. + Otherwise the code here has been copied from `MemoryReactorClock`. + """ + phaseTriggers = self.triggers.setdefault(phase, {}) + eventTypeTriggers = phaseTriggers.setdefault(eventType, []) + trigger = (callable, args, kw) + eventTypeTriggers.append(trigger) + return id(trigger) - # `MemoryReactorClock.removeSystemEventTrigger` is not implemented. - # So manually clear the triggers here. - self.triggers.clear() + def removeSystemEventTrigger(self, triggerID: int) -> None: + """ + Override the unimplemented call from `MemoryReactorClock` in order to actually remove + registered event triggers. + This is necessary for a clean shutdown to occur as these triggers can hold + references to the `SynapseHomeServer`. + + args: + triggerID: Unique ID obtained from `addSystemEventTrigger`. + """ + for phase_triggers in self.triggers.values(): + for event_type_triggers in phase_triggers.values(): + triggers_to_remove = [ + item for item in event_type_triggers if id(item) == triggerID + ] + for trigger in triggers_to_remove: + event_type_triggers.remove(trigger) + + def run(self) -> None: + """ + Override the call from `MemoryReactorClock` to add an additional step that + cleans up any `whenRunningHooks` that have been called. + This is necessary for a clean shutdown to occur as these hooks can hold + references to the `SynapseHomeServer`. + """ + super().run() - # `MemoryReactorClock` never clears the hooks made when `callWhenRunning` is - # called. So manually clear the hooks here. + # `MemoryReactorClock` never clears the hooks that have already been called. + # So manually clear the hooks here after they have been run. self.whenRunningHooks.clear() def installNameResolver(self, resolver: IHostnameResolver) -> IHostnameResolver: From 0f23e500e9926e58b2ffccdbecc3230d909a6bd7 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 25 Sep 2025 14:48:11 -0500 Subject: [PATCH 150/181] Fix test not follow Synapse logcontext rules --- tests/util/test_async_helpers.py | 49 ++++++++++++++------------------ 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index d8d04edebbd..30a78334d56 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -17,8 +17,9 @@ # [This file includes modifications made by New Vector Limited] # # +import logging import traceback -from typing import Any, Coroutine, Generator, List, NoReturn, Optional, Tuple, TypeVar +from typing import Any, Coroutine, List, NoReturn, Optional, Tuple, TypeVar from parameterized import parameterized_class @@ -46,6 +47,8 @@ from tests.server import get_clock from tests.unittest import TestCase +logger = logging.getLogger(__name__) + T = TypeVar("T") @@ -197,34 +200,23 @@ def canceller(_d: Deferred) -> None: self.failureResultOf(timing_out_d, defer.TimeoutError) - def test_logcontext_is_preserved_on_cancellation(self) -> None: + async def test_logcontext_is_preserved_on_cancellation(self) -> None: blocking_was_cancelled = False - @defer.inlineCallbacks - def blocking() -> Generator["Deferred[object]", object, None]: + def mark_blocking_was_cancelled(res: Failure) -> None: nonlocal blocking_was_cancelled + if res.check(CancelledError): + blocking_was_cancelled = True + res.raiseException() + else: + logger.error("Expected incomplete_d to be cancelled but saw %s", res) - non_completing_d: Deferred = Deferred() - with PreserveLoggingContext(): - try: - yield non_completing_d - except CancelledError: - blocking_was_cancelled = True - raise + incomplete_d: Deferred = Deferred() + incomplete_d.addErrback(mark_blocking_was_cancelled) with LoggingContext("one") as context_one: - # the errbacks should be run in the test logcontext - def errback(res: Failure, deferred_name: str) -> Failure: - self.assertIs( - current_context(), - context_one, - "errback %s run in unexpected logcontext %s" - % (deferred_name, current_context()), - ) - return res - - original_deferred = blocking() - original_deferred.addErrback(errback, "orig") + # original_deferred = blocking() + original_deferred = incomplete_d timing_out_d = timeout_deferred( deferred=original_deferred, timeout=1.0, @@ -232,13 +224,16 @@ def errback(res: Failure, deferred_name: str) -> Failure: clock=self.clock, ) self.assertNoResult(timing_out_d) - self.assertIs(current_context(), SENTINEL_CONTEXT) - timing_out_d.addErrback(errback, "timingout") + self.assertIs(current_context(), context_one) - self.reactor.pump((2.0,)) + # We're yielding control to the reactor (and calling any pending callbacks) + # so we need to be in the sentinel logcontext to avoid leaking our current + # logcontext into the reactor. + with PreserveLoggingContext(): + self.reactor.pump((2.0,)) self.assertTrue( - blocking_was_cancelled, "non-completing deferred was not cancelled" + blocking_was_cancelled, "incomplete deferred was not cancelled" ) self.failureResultOf(timing_out_d, defer.TimeoutError) self.assertIs(current_context(), context_one) From 3f5c46310324bc68e34c1df5ed6d572f3f11ede1 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 25 Sep 2025 15:04:41 -0500 Subject: [PATCH 151/181] Clean up test --- tests/util/test_async_helpers.py | 66 ++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index 30a78334d56..189774e73ed 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -201,45 +201,79 @@ def canceller(_d: Deferred) -> None: self.failureResultOf(timing_out_d, defer.TimeoutError) async def test_logcontext_is_preserved_on_cancellation(self) -> None: - blocking_was_cancelled = False + # Sanity check that we start in the sentinel context + self.assertEqual(current_context(), SENTINEL_CONTEXT) - def mark_blocking_was_cancelled(res: Failure) -> None: - nonlocal blocking_was_cancelled + incomplete_deferred_was_cancelled = False + + def mark_was_cancelled(res: Failure) -> None: + """ + A passthrough errback which sets `incomplete_deferred_was_cancelled`. + + This means we re-raise any exception and allows further errbacks (in + `timeout_deferred(...)`) to do their thing. Just trying to be a transparent + proxy of any exception while doing our internal test book-keeping. + """ + nonlocal incomplete_deferred_was_cancelled if res.check(CancelledError): - blocking_was_cancelled = True - res.raiseException() + incomplete_deferred_was_cancelled = True else: - logger.error("Expected incomplete_d to be cancelled but saw %s", res) + logger.error( + "Expected incomplete_d to fail with `CancelledError` because our " + "`timeout_deferred(...)` utility canceled it but saw %s", + res, + ) + # Re-raise the exception so that any further errbacks can do their thing as + # normal + res.raiseException() + + # Create a deferred which we will never complete incomplete_d: Deferred = Deferred() - incomplete_d.addErrback(mark_blocking_was_cancelled) + incomplete_d.addErrback(mark_was_cancelled) with LoggingContext("one") as context_one: - # original_deferred = blocking() - original_deferred = incomplete_d timing_out_d = timeout_deferred( - deferred=original_deferred, + deferred=incomplete_d, timeout=1.0, cancel_on_shutdown=False, clock=self.clock, ) self.assertNoResult(timing_out_d) + # We should still be in the logcontext we started in self.assertIs(current_context(), context_one) - # We're yielding control to the reactor (and calling any pending callbacks) - # so we need to be in the sentinel logcontext to avoid leaking our current - # logcontext into the reactor. + # Pump the reactor until we trigger the timeout + # + # We're manually pumping the reactor (and causing any pending callbacks to + # be called) so we need to be in the sentinel logcontext to avoid leaking + # our current logcontext into the reactor (which would then get picked up + # and associated with the next thing the reactor does). `with + # PreserveLoggingContext()` will reset the logcontext to the sentinel while + # we're pumping the reactor in the block and return us back to our current + # logcontext after the block. with PreserveLoggingContext(): - self.reactor.pump((2.0,)) + self.reactor.pump( + # This is `2.0` (seconds) as we set `timeout_deferred(timeout=1.0)` above + (2.0,) + ) + # We expect the incomplete deferred to have been cancelled because of the + # timeout by this point self.assertTrue( - blocking_was_cancelled, "incomplete deferred was not cancelled" + incomplete_deferred_was_cancelled, + "incomplete deferred was not cancelled", ) + # We should see the `TimeoutError` (instead of a `CancelledError`) self.failureResultOf(timing_out_d, defer.TimeoutError) + # We're still in the same logcontext self.assertIs(current_context(), context_one) + # Back to the sentinel context + self.assertEqual(current_context(), SENTINEL_CONTEXT) + -class _TestException(Exception): +class _TestException(Exception): # pass From 130dcda816df4a9bdc5e868679bf9e869876bc9c Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 25 Sep 2025 14:26:33 -0600 Subject: [PATCH 152/181] Re-add call_later cancellation tracking --- synapse/util/clock.py | 95 ++++++++++++++++--------------------------- 1 file changed, 34 insertions(+), 61 deletions(-) diff --git a/synapse/util/clock.py b/synapse/util/clock.py index e2e3ef879c4..d01b4514d34 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -252,53 +252,13 @@ def call_later( **kwargs: Key arguments to pass to function. """ - # if self._is_shutdown: - # raise Exception("Cannot start delayed call. Clock has been shutdown") - - # TODO: cleanup by passing in callback arg - def wrapped_callback(*args: Any, **kwargs: Any) -> None: - assert context.current_context() is context.SENTINEL_CONTEXT, ( - "Expected `call_later` callback from the reactor to start with the sentinel logcontext " - f"but saw {context.current_context()}. In other words, another task shouldn't have " - "leaked their logcontext to us." - ) - - # Because this is a callback from the reactor, we will be using the - # `sentinel` log context at this point. We want the function to log with - # some logcontext as we want to know which server the logs came from. - # - # We use `PreserveLoggingContext` to prevent our new `call_later` - # logcontext from finishing as soon as we exit this function, in case `f` - # returns an awaitable/deferred which would continue running and may try to - # restore the `loop_call` context when it's done (because it's trying to - # adhere to the Synapse logcontext rules.) - # - # This also ensures that we return to the `sentinel` context when we exit - # this function and yield control back to the reactor to avoid leaking the - # current logcontext to the reactor (which would then get picked up and - # associated with the next thing the reactor does) - with context.PreserveLoggingContext(context.LoggingContext("call_later")): - # We use `run_in_background` to reset the logcontext after `f` (or the - # awaitable returned by `f`) completes to avoid leaking the current - # logcontext to the reactor - context.run_in_background(callback, *args, **kwargs) - return self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] - - if call_later_cancel_on_shutdown: - call_id = self._delayed_call_id - self._delayed_call_id = self._delayed_call_id + 1 - - def tracked_callback(*args: Any, **kwargs: Any) -> None: - try: - callback(*args, **kwargs) - except Exception: - raise - finally: - # We still want to remove the call from the tracking map. Even if - # the callback raises an exception. - self._call_id_to_delayed_call.pop(call_id) + if self._is_shutdown: + raise Exception("Cannot start delayed call. Clock has been shutdown") - def tracked_wrapped_callback(*args: Any, **kwargs: Any) -> None: + def create_wrapped_callback( + track_for_shutdown_cancellation: bool, + ) -> Callable[P, None]: + def wrapped_callback(*args: Any, **kwargs: Any) -> None: assert context.current_context() is context.SENTINEL_CONTEXT, ( "Expected `call_later` callback from the reactor to start with the sentinel logcontext " f"but saw {context.current_context()}. In other words, another task shouldn't have " @@ -319,27 +279,40 @@ def tracked_wrapped_callback(*args: Any, **kwargs: Any) -> None: # this function and yield control back to the reactor to avoid leaking the # current logcontext to the reactor (which would then get picked up and # associated with the next thing the reactor does) - with context.PreserveLoggingContext( - context.LoggingContext("call_later") - ): - # We use `run_in_background` to reset the logcontext after `f` (or the - # awaitable returned by `f`) completes to avoid leaking the current - # logcontext to the reactor - context.run_in_background(tracked_callback, *args, **kwargs) + try: + with context.PreserveLoggingContext( + context.LoggingContext("call_later") + ): + # We use `run_in_background` to reset the logcontext after `f` (or the + # awaitable returned by `f`) completes to avoid leaking the current + # logcontext to the reactor + context.run_in_background(callback, *args, **kwargs) + finally: + if track_for_shutdown_cancellation: + # We still want to remove the call from the tracking map. Even if + # the callback raises an exception. + self._call_id_to_delayed_call.pop(call_id) + + return wrapped_callback + + if call_later_cancel_on_shutdown: + call_id = self._delayed_call_id + self._delayed_call_id = self._delayed_call_id + 1 # We can ignore the lint here since this class is the one location callLater # should be called. - #call = self._reactor.callLater( - # delay, tracked_wrapped_callback, *args, **kwargs - #) # type: ignore[call-later-not-tracked] - #call = DelayedCallWrapper(call, call_id, self) - #self._call_id_to_delayed_call[call_id] = call - #return call - return self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] + call = self._reactor.callLater( + delay, create_wrapped_callback(True), *args, **kwargs + ) # type: ignore[call-later-not-tracked] + call = DelayedCallWrapper(call, call_id, self) + self._call_id_to_delayed_call[call_id] = call + return call else: # We can ignore the lint here since this class is the one location callLater should # be called. - return self._reactor.callLater(delay, wrapped_callback, *args, **kwargs) # type: ignore[call-later-not-tracked] + return self._reactor.callLater( + delay, create_wrapped_callback(False), *args, **kwargs + ) # type: ignore[call-later-not-tracked] def cancel_call_later(self, timer: IDelayedCall, ignore_errs: bool = False) -> None: try: From 30dbc2b55c2b660779360c4cc5ff45902684fc69 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 25 Sep 2025 15:12:05 -0600 Subject: [PATCH 153/181] Add shutdown test where background updates haven't completed --- tests/app/test_homeserver_shutdown.py | 87 ++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index ffa3a98e3ff..5cfb046778b 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -22,6 +22,7 @@ import weakref from synapse.app.homeserver import SynapseHomeServer +from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from tests.server import get_clock, setup_test_homeserver from tests.unittest import HomeserverTestCase @@ -52,18 +53,100 @@ def test_clean_homeserver_shutdown(self) -> None: hs_ref = weakref.ref(self.hs) + # Run the reactor so any `callWhenRunning` functions can be cleared out. + self.reactor.run() + # Cleanup the homeserver. self.get_success(self.hs.shutdown()) + # Cleanup the internal reference in our test case + del self.hs + + # Advance the reactor to allow for any outstanding calls to be run. + # A value of 1 is not enough, so a value of 2 is used. + self.reactor.advance(2) + + # Force garbage collection. + gc.collect() + + # Ensure the `HomeServer` hs been garbage collected by attempting to use the + # weakref to it. + if hs_ref() is not None: + self.fail("HomeServer reference should not be valid at this point") + + # To help debug this test when it fails, it is useful to leverage the + # `objgraph` module. + # The following code serves as an example of what I have found to be useful + # when tracking down references holding the `SynapseHomeServer` in memory: + # + # all_objects = gc.get_objects() + # for obj in all_objects: + # try: + # # These are a subset of types that are typically involved with + # # holding the `HomeServer` in memory. You may want to inspect + # # other types as well. + # if isinstance(obj, DataStore): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # db_obj = obj + # if isinstance(obj, SynapseHomeServer): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # synapse_hs = obj + # if isinstance(obj, SynapseSite): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # sysite = obj + # if isinstance(obj, DatabasePool): + # print(sys.getrefcount(obj), "refs to", obj) + # if not isinstance(obj, weakref.ProxyType): + # dbpool = obj + # except Exception: + # pass + # + # print(sys.getrefcount(hs_ref()), "refs to", hs_ref()) + # + # # The following values for `max_depth` and `too_many` have been found to + # # render a useful amount of information without taking an overly long time + # # to generate the result. + # objgraph.show_backrefs(synapse_hs, max_depth=10, too_many=10) + + def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: + """Ensure the `SynapseHomeServer` can be fully shutdown and garbage collected + before background updates have completed""" + self.reactor, self.clock = get_clock() + self.hs = setup_test_homeserver( + self.addCleanup, + reactor=self.reactor, + homeserver_to_use=SynapseHomeServer, + clock=self.clock, + ) + + # Pump the background updates by a single iteration, just to ensure any extra + # resources it uses have been started. + store = weakref.proxy(self.hs.get_datastores().main) + self.get_success(store.db_pool.updates.do_next_background_update(False), by=0.1) + + hs_ref = weakref.ref(self.hs) + # Run the reactor so any `callWhenRunning` functions can be cleared out. self.reactor.run() + # Also advance the reactor by the delay tracking threshold to ensure all + # cancellable delayed calls have been scheduled. Must be done prior to + # `hs.shutdown()` otherwise they will be scheduled later during the test when we + # advance the reactor to wait out any non-tracked delayed calls. + self.reactor.advance(CALL_LATER_DELAY_TRACKING_THRESHOLD_S) + + # Cleanup the homeserver. + self.get_success(self.hs.shutdown()) # Cleanup the internal reference in our test case del self.hs # Advance the reactor to allow for any outstanding calls to be run. - # A value of 1 is not enough, so a value of 2 is used. - self.reactor.advance(2) + # The limit of 15 is currently set by having to wait for the delayed call + # `synapse.storage.database.py -> _check_safe_to_upsert`. + self.reactor.advance(15) # Force garbage collection. gc.collect() From 03b95947e27e87e0d8055d9e3a45ca454fc702c1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Thu, 25 Sep 2025 15:14:28 -0600 Subject: [PATCH 154/181] Add assert that updates arent complete --- tests/app/test_homeserver_shutdown.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 5cfb046778b..d3a740dc414 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -22,6 +22,7 @@ import weakref from synapse.app.homeserver import SynapseHomeServer +from synapse.storage.background_updates import UpdaterStatus from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from tests.server import get_clock, setup_test_homeserver @@ -125,7 +126,9 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # Pump the background updates by a single iteration, just to ensure any extra # resources it uses have been started. store = weakref.proxy(self.hs.get_datastores().main) - self.get_success(store.db_pool.updates.do_next_background_update(False), by=0.1) + self.get_success( + store.db_pool.updates.do_next_background_update(False), by=0.1 + ) hs_ref = weakref.ref(self.hs) @@ -137,6 +140,9 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # advance the reactor to wait out any non-tracked delayed calls. self.reactor.advance(CALL_LATER_DELAY_TRACKING_THRESHOLD_S) + # Ensure the background updates are not complete. + self.assertNotEqual(store.db_pool.updates.get_status(), UpdaterStatus.COMPLETE) + # Cleanup the homeserver. self.get_success(self.hs.shutdown()) From d7f8c1cf620f466c72c6bb190200283a6c196d72 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 26 Sep 2025 11:33:13 -0600 Subject: [PATCH 155/181] Initial conversion run_as_background_process to use hs --- synapse/_scripts/update_synapse_database.py | 5 +- synapse/crypto/keyring.py | 4 +- synapse/handlers/typing.py | 11 ++-- synapse/metrics/background_process_metrics.py | 1 + synapse/metrics/common_usage_metrics.py | 8 +-- synapse/server.py | 58 +++++++++++++++++++ synapse/storage/background_updates.py | 5 +- synapse/storage/database.py | 2 +- .../storage/databases/main/events_worker.py | 5 +- synapse/util/batching_queue.py | 14 +++-- tests/app/test_homeserver_shutdown.py | 9 +-- tests/util/test_batching_queue.py | 25 ++++---- 12 files changed, 94 insertions(+), 53 deletions(-) diff --git a/synapse/_scripts/update_synapse_database.py b/synapse/_scripts/update_synapse_database.py index caaecda1617..ad02f0ed887 100644 --- a/synapse/_scripts/update_synapse_database.py +++ b/synapse/_scripts/update_synapse_database.py @@ -28,7 +28,6 @@ from twisted.internet import defer, reactor as reactor_ from synapse.config.homeserver import HomeServerConfig -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.server import HomeServer from synapse.storage import DataStore from synapse.types import ISynapseReactor @@ -53,7 +52,6 @@ def __init__(self, config: HomeServerConfig): def run_background_updates(hs: HomeServer) -> None: - server_name = hs.hostname main = hs.get_datastores().main state = hs.get_datastores().state @@ -67,9 +65,8 @@ async def run_background_updates() -> None: def run() -> None: # Apply all background updates on the database. defer.ensureDeferred( - run_as_background_process( + hs.run_as_background_process( "background_updates", - server_name, run_background_updates, ) ) diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index a881534da2a..9df7be04c8a 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -172,7 +172,7 @@ def __init__( _FetchKeyRequest, Dict[str, Dict[str, FetchKeyResult]] ] = BatchingQueue( name="keyring_server", - server_name=self.server_name, + hs=hs, clock=hs.get_clock(), # The method called to fetch each key process_batch_callback=self._inner_fetch_key_requests, @@ -487,7 +487,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self._queue = BatchingQueue( name=self.__class__.__name__, - server_name=self.server_name, + hs=hs, clock=hs.get_clock(), process_batch_callback=self._fetch_keys, ) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 6a7b36ea0c8..7e62df4f4c3 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -28,7 +28,6 @@ from synapse.api.errors import AuthError, ShadowBanError, SynapseError from synapse.appservice import ApplicationService from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.tcp.streams import TypingStream @@ -78,6 +77,7 @@ class FollowerTypingHandler: """ def __init__(self, hs: "HomeServer"): + self.hs = hs self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.server_name = ( @@ -144,9 +144,8 @@ def _handle_timeout_for_member(self, now: int, member: RoomMember) -> None: if self.federation and self.is_mine_id(member.user_id): last_fed_poke = self._member_last_federation_poke.get(member, None) if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now: - run_as_background_process( + self.hs.run_as_background_process( "typing._push_remote", - self.server_name, self._push_remote, member=member, typing=True, @@ -220,9 +219,8 @@ def process_replication_rows( self._rooms_updated.add(row.room_id) if self.federation: - run_as_background_process( + self.hs.run_as_background_process( "_send_changes_in_typing_to_remotes", - self.server_name, self._send_changes_in_typing_to_remotes, row.room_id, prev_typing, @@ -384,9 +382,8 @@ def _stopped_typing(self, member: RoomMember) -> None: def _push_update(self, member: RoomMember, typing: bool) -> None: if self.hs.is_mine_id(member.user_id): # Only send updates for changes to our own users. - run_as_background_process( + self.hs.run_as_background_process( "typing._push_remote", - self.server_name, self._push_remote, member, typing, diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 633705b02a3..6a55e707d65 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -341,6 +341,7 @@ def func(*args): ... multiple places. """ + # TODO: ensure it has a hs object, not just servername def wrapper( func: Callable[Concatenate[HasServerName, P], Awaitable[Optional[R]]], ) -> Callable[P, "defer.Deferred[Optional[R]]"]: diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py index cd1c3c86499..43e0913d279 100644 --- a/synapse/metrics/common_usage_metrics.py +++ b/synapse/metrics/common_usage_metrics.py @@ -23,7 +23,6 @@ import attr from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process if TYPE_CHECKING: from synapse.server import HomeServer @@ -52,6 +51,7 @@ def __init__(self, hs: "HomeServer") -> None: self.server_name = hs.hostname self._store = hs.get_datastores().main self._clock = hs.get_clock() + self._hs = hs async def get_metrics(self) -> CommonUsageMetrics: """Get the CommonUsageMetrics object. If no collection has happened yet, do it @@ -64,16 +64,14 @@ async def get_metrics(self) -> CommonUsageMetrics: async def setup(self) -> None: """Keep the gauges for common usage metrics up to date.""" - run_as_background_process( + self._hs.run_as_background_process( desc="common_usage_metrics_update_gauges", - server_name=self.server_name, func=self._update_gauges, ) self._clock.looping_call( - run_as_background_process, + self._hs.run_as_background_process, 5 * 60 * 1000, desc="common_usage_metrics_update_gauges", - server_name=self.server_name, func=self._update_gauges, ) diff --git a/synapse/server.py b/synapse/server.py index c7c410bd49c..c2ec01eccf4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -32,6 +32,7 @@ from typing import ( TYPE_CHECKING, Any, + Awaitable, Callable, Dict, List, @@ -217,6 +218,7 @@ T: TypeAlias = object F = TypeVar("F", bound=Callable[["HomeServer"], T]) +R = TypeVar("R") def cache_in_self(builder: F) -> F: @@ -359,6 +361,55 @@ def __init__( self._async_shutdown_handlers: List[ShutdownInfo] = [] self._sync_shutdown_handlers: List[ShutdownInfo] = [] + self._background_processes: set[defer.Deferred[Optional[Any]]] = set() + + def run_as_background_process( + self, + desc: "LiteralString", + func: Callable[..., Awaitable[Optional[R]]], + *args: Any, + **kwargs: Any, + ) -> "defer.Deferred[Optional[R]]": + """Run the given function in its own logcontext, with resource metrics + + This should be used to wrap processes which are fired off to run in the + background, instead of being associated with a particular request. + + It returns a Deferred which completes when the function completes, but it doesn't + follow the synapse logcontext rules, which makes it appropriate for passing to + clock.looping_call and friends (or for firing-and-forgetting in the middle of a + normal synapse async function). + + Because the returned Deferred does not follow the synapse logcontext rules, awaiting + the result of this function will result in the log context being cleared (bad). In + order to properly await the result of this function and maintain the current log + context, use `make_deferred_yieldable`. + + Args: + desc: a description for this background process type + server_name: The homeserver name that this background process is being run for + (this should be `hs.hostname`). + func: a function, which may return a Deferred or a coroutine + bg_start_span: Whether to start an opentracing span. Defaults to True. + Should only be disabled for processes that will not log to or tag + a span. + args: positional args for func + kwargs: keyword args for func + + Returns: + Deferred which returns the result of func, or `None` if func raises. + Note that the returned Deferred does not follow the synapse logcontext + rules. + """ + deferred = run_as_background_process(desc, self.hostname, func, *args, **kwargs) + self._background_processes.add(deferred) + + def on_done(res: R) -> R: + self._background_processes.remove(deferred) + return res + + deferred.addBoth(on_done) + return deferred async def shutdown(self) -> None: """ @@ -430,6 +481,13 @@ async def shutdown(self) -> None: self.get_clock().shutdown() + for background_process in list(self._background_processes): + try: + background_process.cancel() + except Exception: + pass + self._background_processes.clear() + for shutdown_handler in self._async_shutdown_handlers: try: self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index ddc806eab08..4509691d79b 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -41,7 +41,6 @@ import attr from synapse._pydantic_compat import BaseModel -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.engines import PostgresEngine from synapse.storage.types import Connection, Cursor from synapse.types import JsonDict, StrCollection @@ -403,14 +402,14 @@ def start_doing_background_updates(self) -> None: # if we start a new background update, not all updates are done. self._all_done = False sleep = self.sleep_enabled - run_as_background_process( + self.hs.run_as_background_process( "background_updates", - self.server_name, self.run_background_updates, sleep, ) async def run_background_updates(self, sleep: bool) -> None: + logger.error("running background updates") if self._running or not self.enabled: return diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 9bc3f0c25c2..fa97e9c93e0 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -688,7 +688,7 @@ async def _check_safe_to_upsert(self) -> None: if background_update_names: self._clock.call_later( 15.0, - run_as_background_process, + self.hs.run_as_background_process, "upsert_safety_check", self.server_name, self._check_safe_to_upsert, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 31e23122115..5b62317c498 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -70,7 +70,6 @@ ) from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.tcp.streams import BackfillStream, UnPartialStatedEventStream @@ -1154,9 +1153,7 @@ def _maybe_start_fetch_thread(self) -> None: should_start = False if should_start: - run_as_background_process( - "fetch_events", self.server_name, self._fetch_thread - ) + self.hs.run_as_background_process("fetch_events", self._fetch_thread) async def _fetch_thread(self) -> None: """Services requests for events from `_event_fetch_list`.""" diff --git a/synapse/util/batching_queue.py b/synapse/util/batching_queue.py index b7070931a5d..f77301afd81 100644 --- a/synapse/util/batching_queue.py +++ b/synapse/util/batching_queue.py @@ -21,6 +21,7 @@ import logging from typing import ( + TYPE_CHECKING, Awaitable, Callable, Dict, @@ -38,9 +39,11 @@ from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.clock import Clock +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -97,12 +100,13 @@ def __init__( self, *, name: str, - server_name: str, + hs: "HomeServer", clock: Clock, process_batch_callback: Callable[[List[V]], Awaitable[R]], ): self._name = name - self.server_name = server_name + self.hs = hs + self.server_name = hs.hostname self._clock = clock # The set of keys currently being processed. @@ -153,9 +157,7 @@ async def add_to_queue(self, value: V, key: Hashable = ()) -> R: # If we're not currently processing the key fire off a background # process to start processing. if key not in self._processing_keys: - run_as_background_process( - self._name, self.server_name, self._process_queue, key - ) + self.hs.run_as_background_process(self._name, self._process_queue, key) with self._number_in_flight_metric.track_inprogress(): return await make_deferred_yieldable(d) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index d3a740dc414..b629a500f27 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -126,9 +126,7 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # Pump the background updates by a single iteration, just to ensure any extra # resources it uses have been started. store = weakref.proxy(self.hs.get_datastores().main) - self.get_success( - store.db_pool.updates.do_next_background_update(False), by=0.1 - ) + self.get_success(store.db_pool.updates.do_next_background_update(False), by=0.1) hs_ref = weakref.ref(self.hs) @@ -149,11 +147,6 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # Cleanup the internal reference in our test case del self.hs - # Advance the reactor to allow for any outstanding calls to be run. - # The limit of 15 is currently set by having to wait for the delayed call - # `synapse.storage.database.py -> _check_safe_to_upsert`. - self.reactor.advance(15) - # Force garbage collection. gc.collect() diff --git a/tests/util/test_batching_queue.py b/tests/util/test_batching_queue.py index 532582cf877..60bfdf38aaa 100644 --- a/tests/util/test_batching_queue.py +++ b/tests/util/test_batching_queue.py @@ -32,13 +32,12 @@ number_queued, ) -from tests.server import get_clock -from tests.unittest import TestCase +from tests.unittest import HomeserverTestCase -class BatchingQueueTestCase(TestCase): +class BatchingQueueTestCase(HomeserverTestCase): def setUp(self) -> None: - self.clock, hs_clock = get_clock() + super().setUp() # We ensure that we remove any existing metrics for "test_queue". try: @@ -51,8 +50,8 @@ def setUp(self) -> None: self._pending_calls: List[Tuple[List[str], defer.Deferred]] = [] self.queue: BatchingQueue[str, str] = BatchingQueue( name="test_queue", - server_name="test_server", - clock=hs_clock, + hs=self.hs, + clock=self.clock, process_batch_callback=self._process_queue, ) @@ -108,7 +107,7 @@ def test_simple(self) -> None: self.assertFalse(queue_d.called) # We should see a call to `_process_queue` after a reactor tick. - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) self.assertEqual(self._pending_calls[0][0], ["foo"]) @@ -134,7 +133,7 @@ def test_batching(self) -> None: self._assert_metrics(queued=2, keys=1, in_flight=2) - self.clock.pump([0]) + self.reactor.pump([0]) # We should see only *one* call to `_process_queue` self.assertEqual(len(self._pending_calls), 1) @@ -158,7 +157,7 @@ def test_queuing(self) -> None: self.assertFalse(self._pending_calls) queue_d1 = defer.ensureDeferred(self.queue.add_to_queue("foo1")) - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) @@ -185,7 +184,7 @@ def test_queuing(self) -> None: self._assert_metrics(queued=2, keys=1, in_flight=2) # We should now see a second call to `_process_queue` - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) self.assertEqual(self._pending_calls[0][0], ["foo2", "foo3"]) self.assertFalse(queue_d2.called) @@ -206,9 +205,9 @@ def test_different_keys(self) -> None: self.assertFalse(self._pending_calls) queue_d1 = defer.ensureDeferred(self.queue.add_to_queue("foo1", key=1)) - self.clock.pump([0]) + self.reactor.pump([0]) queue_d2 = defer.ensureDeferred(self.queue.add_to_queue("foo2", key=2)) - self.clock.pump([0]) + self.reactor.pump([0]) # We queue up another item with key=2 to check that we will keep taking # things off the queue. @@ -240,7 +239,7 @@ def test_different_keys(self) -> None: self.assertFalse(queue_d3.called) # We should now see a call `_pending_calls` for `foo3` - self.clock.pump([0]) + self.reactor.pump([0]) self.assertEqual(len(self._pending_calls), 1) self.assertEqual(self._pending_calls[0][0], ["foo3"]) self.assertFalse(queue_d3.called) From b1f887c799bbd47d06621f3f5860c07c022d60cf Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 26 Sep 2025 11:51:43 -0600 Subject: [PATCH 156/181] Can't subscript Deferred in python 3.9 --- tests/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server.py b/tests/server.py index 3378b538e8c..83a30b211d8 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1089,7 +1089,7 @@ class TestHomeServer(HomeServer): def setup_test_homeserver( - cleanup_func: Callable[[Callable[[], Optional[Deferred[None]]]], None], + cleanup_func: Callable[[Callable[[], Optional["Deferred[None]"]]], None], name: str = "test", config: Optional[HomeServerConfig] = None, reactor: Optional[ISynapseReactor] = None, From d6d4780eee653ed79c7651f8b59f6ad0a452d276 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 26 Sep 2025 12:01:15 -0600 Subject: [PATCH 157/181] Can't subscript Deferred in python 3.9 --- tests/server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/server.py b/tests/server.py index 83a30b211d8..32f39e389e2 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1220,9 +1220,9 @@ def setup_test_homeserver( # cleanup functions result in holding the `hs` in memory. cleanup_hs_ref = weakref.ref(hs) - def shutdown_hs_on_cleanup() -> Deferred[None]: + def shutdown_hs_on_cleanup() -> "Deferred[None]": cleanup_hs = cleanup_hs_ref() - deferred: Deferred[None] = defer.succeed(None) + deferred: "Deferred[None]" = defer.succeed(None) if cleanup_hs is not None: deferred = defer.ensureDeferred(cleanup_hs.shutdown()) return deferred From 34d314beca49849e3c3ec8f6068e494e6099bf75 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 26 Sep 2025 15:08:54 -0600 Subject: [PATCH 158/181] Fix removeSystemEventTrigger in tests --- synapse/storage/background_updates.py | 1 - tests/app/test_homeserver_shutdown.py | 13 ++++++- tests/server.py | 53 +++++++-------------------- 3 files changed, 25 insertions(+), 42 deletions(-) diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index 4509691d79b..e3e793d5f59 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -409,7 +409,6 @@ def start_doing_background_updates(self) -> None: ) async def run_background_updates(self, sleep: bool) -> None: - logger.error("running background updates") if self._running or not self.enabled: return diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index b629a500f27..ed592774152 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -25,7 +25,11 @@ from synapse.storage.background_updates import UpdaterStatus from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S -from tests.server import get_clock, setup_test_homeserver +from tests.server import ( + cleanup_test_reactor_system_event_triggers, + get_clock, + setup_test_homeserver, +) from tests.unittest import HomeserverTestCase @@ -56,6 +60,9 @@ def test_clean_homeserver_shutdown(self) -> None: # Run the reactor so any `callWhenRunning` functions can be cleared out. self.reactor.run() + # Cleanup the registered system event triggers since the `MemoryReactor` doesn't + # do this on it's own. + cleanup_test_reactor_system_event_triggers(self.reactor) # Cleanup the homeserver. self.get_success(self.hs.shutdown()) @@ -132,6 +139,10 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # Run the reactor so any `callWhenRunning` functions can be cleared out. self.reactor.run() + # Cleanup the registered system event triggers since the `MemoryReactor` doesn't + # do this on it's own. + cleanup_test_reactor_system_event_triggers(self.reactor) + # Also advance the reactor by the delay tracking threshold to ensure all # cancellable delayed calls have been scheduled. Must be done prior to # `hs.shutdown()` otherwise they will be scheduled later during the test when we diff --git a/tests/server.py b/tests/server.py index 32f39e389e2..dbd476c824e 100644 --- a/tests/server.py +++ b/tests/server.py @@ -526,46 +526,6 @@ def getHostByName( # overwrite it again. self.nameResolver = SimpleResolverComplexifier(FakeResolver()) - # type-ignore: `MemoryReactor` returns `None` whereas the - # `twisted.internet.interfaces.IReactorCore` interface allows a return type - # of `Any`, so long as the ID type matches that used by `removeSystemEventTrigger`. - def addSystemEventTrigger( # type: ignore[override] - self, - phase: str, - eventType: str, - callable: Callable[P, object], - *args: P.args, - **kw: P.kwargs, - ) -> int: - """ - Override the call from `MemoryReactorClock` in order to track registered event - triggers by `triggerID`. - Otherwise the code here has been copied from `MemoryReactorClock`. - """ - phaseTriggers = self.triggers.setdefault(phase, {}) - eventTypeTriggers = phaseTriggers.setdefault(eventType, []) - trigger = (callable, args, kw) - eventTypeTriggers.append(trigger) - return id(trigger) - - def removeSystemEventTrigger(self, triggerID: int) -> None: - """ - Override the unimplemented call from `MemoryReactorClock` in order to actually remove - registered event triggers. - This is necessary for a clean shutdown to occur as these triggers can hold - references to the `SynapseHomeServer`. - - args: - triggerID: Unique ID obtained from `addSystemEventTrigger`. - """ - for phase_triggers in self.triggers.values(): - for event_type_triggers in phase_triggers.values(): - triggers_to_remove = [ - item for item in event_type_triggers if id(item) == triggerID - ] - for trigger in triggers_to_remove: - event_type_triggers.remove(trigger) - def run(self) -> None: """ Override the call from `MemoryReactorClock` to add an additional step that @@ -704,6 +664,19 @@ def advance(self, amount: float) -> None: super().advance(0) +def cleanup_test_reactor_system_event_triggers( + reactor: ThreadedMemoryReactorClock, +) -> None: + """Cleanup any registered system event triggers. + The `twisted.internet.test.ThreadedMemoryReactor` does not implement + `removeSystemEventTrigger` so won't clean these triggers up on it's own properly. + When trying to override `removeSystemEventTrigger` in `ThreadedMemoryReactorClock` + in order to implement this functionality, twisted complains about the reactor being + unclean and fails some tests. + """ + reactor.triggers.clear() + + def validate_connector(connector: tcp.Connector, expected_ip: str) -> None: """Try to validate the obtained connector as it would happen when synapse is running and the conection will be established. From 527bd48b8fddcd6fc2a2215752cf06d512548c74 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 26 Sep 2025 16:09:56 -0600 Subject: [PATCH 159/181] Fix postgres shutdown tests --- synapse/server.py | 3 +++ tests/server.py | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/synapse/server.py b/synapse/server.py index c2ec01eccf4..28028c5e8d2 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -504,6 +504,9 @@ async def shutdown(self) -> None: logger.error("Error calling shutdown sync handler: %s", e) self._sync_shutdown_handlers.clear() + for db in self.get_datastores().databases: + db._db_pool.close() + def register_async_shutdown_handler( self, *, diff --git a/tests/server.py b/tests/server.py index dbd476c824e..6ab0cbf7554 100644 --- a/tests/server.py +++ b/tests/server.py @@ -1232,14 +1232,18 @@ def shutdown_hs_on_cleanup() -> "Deferred[None]": hs.get_datastores().main.USE_DEDICATED_DB_THREADS_FOR_EVENT_FETCHING = False if USE_POSTGRES_FOR_TESTS: - database_pool = hs.get_datastores().databases[0] + # Capture the `database_pool` as a `weakref` here to ensure there is no scenario where uncalled + # cleanup functions result in holding the `hs` in memory. + database_pool = weakref.ref(hs.get_datastores().databases[0]) # We need to do cleanup on PostgreSQL def cleanup() -> None: import psycopg2 # Close all the db pools - database_pool._db_pool.close() + db_pool = database_pool() + if db_pool is not None: + db_pool._db_pool.close() dropped = False From 28fdf12ecfb72d7590e2dbee8b843fc996d6f300 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 26 Sep 2025 16:42:46 -0600 Subject: [PATCH 160/181] Add lint for new Clock creation --- synapse/federation/federation_server.py | 2 +- synapse/handlers/appservice.py | 1 + synapse/handlers/device.py | 2 ++ synapse/handlers/e2e_keys.py | 5 ++- synapse/handlers/federation.py | 3 +- synapse/handlers/federation_event.py | 2 +- synapse/handlers/message.py | 1 + synapse/handlers/presence.py | 1 + synapse/handlers/read_marker.py | 2 +- synapse/handlers/room_member.py | 2 ++ synapse/handlers/sso.py | 2 +- .../federation/matrix_federation_agent.py | 7 ++++ .../http/federation/well_known_resolver.py | 4 ++- synapse/http/matrixfederationclient.py | 7 ++-- synapse/http/server.py | 22 +++++++++++-- synapse/media/media_repository.py | 1 + synapse/replication/tcp/client.py | 1 + synapse/rest/client/push_rule.py | 2 +- synapse/server.py | 3 +- synapse/state/__init__.py | 1 + synapse/storage/controllers/state.py | 1 + synapse/util/async_helpers.py | 8 +---- tests/handlers/test_typing.py | 5 +-- .../test_matrix_federation_agent.py | 8 +++-- tests/logging/test_opentracing.py | 8 +++-- .../test_federation_sender_shard.py | 3 +- tests/rest/client/test_transactions.py | 4 ++- tests/server.py | 4 ++- tests/util/test_linearizer.py | 24 +++++++------- tests/util/test_logcontext.py | 33 ++++++++++++------- 30 files changed, 118 insertions(+), 51 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index a8d5c3c280a..27a3a664bd1 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -159,7 +159,7 @@ def __init__(self, hs: "HomeServer"): # with FederationHandlerRegistry. hs.get_directory_handler() - self._server_linearizer = Linearizer("fed_server") + self._server_linearizer = Linearizer(hs.get_clock(), "fed_server") # origins that we are currently processing a transaction from. # a dict from origin to txn id. diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 70968caaa2b..00fe763e708 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -98,6 +98,7 @@ def __init__(self, hs: "HomeServer"): self.is_processing = False self._ephemeral_events_linearizer = Linearizer( + hs.get_clock(), name="appservice_ephemeral_events", ) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 5715d79ba2d..a7695c8b464 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1451,9 +1451,11 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): self.device_handler = device_handler self._remote_edu_linearizer = Linearizer( + hs.get_clock(), name="remote_device_list", ) self._resync_linearizer = Linearizer( + hs.get_clock(), name="remote_device_resync", ) diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index fa3d207a905..37c86d323b4 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -112,6 +112,7 @@ def __init__(self, hs: "HomeServer"): # Limit the number of in-flight requests from a single device. self._query_devices_linearizer = Linearizer( + hs.get_clock(), name="query_devices", max_count=10, ) @@ -1765,7 +1766,9 @@ def __init__(self, hs: "HomeServer"): assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler - self._remote_edu_linearizer = Linearizer(name="remote_signing_key") + self._remote_edu_linearizer = Linearizer( + hs.get_clock(), name="remote_signing_key" + ) # user_id -> list of updates waiting to be handled. self._pending_updates: Dict[str, List[Tuple[JsonDict, JsonDict]]] = {} diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 3ba5a821323..9185d436c8f 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -160,7 +160,7 @@ def __init__(self, hs: "HomeServer"): self._notifier = hs.get_notifier() self._worker_locks = hs.get_worker_locks_handler() - self._room_backfill = Linearizer("room_backfill") + self._room_backfill = Linearizer(hs.get_clock(), "room_backfill") self._third_party_event_rules = ( hs.get_module_api_callbacks().third_party_event_rules @@ -180,6 +180,7 @@ def __init__(self, hs: "HomeServer"): # When the lock is held for a given room, no other concurrent code may # partial state or un-partial state the room. self._is_partial_state_room_linearizer = Linearizer( + hs.get_clock(), name="_is_partial_state_room_linearizer", ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 63e17489438..4cd8e7d04d5 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -192,7 +192,7 @@ def __init__(self, hs: "HomeServer"): # federation event staging area. self.room_queues: Dict[str, List[Tuple[EventBase, str]]] = {} - self._room_pdu_linearizer = Linearizer("fed_room_pdu") + self._room_pdu_linearizer = Linearizer(hs.get_clock(), "fed_room_pdu") async def on_receive_pdu(self, origin: str, pdu: EventBase) -> None: """Process a PDU received via a federation /send/ transaction diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 843beb91c0d..ed0c686ee05 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -519,6 +519,7 @@ def __init__(self, hs: "HomeServer"): # We limit concurrent event creation for a room to 1. This prevents state resolution # from occurring when sending bursts of events to a local room self.limiter = Linearizer( + hs.get_clock(), max_count=1, name="room_event_creation_limit", ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 1fc5497c591..b7ac701488e 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -868,6 +868,7 @@ def __init__(self, hs: "HomeServer"): self.external_process_last_updated_ms: Dict[str, int] = {} self.external_sync_linearizer = Linearizer( + hs.get_clock(), name="external_sync_linearizer", ) diff --git a/synapse/handlers/read_marker.py b/synapse/handlers/read_marker.py index fb39c8e04b5..6a311055dc2 100644 --- a/synapse/handlers/read_marker.py +++ b/synapse/handlers/read_marker.py @@ -36,7 +36,7 @@ class ReadMarkerHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self.account_data_handler = hs.get_account_data_handler() - self.read_marker_linearizer = Linearizer(name="read_marker") + self.read_marker_linearizer = Linearizer(hs.get_clock(), name="read_marker") async def received_client_read_marker( self, room_id: str, user_id: str, event_id: str diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index ca39d432e19..39f8963ec5e 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -115,9 +115,11 @@ def __init__(self, hs: "HomeServer"): self._membership_types_to_include_profile_data_in.add(Membership.INVITE) self.member_linearizer: Linearizer = Linearizer( + hs.get_clock(), name="member", ) self.member_as_limiter = Linearizer( + hs.get_clock(), max_count=10, name="member_as_limiter", ) diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 05a9da69949..27fe035ebf0 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -224,7 +224,7 @@ def __init__(self, hs: "HomeServer"): ) # a lock on the mappings - self._mapping_lock = Linearizer(name="sso_user_mapping") + self._mapping_lock = Linearizer(hs.get_clock(), name="sso_user_mapping") # a map from session id to session data self._username_mapping_sessions: Dict[str, UsernameMappingSession] = {} diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 8353a8b1adb..9d87514be00 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -49,6 +49,7 @@ from synapse.http.proxyagent import ProxyAgent from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.types import ISynapseReactor +from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -66,6 +67,9 @@ class MatrixFederationAgent: Args: reactor: twisted reactor to use for underlying requests + clock: Internal `HomeServer` clock used to track delayed and looping calls. + Should be obtained from `hs.get_clock()`. + tls_client_options_factory: factory to use for fetching client tls options, or none to disable TLS. @@ -96,6 +100,7 @@ def __init__( *, server_name: str, reactor: ISynapseReactor, + clock: Clock, tls_client_options_factory: Optional[FederationPolicyForHTTPS], user_agent: bytes, ip_allowlist: Optional[IPSet], @@ -108,6 +113,7 @@ def __init__( Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). reactor + clock: Should be the `hs` clock from `hs.get_clock()` tls_client_options_factory user_agent ip_allowlist @@ -145,6 +151,7 @@ def __init__( _well_known_resolver = WellKnownResolver( server_name=server_name, reactor=reactor, + clock=clock, agent=BlocklistingAgentWrapper( ProxyAgent( reactor=reactor, diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 49e58c82288..2f52abcc035 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -90,6 +90,7 @@ def __init__( self, server_name: str, reactor: ISynapseThreadlessReactor, + clock: Clock, agent: IAgent, user_agent: bytes, well_known_cache: Optional[TTLCache[bytes, Optional[bytes]]] = None, @@ -99,6 +100,7 @@ def __init__( Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). reactor + clock: Should be the `hs` clock from `hs.get_clock()` agent user_agent well_known_cache @@ -107,7 +109,7 @@ def __init__( self.server_name = server_name self._reactor = reactor - self._clock = Clock(reactor) + self._clock = clock if well_known_cache is None: well_known_cache = TTLCache( diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 236ead4141b..2f5cf63dae9 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -418,6 +418,7 @@ def __init__( self.server_name = hs.hostname self.reactor = hs.get_reactor() + self.clock = hs.get_clock() user_agent = hs.version_string if hs.config.server.user_agent_suffix: @@ -431,6 +432,7 @@ def __init__( federation_agent: IAgent = MatrixFederationAgent( server_name=self.server_name, reactor=self.reactor, + clock=self.clock, tls_client_options_factory=tls_client_options_factory, user_agent=user_agent.encode("ascii"), ip_allowlist=hs.config.server.federation_ip_range_allowlist, @@ -464,7 +466,6 @@ def __init__( ip_blocklist=hs.config.server.federation_ip_range_blocklist, ) - self.clock = hs.get_clock() self._store = hs.get_datastores().main self.version_string_bytes = hs.version_string.encode("ascii") self.default_timeout_seconds = hs.config.federation.client_timeout_ms / 1000 @@ -488,7 +489,9 @@ def __init__( use_proxy=True, ) - self.remote_download_linearizer = Linearizer("remote_download_linearizer", 6) + self.remote_download_linearizer = Linearizer( + hs.get_clock(), "remote_download_linearizer", 6 + ) self._is_shutdown = False def shutdown(self) -> None: diff --git a/synapse/http/server.py b/synapse/http/server.py index 0b987f6aa9a..a91963ace1b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -412,7 +412,16 @@ def __init__( clock: Optional[Clock] = None, ): if clock is None: - clock = Clock(cast(ISynapseThreadlessReactor, reactor)) + # Ideally we wouldn't ignore the linter error here and instead enforce a + # required `Clock` be passed into the `__init__` function. + # However, this would change the function signature which is currently being + # exported to the module api. Since we don't want to break that api, we have + # to settle with ignoring the linter error here. + # As of the time of writing this, all Synapse internal usages of + # `DirectServeJsonResource` pass in the existing homeserver clock instance. + clock = Clock( # type: ignore[multiple-internal-clocks] + cast(ISynapseThreadlessReactor, reactor) + ) super().__init__(clock, extract_context) self.canonical_json = canonical_json @@ -591,7 +600,16 @@ def __init__( clock: Optional[Clock] = None, ): if clock is None: - clock = Clock(cast(ISynapseThreadlessReactor, reactor)) + # Ideally we wouldn't ignore the linter error here and instead enforce a + # required `Clock` be passed into the `__init__` function. + # However, this would change the function signature which is currently being + # exported to the module api. Since we don't want to break that api, we have + # to settle with ignoring the linter error here. + # As of the time of writing this, all Synapse internal usages of + # `DirectServeHtmlResource` pass in the existing homeserver clock instance. + clock = Clock( # type: ignore[multiple-internal-clocks] + cast(ISynapseThreadlessReactor, reactor) + ) super().__init__(clock, extract_context) diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 43c1e4856a7..17800dfdd97 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -109,6 +109,7 @@ def __init__(self, hs: "HomeServer"): self.thumbnail_requirements = hs.config.media.thumbnail_requirements self.remote_media_linearizer = Linearizer( + hs.get_clock(), name="media_remote", ) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 2237983f39b..60dd8cda6fe 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -440,6 +440,7 @@ def __init__(self, hs: "HomeServer"): self.federation_position: Optional[int] = None self._fed_position_linearizer = Linearizer( + hs.get_clock(), name="_fed_position_linearizer", ) diff --git a/synapse/rest/client/push_rule.py b/synapse/rest/client/push_rule.py index c20de89bf71..093f376c275 100644 --- a/synapse/rest/client/push_rule.py +++ b/synapse/rest/client/push_rule.py @@ -65,7 +65,7 @@ def __init__(self, hs: "HomeServer"): hs.get_instance_name() in hs.config.worker.writers.push_rules ) self._push_rules_handler = hs.get_push_rules_handler() - self._push_rule_linearizer = Linearizer(name="push_rules") + self._push_rule_linearizer = Linearizer(hs.get_clock(), name="push_rules") async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]: if not self._is_push_worker: diff --git a/synapse/server.py b/synapse/server.py index 28028c5e8d2..d3d4ebd9ca5 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -672,7 +672,8 @@ def is_mine_server_name(self, server_name: str) -> bool: @cache_in_self def get_clock(self) -> Clock: - return Clock(self._reactor) + # Ignore the linter error since this is the one place the `Clock` should be created. + return Clock(self._reactor) # type: ignore[multiple-internal-clocks] def get_datastores(self) -> Databases: if not self.datastores: diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 54fededcfbe..45993476216 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -643,6 +643,7 @@ def __init__(self, hs: "HomeServer"): self.clock = hs.get_clock() self.resolve_linearizer = Linearizer( + hs.get_clock(), name="state_resolve_lock", ) diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 003a456515e..44fb15eae7f 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -78,6 +78,7 @@ def __init__(self, hs: "HomeServer", stores: "Databases"): # Used by `_get_joined_hosts` to ensure only one thing mutates the cache # at a time. Keyed by room_id. self._joined_host_linearizer = Linearizer( + hs.get_clock(), "_JoinedHostsCache", ) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 8d4825ee9c7..144df2544ea 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -47,7 +47,6 @@ Tuple, TypeVar, Union, - cast, overload, ) @@ -64,7 +63,6 @@ run_coroutine_in_background, run_in_background, ) -from synapse.types import ISynapseThreadlessReactor from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock logger = logging.getLogger(__name__) @@ -550,9 +548,9 @@ class Linearizer: def __init__( self, + clock: Clock, name: Optional[str] = None, max_count: int = 1, - clock: Optional[Clock] = None, ): """ Args: @@ -563,10 +561,6 @@ def __init__( else: self.name = name - if not clock: - from twisted.internet import reactor - - clock = Clock(cast(ISynapseThreadlessReactor, reactor)) self._clock = clock self.max_count = max_count diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 073256190a7..90c185bc3d4 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -88,7 +88,8 @@ def make_homeserver( self.mock_federation_client.put_json.return_value = (200, "OK") self.mock_federation_client.agent = MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", - reactor=reactor, + reactor=self.reactor, + clock=self.clock, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, @@ -97,7 +98,7 @@ def make_homeserver( ) # the tests assume that we are starting at unix time 1000 - reactor.pump((1000,)) + self.reactor.pump((1000,)) self.mock_hs_notifier = Mock() hs = self.setup_test_homeserver( diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 12428e64a96..0db260cfe66 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -65,7 +65,7 @@ from tests import unittest from tests.http import dummy_address, get_test_ca_cert_file, wrap_server_factory_for_tls -from tests.server import FakeTransport, ThreadedMemoryReactorClock +from tests.server import FakeTransport, get_clock from tests.utils import checked_cast, default_config logger = logging.getLogger(__name__) @@ -73,7 +73,7 @@ class MatrixFederationAgentTests(unittest.TestCase): def setUp(self) -> None: - self.reactor = ThreadedMemoryReactorClock() + self.reactor, self.clock = get_clock() self.mock_resolver = AsyncMock(spec=SrvResolver) @@ -98,6 +98,7 @@ def setUp(self) -> None: self.well_known_resolver = WellKnownResolver( server_name="OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, + clock=self.clock, agent=Agent(self.reactor, contextFactory=self.tls_factory), user_agent=b"test-agent", well_known_cache=self.well_known_cache, @@ -277,6 +278,7 @@ def _make_agent(self) -> MatrixFederationAgent: return MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), + clock=self.clock, tls_client_options_factory=self.tls_factory, user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. ip_allowlist=IPSet(), @@ -1021,6 +1023,7 @@ def test_get_well_known_unsigned_cert(self) -> None: agent = MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, + clock=self.clock, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. ip_allowlist=IPSet(), @@ -1030,6 +1033,7 @@ def test_get_well_known_unsigned_cert(self) -> None: _well_known_resolver=WellKnownResolver( server_name="OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), + clock=self.clock, agent=Agent(self.reactor, contextFactory=tls_factory), user_agent=b"test-agent", well_known_cache=self.well_known_cache, diff --git a/tests/logging/test_opentracing.py b/tests/logging/test_opentracing.py index d102d501386..c891a52a5ea 100644 --- a/tests/logging/test_opentracing.py +++ b/tests/logging/test_opentracing.py @@ -163,7 +163,9 @@ def test_overlapping_spans(self) -> None: # implements `ISynapseThreadlessReactor` (combination of the normal Twisted # Reactor/Clock interfaces), via inheritance from # `twisted.internet.testing.MemoryReactor` and `twisted.internet.testing.Clock` - clock = Clock( + # Ignore `multiple-internal-clocks` linter error here since we are creating a `Clock` + # for testing purposes. + clock = Clock( # type: ignore[multiple-internal-clocks] reactor # type: ignore[arg-type] ) @@ -233,7 +235,9 @@ def test_run_in_background_active_scope_still_available(self) -> None: # implements `ISynapseThreadlessReactor` (combination of the normal Twisted # Reactor/Clock interfaces), via inheritance from # `twisted.internet.testing.MemoryReactor` and `twisted.internet.testing.Clock` - clock = Clock( + # Ignore `multiple-internal-clocks` linter error here since we are creating a `Clock` + # for testing purposes. + clock = Clock( # type: ignore[multiple-internal-clocks] reactor # type: ignore[arg-type] ) diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 92259f2542a..3896e0ce8a0 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -66,10 +66,11 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): def setUp(self) -> None: super().setUp() - reactor, _ = get_clock() + reactor, clock = get_clock() self.matrix_federation_agent = MatrixFederationAgent( server_name="OUR_STUB_HOMESERVER_NAME", reactor=reactor, + clock=clock, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, diff --git a/tests/rest/client/test_transactions.py b/tests/rest/client/test_transactions.py index 967f7836800..7eb7a91532c 100644 --- a/tests/rest/client/test_transactions.py +++ b/tests/rest/client/test_transactions.py @@ -90,7 +90,9 @@ def test_logcontexts_with_async_result( ) -> Generator["defer.Deferred[Any]", object, None]: @defer.inlineCallbacks def cb() -> Generator["defer.Deferred[object]", object, Tuple[int, JsonDict]]: - yield defer.ensureDeferred(Clock(reactor).sleep(0)) + # Ignore `multiple-internal-clocks` linter error here since we are creating a `Clock` + # for testing purposes. + yield defer.ensureDeferred(Clock(reactor).sleep(0)) # type: ignore[multiple-internal-clocks] return 1, {} @defer.inlineCallbacks diff --git a/tests/server.py b/tests/server.py index 6ab0cbf7554..b107031d0ba 100644 --- a/tests/server.py +++ b/tests/server.py @@ -817,7 +817,9 @@ def _(res: Any) -> None: def get_clock() -> Tuple[ThreadedMemoryReactorClock, Clock]: clock = ThreadedMemoryReactorClock() - hs_clock = Clock(clock) + # Ignore the linter error since this is an expected usage of creating a `Clock` for + # testing purposes. + hs_clock = Clock(clock) # type: ignore[multiple-internal-clocks] return clock, hs_clock diff --git a/tests/util/test_linearizer.py b/tests/util/test_linearizer.py index 7510657b858..26699edcf83 100644 --- a/tests/util/test_linearizer.py +++ b/tests/util/test_linearizer.py @@ -21,14 +21,14 @@ from typing import Hashable, Protocol, Tuple -from twisted.internet import defer, reactor -from twisted.internet.base import ReactorBase +from twisted.internet import defer from twisted.internet.defer import CancelledError, Deferred from synapse.logging.context import LoggingContext, current_context from synapse.util.async_helpers import Linearizer from tests import unittest +from tests.server import get_clock class UnblockFunction(Protocol): @@ -36,6 +36,9 @@ def __call__(self, pump_reactor: bool = True) -> None: ... class LinearizerTestCase(unittest.TestCase): + def setUp(self) -> None: + self.reactor, self.clock = get_clock() + def _start_task( self, linearizer: Linearizer, key: Hashable ) -> Tuple["Deferred[None]", "Deferred[None]", UnblockFunction]: @@ -73,13 +76,12 @@ def unblock(pump_reactor: bool = True) -> None: def _pump(self) -> None: """Pump the reactor to advance `Linearizer`s.""" - assert isinstance(reactor, ReactorBase) - while reactor.getDelayedCalls(): - reactor.runUntilCurrent() + while self.reactor.getDelayedCalls(): + self.reactor.advance(0) def test_linearizer(self) -> None: """Tests that a task is queued up behind an earlier task.""" - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() @@ -100,7 +102,7 @@ def test_linearizer_is_queued(self) -> None: Runs through the same scenario as `test_linearizer`. """ - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() @@ -131,7 +133,7 @@ def test_lots_of_queued_things(self) -> None: The stack should *not* explode when the slow thing completes. """ - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = "" async def func(i: int) -> None: @@ -151,7 +153,7 @@ async def func(i: int) -> None: def test_multiple_entries(self) -> None: """Tests a `Linearizer` with a concurrency above 1.""" - limiter = Linearizer(max_count=3) + limiter = Linearizer(self.clock, max_count=3) key = object() @@ -192,7 +194,7 @@ def test_multiple_entries(self) -> None: def test_cancellation(self) -> None: """Tests cancellation while waiting for a `Linearizer`.""" - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() @@ -226,7 +228,7 @@ def test_cancellation(self) -> None: def test_cancellation_during_sleep(self) -> None: """Tests cancellation during the sleep just after waiting for a `Linearizer`.""" - linearizer = Linearizer() + linearizer = Linearizer(self.clock) key = object() diff --git a/tests/util/test_logcontext.py b/tests/util/test_logcontext.py index 96700ad996f..7f009820453 100644 --- a/tests/util/test_logcontext.py +++ b/tests/util/test_logcontext.py @@ -66,7 +66,8 @@ async def test_sleep(self) -> None: """ Test `Clock.sleep` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -111,7 +112,8 @@ async def test_looping_call(self) -> None: """ Test `Clock.looping_call` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -161,7 +163,8 @@ async def test_looping_call_now(self) -> None: """ Test `Clock.looping_call_now` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -209,7 +212,8 @@ async def test_call_later(self) -> None: """ Test `Clock.call_later` """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -261,7 +265,8 @@ async def test_deferred_callback_await_in_current_logcontext(self) -> None: `d.callback(None)` without anything else. See the *Deferred callbacks* section of docs/log_contexts.md for more details. """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -318,7 +323,8 @@ async def test_deferred_callback_preserve_logging_context(self) -> None: `d.callback(None)` without anything else. See the *Deferred callbacks* section of docs/log_contexts.md for more details. """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -379,7 +385,8 @@ async def test_deferred_callback_fire_and_forget_with_current_context(self) -> N `d.callback(None)` without anything else. See the *Deferred callbacks* section of docs/log_contexts.md for more details. """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -448,7 +455,8 @@ async def competing_callback() -> None: self._check_test_key("sentinel") async def _test_run_in_background(self, function: Callable[[], object]) -> None: - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") @@ -490,7 +498,8 @@ def callback(result: object) -> object: @logcontext_clean async def test_run_in_background_with_blocking_fn(self) -> None: async def blocking_function() -> None: - await Clock(reactor).sleep(0) + # Ignore linter error since we are creating a `Clock` for testing purposes. + await Clock(reactor).sleep(0) # type: ignore[multiple-internal-clocks] await self._test_run_in_background(blocking_function) @@ -523,7 +532,8 @@ async def test_run_in_background_with_coroutine(self) -> None: async def testfunc() -> None: self._check_test_key("foo") - d = defer.ensureDeferred(Clock(reactor).sleep(0)) + # Ignore linter error since we are creating a `Clock` for testing purposes. + d = defer.ensureDeferred(Clock(reactor).sleep(0)) # type: ignore[multiple-internal-clocks] self.assertIs(current_context(), SENTINEL_CONTEXT) await d self._check_test_key("foo") @@ -552,7 +562,8 @@ async def test_run_coroutine_in_background(self) -> None: This will stress the logic around incomplete deferreds in `run_coroutine_in_background`. """ - clock = Clock(reactor) + # Ignore linter error since we are creating a `Clock` for testing purposes. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] # Sanity check that we start in the sentinel context self._check_test_key("sentinel") From 50a3cd1fb6a78c4c4eb4839862904bdfd6c97115 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Fri, 26 Sep 2025 17:39:48 -0600 Subject: [PATCH 161/181] Lint new Clock creation --- scripts-dev/mypy_synapse_plugin.py | 28 +++++++++++ synapse/appservice/__init__.py | 29 +++++++++++- synapse/handlers/profile.py | 2 +- synapse/handlers/sync.py | 1 + synapse/rest/client/sync.py | 1 + .../server_notices/server_notices_manager.py | 1 + synapse/storage/_base.py | 2 +- synapse/storage/controllers/state.py | 6 +-- synapse/storage/databases/main/cache.py | 4 +- .../storage/databases/main/censor_events.py | 2 +- synapse/storage/databases/main/client_ips.py | 9 ++-- synapse/storage/databases/main/deviceinbox.py | 12 ++--- synapse/storage/databases/main/devices.py | 8 ++-- .../storage/databases/main/end_to_end_keys.py | 4 +- .../databases/main/event_federation.py | 13 ++--- .../databases/main/event_push_actions.py | 18 +++---- .../databases/main/events_bg_updates.py | 2 +- .../storage/databases/main/events_worker.py | 9 ++-- synapse/storage/databases/main/lock.py | 12 ++--- .../databases/main/media_repository.py | 4 +- synapse/storage/databases/main/metrics.py | 12 ++--- .../databases/main/monthly_active_users.py | 7 ++- synapse/storage/databases/main/receipts.py | 2 +- .../storage/databases/main/registration.py | 37 +++++++-------- synapse/storage/databases/main/roommember.py | 2 +- synapse/storage/databases/main/session.py | 6 +-- .../storage/databases/main/sliding_sync.py | 4 +- .../storage/databases/main/transactions.py | 6 +-- synapse/storage/databases/state/store.py | 2 + synapse/util/caches/deferred_cache.py | 4 ++ synapse/util/caches/descriptors.py | 16 ++++++- synapse/util/caches/dictionary_cache.py | 7 ++- synapse/util/caches/lrucache.py | 20 +++----- synmark/suites/logging.py | 4 +- synmark/suites/lrucache.py | 3 +- synmark/suites/lrucache_evict.py | 3 +- tests/config/test_cache.py | 17 ++++--- tests/metrics/test_metrics.py | 9 ++-- .../test_module_cache_invalidation.py | 2 + tests/util/caches/test_deferred_cache.py | 25 ++++++---- tests/util/caches/test_descriptors.py | 30 ++++++++++++ tests/util/test_dict_cache.py | 4 +- tests/util/test_lrucache.py | 47 ++++++++++++------- 43 files changed, 285 insertions(+), 151 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index d7ff815f8fe..32d65776087 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -92,6 +92,12 @@ category="synapse-reactor-clock", ) +MULTIPLE_INTERNAL_CLOCKS_CREATED = ErrorCode( + "multiple-internal-clocks", + "Only one instance of `clock.Clock` should be created", + category="synapse-reactor-clock", +) + class Sentinel(enum.Enum): # defining a sentinel in this way allows mypy to correctly handle the @@ -237,6 +243,9 @@ def get_function_signature_hook( if fullname == "twisted.internet.task.LoopingCall": return check_looping_call + if fullname == "synapse.util.clock.Clock": + return check_clock_creation + return None def get_method_signature_hook( @@ -331,6 +340,25 @@ def check_looping_call(ctx: FunctionSigContext) -> CallableType: return signature +def check_clock_creation(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that the only `clock.Clock` is the one used by the `HomeServer`. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected the only `clock.Clock` instance to be the one used by the `HomeServer`. " + "This is so that the `HomeServer` can cancel any tracked delayed or looping calls " + "during server shutdown", + ctx.context, + code=MULTIPLE_INTERNAL_CLOCKS_CREATED, + ) + + return signature + + def check_call_when_running(ctx: MethodSigContext) -> CallableType: """ Ensure that the `reactor.callWhenRunning` callsites aren't used. diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 2d8d382e68c..87cfec5982a 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -23,15 +23,33 @@ import logging import re from enum import Enum -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Pattern, Sequence +from typing import ( + TYPE_CHECKING, + Dict, + Iterable, + List, + Optional, + Pattern, + Sequence, + cast, +) import attr from netaddr import IPSet +from twisted.internet import reactor + from synapse.api.constants import EventTypes from synapse.events import EventBase -from synapse.types import DeviceListUpdates, JsonDict, JsonMapping, UserID +from synapse.types import ( + DeviceListUpdates, + ISynapseThreadlessReactor, + JsonDict, + JsonMapping, + UserID, +) from synapse.util.caches.descriptors import _CacheContext, cached +from synapse.util.clock import Clock if TYPE_CHECKING: from synapse.appservice.api import ApplicationServiceApi @@ -98,6 +116,13 @@ def __init__( self.sender = sender # The application service user should be part of the server's domain. self.server_name = sender.domain # nb must be called this for @cached + + # Ideally we would require passing in the `HomeServer` `Clock` instance. + # However this is not currently possible as there are places which use + # `@cached` that aren't aware of the `HomeServer` instance. + # nb must be called this for @cached + self.clock = Clock(cast(ISynapseThreadlessReactor, reactor)) # type: ignore[multiple-internal-clocks] + self.namespaces = self._check_namespaces(namespaces) self.id = id self.ip_range_whitelist = ip_range_whitelist diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index dbff28e7fb5..9dda89d85bb 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -56,8 +56,8 @@ class ProfileHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached + self.clock = hs.get_clock() # nb must be called this for @cached self.store = hs.get_datastores().main - self.clock = hs.get_clock() self.hs = hs self.federation = hs.get_federation_client() diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 0b602384387..1c7ce0f019e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -982,6 +982,7 @@ def get_lazy_loaded_members_cache( logger.debug("creating LruCache for %r", cache_key) cache = LruCache( max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE, + clock=self.clock, ) self.lazy_loaded_members_cache[cache_key] = cache else: diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index bb63b51599b..0f3cc84dcc9 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -126,6 +126,7 @@ def __init__(self, hs: "HomeServer"): self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, + clock=self.clock, cache_name="sync_valid_filter", server_name=self.server_name, ) diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 19f86b5a563..73cf4091eb4 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -36,6 +36,7 @@ class ServerNoticesManager: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @cached + self.clock = hs.get_clock() # nb must be called this for @cached self._store = hs.get_datastores().main self._config = hs.config self._account_data_handler = hs.get_account_data_handler() diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index f214f558978..1fddcc0799a 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -56,7 +56,7 @@ def __init__( ): self.hs = hs self.server_name = hs.hostname # nb must be called this for @cached - self._clock = hs.get_clock() + self.clock = hs.get_clock() # nb must be called this for @cached self.database_engine = database.engine self.db_pool = database diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index 44fb15eae7f..2660f1a8ec1 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -69,8 +69,8 @@ class StateStorageController: def __init__(self, hs: "HomeServer", stores: "Databases"): self.server_name = hs.hostname # nb must be called this for @cached + self.clock = hs.get_clock() self._is_mine_id = hs.is_mine_id - self._clock = hs.get_clock() self.stores = stores self._partial_state_events_tracker = PartialStateEventsTracker(stores.main) self._partial_state_room_tracker = PartialCurrentStateTracker(stores.main) @@ -818,9 +818,7 @@ async def get_joined_hosts( state_group = object() assert state_group is not None - with Measure( - self._clock, name="get_joined_hosts", server_name=self.server_name - ): + with Measure(self.clock, name="get_joined_hosts", server_name=self.server_name): return await self._get_joined_hosts( room_id, state_group, state_entry=state_entry ) diff --git a/synapse/storage/databases/main/cache.py b/synapse/storage/databases/main/cache.py index ed735d8066d..674c6b921ee 100644 --- a/synapse/storage/databases/main/cache.py +++ b/synapse/storage/databases/main/cache.py @@ -751,7 +751,7 @@ def _send_invalidation_to_replication( "instance_name": self._instance_name, "cache_func": cache_name, "keys": keys, - "invalidation_ts": self._clock.time_msec(), + "invalidation_ts": self.clock.time_msec(), }, ) @@ -778,7 +778,7 @@ def _send_invalidation_to_replication_bulk( assert self._cache_id_gen is not None stream_ids = self._cache_id_gen.get_next_mult_txn(txn, len(key_tuples)) - ts = self._clock.time_msec() + ts = self.clock.time_msec() txn.call_after(self.hs.get_notifier().on_new_replication_data) self.db_pool.simple_insert_many_txn( txn, diff --git a/synapse/storage/databases/main/censor_events.py b/synapse/storage/databases/main/censor_events.py index 3f9f482adda..45cfe97dba2 100644 --- a/synapse/storage/databases/main/censor_events.py +++ b/synapse/storage/databases/main/censor_events.py @@ -77,7 +77,7 @@ async def _censor_redactions(self) -> None: return before_ts = ( - self._clock.time_msec() - self.hs.config.server.redaction_retention_period + self.clock.time_msec() - self.hs.config.server.redaction_retention_period ) # We fetch all redactions that: diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 86328af62f6..c9522cef626 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -438,10 +438,11 @@ def __init__( cache_name="client_ip_last_seen", server_name=self.server_name, max_size=50000, + clock=hs.get_clock(), ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: - self._clock.looping_call(self._prune_old_user_ips, 5 * 1000) + self.clock.looping_call(self._prune_old_user_ips, 5 * 1000) if self._update_on_this_worker: # This is the designated worker that can write to the client IP @@ -452,7 +453,7 @@ def __init__( Tuple[str, str, str], Tuple[str, Optional[str], int] ] = {} - self._clock.looping_call(self._update_client_ips_batch, 5 * 1000) + self.clock.looping_call(self._update_client_ips_batch, 5 * 1000) hs.register_async_shutdown_handler( phase="before", eventType="shutdown", @@ -493,7 +494,7 @@ async def _prune_old_user_ips(self) -> None: ) """ - timestamp = self._clock.time_msec() - self.user_ips_max_age + timestamp = self.clock.time_msec() - self.user_ips_max_age def _prune_old_user_ips_txn(txn: LoggingTransaction) -> None: txn.execute(sql, (timestamp,)) @@ -629,7 +630,7 @@ async def insert_client_ip( return if not now: - now = int(self._clock.time_msec()) + now = int(self.clock.time_msec()) key = (user_id, access_token, ip) try: diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index f6f3c94a0d0..7857939c461 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -96,7 +96,7 @@ def __init__( ] = ExpiringCache( cache_name="last_device_delete_cache", server_name=self.server_name, - clock=self._clock, + clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, ) @@ -154,7 +154,7 @@ def __init__( ) if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( run_as_background_process, DEVICE_FEDERATION_INBOX_CLEANUP_INTERVAL_MS, "_delete_old_federation_inbox_rows", @@ -826,7 +826,7 @@ def add_messages_txn( ) async with self._to_device_msg_id_gen.get_next() as stream_id: - now_ms = self._clock.time_msec() + now_ms = self.clock.time_msec() await self.db_pool.runInteraction( "add_messages_to_device_inbox", add_messages_txn, now_ms, stream_id ) @@ -881,7 +881,7 @@ def add_messages_txn( ) async with self._to_device_msg_id_gen.get_next() as stream_id: - now_ms = self._clock.time_msec() + now_ms = self.clock.time_msec() await self.db_pool.runInteraction( "add_messages_from_remote_to_device_inbox", add_messages_txn, @@ -1002,7 +1002,7 @@ def _delete_old_federation_inbox_rows_txn(txn: LoggingTransaction) -> bool: # We delete at most 100 rows that are older than # DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS delete_before_ts = ( - self._clock.time_msec() - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS + self.clock.time_msec() - DEVICE_FEDERATION_INBOX_CLEANUP_DELAY_MS ) sql = """ WITH to_delete AS ( @@ -1032,7 +1032,7 @@ def _delete_old_federation_inbox_rows_txn(txn: LoggingTransaction) -> bool: # We sleep a bit so that we don't hammer the database in a tight # loop first time we run this. - await self._clock.sleep(1) + await self.clock.sleep(1) async def get_devices_with_messages( self, user_id: str, device_ids: StrCollection diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index fc1e1c73f18..d4b9ce0ea0a 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -195,7 +195,7 @@ def __init__( ) if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( self._prune_old_outbound_device_pokes, 60 * 60 * 1000 ) @@ -1390,7 +1390,7 @@ def _mark_remote_users_device_caches_as_stale_txn( table="device_lists_remote_resync", keyvalues={"user_id": user_id}, values={}, - insertion_values={"added_ts": self._clock.time_msec()}, + insertion_values={"added_ts": self.clock.time_msec()}, ) await self.db_pool.runInteraction( @@ -1601,7 +1601,7 @@ async def _prune_old_outbound_device_pokes( that user when the destination comes back. It doesn't matter which device we keep. """ - yesterday = self._clock.time_msec() - prune_age + yesterday = self.clock.time_msec() - prune_age def _prune_txn(txn: LoggingTransaction) -> None: # look for (user, destination) pairs which have an update older than @@ -2086,7 +2086,7 @@ def _add_device_outbound_poke_to_stream_txn( stream_id, ) - now = self._clock.time_msec() + now = self.clock.time_msec() encoded_context = json_encoder.encode(context) mark_sent = not self.hs.is_mine_id(user_id) diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 2e9f62075a8..2d3d0c0036e 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -1564,7 +1564,7 @@ def impl(txn: LoggingTransaction) -> Tuple[List[str], int]: DELETE FROM e2e_one_time_keys_json WHERE {clause} AND ts_added_ms < ? AND length(key_id) = 6 """ - args.append(self._clock.time_msec() - (7 * 24 * 3600 * 1000)) + args.append(self.clock.time_msec() - (7 * 24 * 3600 * 1000)) txn.execute(sql, args) return users, txn.rowcount @@ -1585,7 +1585,7 @@ async def allow_master_cross_signing_key_replacement_without_uia( None, if there is no such key. Otherwise, the timestamp before which replacement is allowed without UIA. """ - timestamp = self._clock.time_msec() + duration_ms + timestamp = self.clock.time_msec() + duration_ms def impl(txn: LoggingTransaction) -> Optional[int]: txn.execute( diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index 5c9bd2e848d..d77420ff475 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -167,6 +167,7 @@ def __init__( # Cache of event ID to list of auth event IDs and their depths. self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache( max_size=500000, + clock=self.hs.get_clock(), server_name=self.server_name, cache_name="_event_auth_cache", size_callback=len, @@ -176,7 +177,7 @@ def __init__( # index. self.tests_allow_no_chain_cover_index = True - self._clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000) + self.clock.looping_call(self._get_stats_for_federation_staging, 30 * 1000) if isinstance(self.database_engine, PostgresEngine): self.db_pool.updates.register_background_validate_constraint_and_delete_rows( @@ -1328,7 +1329,7 @@ def get_backfill_points_in_room_txn( ( room_id, current_depth, - self._clock.time_msec(), + self.clock.time_msec(), BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS, BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS, limit, @@ -1841,7 +1842,7 @@ def _record_event_failed_pull_attempt_upsert_txn( last_cause=EXCLUDED.last_cause; """ - txn.execute(sql, (room_id, event_id, 1, self._clock.time_msec(), cause)) + txn.execute(sql, (room_id, event_id, 1, self.clock.time_msec(), cause)) @trace async def get_event_ids_with_failed_pull_attempts( @@ -1905,7 +1906,7 @@ async def get_event_ids_to_not_pull_from_backoff( ), ) - current_time = self._clock.time_msec() + current_time = self.clock.time_msec() event_ids_with_backoff = {} for event_id, last_attempt_ts, num_attempts in event_failed_pull_attempts: @@ -2025,7 +2026,7 @@ async def insert_received_event_to_staging( values={}, insertion_values={ "room_id": event.room_id, - "received_ts": self._clock.time_msec(), + "received_ts": self.clock.time_msec(), "event_json": json_encoder.encode(event.get_dict()), "internal_metadata": json_encoder.encode( event.internal_metadata.get_dict() @@ -2299,7 +2300,7 @@ def _get_stats_for_federation_staging_txn( # If there is nothing in the staging area default it to 0. age = 0 if received_ts is not None: - age = self._clock.time_msec() - received_ts + age = self.clock.time_msec() - received_ts return count, age diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index e47d64d44af..ec26aedc6bc 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -267,7 +267,7 @@ def __init__( super().__init__(database, db_conn, hs) # Track when the process started. - self._started_ts = self._clock.time_msec() + self._started_ts = self.clock.time_msec() # These get correctly set by _find_stream_orderings_for_times_txn self.stream_ordering_month_ago: Optional[int] = None @@ -277,14 +277,14 @@ def __init__( self._find_stream_orderings_for_times_txn(cur) cur.close() - self._clock.looping_call(self._find_stream_orderings_for_times, 10 * 60 * 1000) + self.clock.looping_call(self._find_stream_orderings_for_times, 10 * 60 * 1000) self._rotate_count = 10000 self._doing_notif_rotation = False if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._rotate_notifs, 30 * 1000) + self.clock.looping_call(self._rotate_notifs, 30 * 1000) - self._clock.looping_call( + self.clock.looping_call( self._clear_old_push_actions_staging, 30 * 60 * 1000 ) @@ -1190,7 +1190,7 @@ def _gen_entry( is_highlight, # highlight column int(count_as_unread), # unread column thread_id, # thread_id column - self._clock.time_msec(), # inserted_ts column + self.clock.time_msec(), # inserted_ts column ) await self.db_pool.simple_insert_many( @@ -1241,14 +1241,14 @@ async def _find_stream_orderings_for_times(self) -> None: def _find_stream_orderings_for_times_txn(self, txn: LoggingTransaction) -> None: logger.info("Searching for stream ordering 1 month ago") self.stream_ordering_month_ago = self._find_first_stream_ordering_after_ts_txn( - txn, self._clock.time_msec() - 30 * 24 * 60 * 60 * 1000 + txn, self.clock.time_msec() - 30 * 24 * 60 * 60 * 1000 ) logger.info( "Found stream ordering 1 month ago: it's %d", self.stream_ordering_month_ago ) logger.info("Searching for stream ordering 1 day ago") self.stream_ordering_day_ago = self._find_first_stream_ordering_after_ts_txn( - txn, self._clock.time_msec() - 24 * 60 * 60 * 1000 + txn, self.clock.time_msec() - 24 * 60 * 60 * 1000 ) logger.info( "Found stream ordering 1 day ago: it's %d", self.stream_ordering_day_ago @@ -1787,7 +1787,7 @@ async def _clear_old_push_actions_staging(self) -> None: # We delete anything more than an hour old, on the assumption that we'll # never take more than an hour to persist an event. - delete_before_ts = self._clock.time_msec() - 60 * 60 * 1000 + delete_before_ts = self.clock.time_msec() - 60 * 60 * 1000 if self._started_ts > delete_before_ts: # We need to wait for at least an hour before we started deleting, @@ -1824,7 +1824,7 @@ def _clear_old_push_actions_staging_txn(txn: LoggingTransaction) -> bool: return # We sleep to ensure that we don't overwhelm the DB. - await self._clock.sleep(1.0) + await self.clock.sleep(1.0) async def get_push_actions_for_user( self, diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 0a0102ee64b..37dd8e48d5d 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -730,7 +730,7 @@ def _redactions_received_ts_txn(txn: LoggingTransaction) -> int: WHERE ? <= event_id AND event_id <= ? """ - txn.execute(sql, (self._clock.time_msec(), last_event_id, upper_event_id)) + txn.execute(sql, (self.clock.time_msec(), last_event_id, upper_event_id)) self.db_pool.updates._background_update_progress_txn( txn, "redactions_received_ts", {"last_event_id": upper_event_id} diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 5b62317c498..4f9a1a4f780 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -281,13 +281,14 @@ def __init__( if hs.config.worker.run_background_tasks: # We periodically clean out old transaction ID mappings - self._clock.looping_call( + self.clock.looping_call( self._cleanup_old_transaction_ids, 5 * 60 * 1000, ) self._get_event_cache: AsyncLruCache[Tuple[str], EventCacheEntry] = ( AsyncLruCache( + clock=hs.get_clock(), server_name=self.server_name, cache_name="*getEvent*", max_size=hs.config.caches.event_cache_size, @@ -1273,7 +1274,7 @@ def _fetch_event_list( were not part of this request. """ with Measure( - self._clock, name="_fetch_event_list", server_name=self.server_name + self.clock, name="_fetch_event_list", server_name=self.server_name ): try: events_to_fetch = { @@ -2275,7 +2276,7 @@ async def _cleanup_old_transaction_ids(self) -> None: """Cleans out transaction id mappings older than 24hrs.""" def _cleanup_old_transaction_ids_txn(txn: LoggingTransaction) -> None: - one_day_ago = self._clock.time_msec() - 24 * 60 * 60 * 1000 + one_day_ago = self.clock.time_msec() - 24 * 60 * 60 * 1000 sql = """ DELETE FROM event_txn_id_device_id WHERE inserted_ts < ? @@ -2630,7 +2631,7 @@ def mark_event_rejected_txn( keyvalues={"event_id": event_id}, values={ "reason": rejection_reason, - "last_check": self._clock.time_msec(), + "last_check": self.clock.time_msec(), }, ) self.db_pool.simple_update_txn( diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index ddf695ba85d..aa2de4fe06c 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -108,7 +108,7 @@ def __init__( self._acquiring_locks: Set[Tuple[str, str]] = set() - self._clock.looping_call( + self.clock.looping_call( self._reap_stale_read_write_locks, _LOCK_TIMEOUT_MS / 10.0 ) @@ -154,7 +154,7 @@ async def _try_acquire_lock( if lock and await lock.is_still_valid(): return None - now = self._clock.time_msec() + now = self.clock.time_msec() token = random_string(6) def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: @@ -203,7 +203,7 @@ def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: lock = Lock( self.server_name, self._reactor, - self._clock, + self.clock, self, read_write=False, lock_name=lock_name, @@ -252,7 +252,7 @@ def _try_acquire_read_write_lock_txn( # constraints. If it doesn't then we have acquired the lock, # otherwise we haven't. - now = self._clock.time_msec() + now = self.clock.time_msec() token = random_string(6) self.db_pool.simple_insert_txn( @@ -271,7 +271,7 @@ def _try_acquire_read_write_lock_txn( lock = Lock( self.server_name, self._reactor, - self._clock, + self.clock, self, read_write=True, lock_name=lock_name, @@ -339,7 +339,7 @@ async def _reap_stale_read_write_locks(self) -> None: """ def reap_stale_read_write_locks_txn(txn: LoggingTransaction) -> None: - txn.execute(delete_sql, (self._clock.time_msec() - _LOCK_TIMEOUT_MS,)) + txn.execute(delete_sql, (self.clock.time_msec() - _LOCK_TIMEOUT_MS,)) if txn.rowcount: logger.info("Reaped %d stale locks", txn.rowcount) diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index f726846e57f..b8bd0042d78 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -565,7 +565,7 @@ def get_pending_media_txn(txn: LoggingTransaction) -> Tuple[int, int]: sql, ( user_id.to_string(), - self._clock.time_msec() - self.unused_expiration_time, + self.clock.time_msec() - self.unused_expiration_time, ), ) row = txn.fetchone() @@ -1059,7 +1059,7 @@ def _get_media_uploaded_size_for_user_txn( txn: LoggingTransaction, ) -> int: # Calculate the timestamp for the start of the time period - start_ts = self._clock.time_msec() - time_period_ms + start_ts = self.clock.time_msec() - time_period_ms txn.execute(sql, (user_id, start_ts)) row = txn.fetchone() if row is None: diff --git a/synapse/storage/databases/main/metrics.py b/synapse/storage/databases/main/metrics.py index a3467bff3dc..49411ed0341 100644 --- a/synapse/storage/databases/main/metrics.py +++ b/synapse/storage/databases/main/metrics.py @@ -78,7 +78,7 @@ def __init__( # Read the extrems every 60 minutes if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000) + self.clock.looping_call(self._read_forward_extremities, 60 * 60 * 1000) # Used in _generate_user_daily_visits to keep track of progress self._last_user_visit_update = self._get_start_of_day() @@ -224,7 +224,7 @@ async def count_daily_users(self) -> int: """ Counts the number of users who used this homeserver in the last 24 hours. """ - yesterday = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24) + yesterday = int(self.clock.time_msec()) - (1000 * 60 * 60 * 24) return await self.db_pool.runInteraction( "count_daily_users", self._count_users, yesterday ) @@ -236,7 +236,7 @@ async def count_monthly_users(self) -> int: from the mau figure in synapse.storage.monthly_active_users which, amongst other things, includes a 3 day grace period before a user counts. """ - thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) + thirty_days_ago = int(self.clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) return await self.db_pool.runInteraction( "count_monthly_users", self._count_users, thirty_days_ago ) @@ -281,7 +281,7 @@ async def count_r30v2_users(self) -> Dict[str, int]: def _count_r30v2_users(txn: LoggingTransaction) -> Dict[str, int]: thirty_days_in_secs = 86400 * 30 - now = int(self._clock.time()) + now = int(self.clock.time()) sixty_days_ago_in_secs = now - 2 * thirty_days_in_secs one_day_from_now_in_secs = now + 86400 @@ -389,7 +389,7 @@ def _get_start_of_day(self) -> int: """ Returns millisecond unixtime for start of UTC day. """ - now = time.gmtime(self._clock.time()) + now = time.gmtime(self.clock.time()) today_start = calendar.timegm((now.tm_year, now.tm_mon, now.tm_mday, 0, 0, 0)) return today_start * 1000 @@ -403,7 +403,7 @@ def _generate_user_daily_visits(txn: LoggingTransaction) -> None: logger.info("Calling _generate_user_daily_visits") today_start = self._get_start_of_day() a_day_in_milliseconds = 24 * 60 * 60 * 1000 - now = self._clock.time_msec() + now = self.clock.time_msec() # A note on user_agent. Technically a given device can have multiple # user agents, so we need to decide which one to pick. We could have diff --git a/synapse/storage/databases/main/monthly_active_users.py b/synapse/storage/databases/main/monthly_active_users.py index f5a6b98be71..86744f616ce 100644 --- a/synapse/storage/databases/main/monthly_active_users.py +++ b/synapse/storage/databases/main/monthly_active_users.py @@ -49,7 +49,6 @@ def __init__( hs: "HomeServer", ): super().__init__(database, db_conn, hs) - self._clock = hs.get_clock() self.hs = hs if hs.config.redis.redis_enabled: @@ -226,7 +225,7 @@ def _reap_users(txn: LoggingTransaction, reserved_users: List[str]) -> None: reserved_users: reserved users to preserve """ - thirty_days_ago = int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) + thirty_days_ago = int(self.clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) in_clause, in_clause_args = make_in_list_sql_clause( self.database_engine, "user_id", reserved_users @@ -328,7 +327,7 @@ def _initialise_reserved_users( txn, table="monthly_active_users", keyvalues={"user_id": user_id}, - values={"timestamp": int(self._clock.time_msec())}, + values={"timestamp": int(self.clock.time_msec())}, ) else: logger.warning("mau limit reserved threepid %s not found in db", tp) @@ -391,7 +390,7 @@ def upsert_monthly_active_user_txn( txn, table="monthly_active_users", keyvalues={"user_id": user_id}, - values={"timestamp": int(self._clock.time_msec())}, + values={"timestamp": int(self.clock.time_msec())}, ) self._invalidate_cache_and_stream(txn, self.get_monthly_active_count, ()) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index ff4eb9acb29..f1dbf68971d 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -1073,7 +1073,7 @@ async def insert_receipt( if event_ts is None: return None - now = self._clock.time_msec() + now = self.clock.time_msec() logger.debug( "Receipt %s for event %s in %s (%i ms old)", receipt_type, diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index a0016b82595..cf4309ec645 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -212,7 +212,7 @@ def __init__( ) if hs.config.worker.run_background_tasks: - self._clock.call_later( + self.clock.call_later( 0.0, self._set_expiration_date_when_missing, call_later_cancel_on_shutdown=False, # We don't track this call since it's short @@ -227,7 +227,7 @@ def __init__( # Create a background job for culling expired 3PID validity tokens if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( self.cull_expired_threepid_validation_tokens, THIRTY_MINUTES_IN_MS ) @@ -299,7 +299,7 @@ def _register_user( ) -> None: user_id_obj = UserID.from_string(user_id) - now = int(self._clock.time()) + now = int(self.clock.time()) user_approved = approved or not self._require_approval @@ -458,7 +458,7 @@ async def is_trial_user(self, user_id: str) -> bool: if not info: return False - now = self._clock.time_msec() + now = self.clock.time_msec() days = self.config.server.mau_appservice_trial_days.get( info.appservice_id, self.config.server.mau_trial_days ) @@ -641,7 +641,7 @@ def select_users_txn( return await self.db_pool.runInteraction( "get_users_expiring_soon", select_users_txn, - self._clock.time_msec(), + self.clock.time_msec(), self.config.account_validity.account_validity_renew_at, ) @@ -1085,7 +1085,7 @@ async def count_daily_user_type(self) -> Dict[str, int]: """ def _count_daily_user_type(txn: LoggingTransaction) -> Dict[str, int]: - yesterday = int(self._clock.time()) - (60 * 60 * 24) + yesterday = int(self.clock.time()) - (60 * 60 * 24) sql = """ SELECT user_type, COUNT(*) AS count FROM ( @@ -1497,7 +1497,7 @@ def cull_expired_threepid_validation_tokens_txn( await self.db_pool.runInteraction( "cull_expired_threepid_validation_tokens", cull_expired_threepid_validation_tokens_txn, - self._clock.time_msec(), + self.clock.time_msec(), ) @wrap_as_background_process("account_validity_set_expiration_dates") @@ -1538,7 +1538,7 @@ def set_expiration_date_for_user_txn( random value in the [now + period - d ; now + period] range, d being a delta equal to 10% of the validity period. """ - now_ms = self._clock.time_msec() + now_ms = self.clock.time_msec() assert self._account_validity_period is not None expiration_ts = now_ms + self._account_validity_period @@ -1609,7 +1609,7 @@ async def update_access_token_last_validated(self, token_id: int) -> None: Raises: StoreError if there was a problem updating this. """ - now = self._clock.time_msec() + now = self.clock.time_msec() await self.db_pool.simple_update_one( "access_tokens", @@ -1640,7 +1640,7 @@ async def registration_token_is_valid(self, token: str) -> bool: uses_allowed, pending, completed, expiry_time = res # Check if the token has expired - now = self._clock.time_msec() + now = self.clock.time_msec() if expiry_time and expiry_time < now: return False @@ -1772,7 +1772,7 @@ def select_registration_tokens_txn( return await self.db_pool.runInteraction( "select_registration_tokens", select_registration_tokens_txn, - self._clock.time_msec(), + self.clock.time_msec(), valid, ) @@ -2252,7 +2252,7 @@ async def consume_login_token(self, token: str) -> LoginTokenLookupResult: "consume_login_token", self._consume_login_token, token, - self._clock.time_msec(), + self.clock.time_msec(), ) async def invalidate_login_tokens_by_session_id( @@ -2272,7 +2272,7 @@ async def invalidate_login_tokens_by_session_id( "auth_provider_id": auth_provider_id, "auth_provider_session_id": auth_provider_session_id, }, - updatevalues={"used_ts": self._clock.time_msec()}, + updatevalues={"used_ts": self.clock.time_msec()}, desc="invalidate_login_tokens_by_session_id", ) @@ -2641,7 +2641,6 @@ def __init__( ): super().__init__(database, db_conn, hs) - self._clock = hs.get_clock() self.config = hs.config self.db_pool.updates.register_background_index_update( @@ -2762,7 +2761,7 @@ def __init__( # Create a background job for removing expired login tokens if hs.config.worker.run_background_tasks: - self._clock.looping_call( + self.clock.looping_call( self._delete_expired_login_tokens, THIRTY_MINUTES_IN_MS ) @@ -2791,7 +2790,7 @@ async def add_access_token_to_user( The token ID """ next_id = self._access_tokens_id_gen.get_next() - now = self._clock.time_msec() + now = self.clock.time_msec() await self.db_pool.simple_insert( "access_tokens", @@ -2875,7 +2874,7 @@ def f(txn: LoggingTransaction) -> None: keyvalues={"name": user_id}, updatevalues={ "consent_version": consent_version, - "consent_ts": self._clock.time_msec(), + "consent_ts": self.clock.time_msec(), }, ) self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) @@ -2987,7 +2986,7 @@ def validate_threepid_session_txn(txn: LoggingTransaction) -> Optional[str]: txn, table="threepid_validation_session", keyvalues={"session_id": session_id}, - updatevalues={"validated_at": self._clock.time_msec()}, + updatevalues={"validated_at": self.clock.time_msec()}, ) return next_link @@ -3065,7 +3064,7 @@ def _delete_expired_login_tokens_txn(txn: LoggingTransaction, ts: int) -> None: # We keep the expired tokens for an extra 5 minutes so we can measure how many # times a token is being used after its expiry - now = self._clock.time_msec() + now = self.clock.time_msec() await self.db_pool.runInteraction( "delete_expired_login_tokens", _delete_expired_login_tokens_txn, diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index c57e3a1ed7e..a0f863c9155 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1003,7 +1003,7 @@ async def get_joined_user_ids_from_state( """ with Measure( - self._clock, + self.clock, name="get_joined_user_ids_from_state", server_name=self.server_name, ): diff --git a/synapse/storage/databases/main/session.py b/synapse/storage/databases/main/session.py index 8a5fa8386cd..1154bb2d599 100644 --- a/synapse/storage/databases/main/session.py +++ b/synapse/storage/databases/main/session.py @@ -55,7 +55,7 @@ def __init__( # Create a background job for culling expired sessions. if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000) + self.clock.looping_call(self._delete_expired_sessions, 30 * 60 * 1000) async def create_session( self, session_type: str, value: JsonDict, expiry_ms: int @@ -133,7 +133,7 @@ def _get_session( _get_session, session_type, session_id, - self._clock.time_msec(), + self.clock.time_msec(), ) @wrap_as_background_process("delete_expired_sessions") @@ -147,5 +147,5 @@ def _delete_expired_sessions_txn(txn: LoggingTransaction, ts: int) -> None: await self.db_pool.runInteraction( "delete_expired_sessions", _delete_expired_sessions_txn, - self._clock.time_msec(), + self.clock.time_msec(), ) diff --git a/synapse/storage/databases/main/sliding_sync.py b/synapse/storage/databases/main/sliding_sync.py index f7af3e88d3f..c0c5087b13c 100644 --- a/synapse/storage/databases/main/sliding_sync.py +++ b/synapse/storage/databases/main/sliding_sync.py @@ -201,7 +201,7 @@ def persist_per_connection_state_txn( "user_id": user_id, "effective_device_id": device_id, "conn_id": conn_id, - "created_ts": self._clock.time_msec(), + "created_ts": self.clock.time_msec(), }, returning=("connection_key",), ) @@ -212,7 +212,7 @@ def persist_per_connection_state_txn( table="sliding_sync_connection_positions", values={ "connection_key": connection_key, - "created_ts": self._clock.time_msec(), + "created_ts": self.clock.time_msec(), }, returning=("connection_position",), ) diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index bfc324b80d2..41c94839273 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -81,11 +81,11 @@ def __init__( super().__init__(database, db_conn, hs) if hs.config.worker.run_background_tasks: - self._clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000) + self.clock.looping_call(self._cleanup_transactions, 30 * 60 * 1000) @wrap_as_background_process("cleanup_transactions") async def _cleanup_transactions(self) -> None: - now = self._clock.time_msec() + now = self.clock.time_msec() day_ago = now - 24 * 60 * 60 * 1000 def _cleanup_transactions_txn(txn: LoggingTransaction) -> None: @@ -160,7 +160,7 @@ async def set_received_txn_response( insertion_values={ "response_code": code, "response_json": db_binary_type(encode_canonical_json(response_dict)), - "ts": self._clock.time_msec(), + "ts": self.clock.time_msec(), }, desc="set_received_txn_response", ) diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index 9b3b7e086f9..b62f3e6f5ba 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -125,6 +125,7 @@ def __init__( self._state_group_cache: DictionaryCache[int, StateKey, str] = DictionaryCache( name="*stateGroupCache*", + clock=hs.get_clock(), server_name=self.server_name, # TODO: this hasn't been tuned yet max_entries=50000, @@ -132,6 +133,7 @@ def __init__( self._state_group_members_cache: DictionaryCache[int, StateKey, str] = ( DictionaryCache( name="*stateGroupMembersCache*", + clock=hs.get_clock(), server_name=self.server_name, max_entries=500000, ) diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index 92d446ce2aa..016acbac710 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -47,6 +47,7 @@ from synapse.util.async_helpers import ObservableDeferred from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache, iterate_tree_cache_entry +from synapse.util.clock import Clock cache_pending_metric = Gauge( "synapse_util_caches_cache_pending", @@ -82,6 +83,7 @@ def __init__( self, *, name: str, + clock: Clock, server_name: str, max_entries: int = 1000, tree: bool = False, @@ -103,6 +105,7 @@ def __init__( prune_unread_entries: If True, cache entries that haven't been read recently will be evicted from the cache in the background. Set to False to opt-out of this behaviour. + clock: The homeserver `Clock` instance """ cache_type = TreeCache if tree else dict @@ -120,6 +123,7 @@ def metrics_cb() -> None: # a Deferred. self.cache: LruCache[KT, VT] = LruCache( max_size=max_entries, + clock=clock, server_name=server_name, cache_name=name, cache_type=cache_type, diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 47b8f4ddc81..6e3c8eada98 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -53,6 +53,7 @@ from synapse.util.async_helpers import delay_cancellation from synapse.util.caches.deferred_cache import DeferredCache from synapse.util.caches.lrucache import LruCache +from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -154,13 +155,20 @@ def __init__( ) -class HasServerName(Protocol): +class HasServerNameAndClock(Protocol): server_name: str """ The homeserver name that this cache is associated with (used to label the metric) (`hs.hostname`). """ + clock: Clock + """ + The homeserver clock instance used to track delayed and looping calls. Important to + be able to fully cleanup the homeserver instance on server shutdown. + (`hs.get_clock()`). + """ + class DeferredCacheDescriptor(_CacheDescriptorBase): """A method decorator that applies a memoizing cache around the function. @@ -239,7 +247,7 @@ def __init__( self.prune_unread_entries = prune_unread_entries def __get__( - self, obj: Optional[HasServerName], owner: Optional[Type] + self, obj: Optional[HasServerNameAndClock], owner: Optional[Type] ) -> Callable[..., "defer.Deferred[Any]"]: # We need access to instance-level `obj.server_name` attribute assert obj is not None, ( @@ -249,9 +257,13 @@ def __get__( assert obj.server_name is not None, ( "The `server_name` attribute must be set on the object where `@cached` decorator is used." ) + assert obj.clock is not None, ( + "The `clock` attribute must be set on the object where `@cached` decorator is used." + ) cache: DeferredCache[CacheKey, Any] = DeferredCache( name=self.name, + clock=obj.clock, server_name=obj.server_name, max_entries=self.max_entries, tree=self.tree, diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 168ddc51cd5..eb5493d322d 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -37,6 +37,7 @@ from synapse.util.caches.lrucache import LruCache from synapse.util.caches.treecache import TreeCache +from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -127,10 +128,13 @@ class DictionaryCache(Generic[KT, DKT, DV]): for the '2' dict key. """ - def __init__(self, *, name: str, server_name: str, max_entries: int = 1000): + def __init__( + self, *, name: str, clock: Clock, server_name: str, max_entries: int = 1000 + ): """ Args: name + clock: The homeserver `Clock` instance server_name: The homeserver name that this cache is associated with (used to label the metric) (`hs.hostname`). max_entries @@ -160,6 +164,7 @@ def __init__(self, *, name: str, server_name: str, max_entries: int = 1000): Union[_PerKeyValue, Dict[DKT, DV]], ] = LruCache( max_size=max_entries, + clock=clock, server_name=server_name, cache_name=name, cache_type=TreeCache, diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 187380c433e..3deeb466150 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -45,14 +45,13 @@ overload, ) -from twisted.internet import defer, reactor +from twisted.internet import defer from synapse.config import cache as cache_config from synapse.metrics.background_process_metrics import ( run_as_background_process, ) from synapse.metrics.jemalloc import get_jemalloc_stats -from synapse.types import ISynapseThreadlessReactor from synapse.util import caches from synapse.util.caches import CacheMetric, EvictionReason, register_cache from synapse.util.caches.treecache import ( @@ -404,13 +403,13 @@ def __init__( self, *, max_size: int, + clock: Clock, server_name: str, cache_name: str, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, metrics_collection_callback: Optional[Callable[[], None]] = None, apply_cache_factor_from_config: bool = True, - clock: Optional[Clock] = None, prune_unread_entries: bool = True, extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, ): ... @@ -420,13 +419,13 @@ def __init__( self, *, max_size: int, + clock: Clock, server_name: Literal[None] = None, cache_name: Literal[None] = None, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, metrics_collection_callback: Optional[Callable[[], None]] = None, apply_cache_factor_from_config: bool = True, - clock: Optional[Clock] = None, prune_unread_entries: bool = True, extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, ): ... @@ -435,13 +434,13 @@ def __init__( self, *, max_size: int, + clock: Clock, server_name: Optional[str] = None, cache_name: Optional[str] = None, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, metrics_collection_callback: Optional[Callable[[], None]] = None, apply_cache_factor_from_config: bool = True, - clock: Optional[Clock] = None, prune_unread_entries: bool = True, extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, ): @@ -494,13 +493,6 @@ def __init__( Note: The new key does not have to be unique. """ - # Default `clock` to something sensible. Note that we rename it to - # `real_clock` so that mypy doesn't think its still `Optional`. - if clock is None: - real_clock = Clock(cast(ISynapseThreadlessReactor, reactor)) - else: - real_clock = clock - cache: Union[Dict[KT, _Node[KT, VT]], TreeCache] = cache_type() self.cache = cache # Used for introspection. self.apply_cache_factor_from_config = apply_cache_factor_from_config @@ -592,7 +584,7 @@ def add_node( key, value, weak_ref_to_self, - real_clock, + clock, callbacks, prune_unread_entries, ) @@ -610,7 +602,7 @@ def add_node( metrics.inc_memory_usage(node.memory) def move_node_to_front(node: _Node[KT, VT]) -> None: - node.move_to_front(real_clock, list_root) + node.move_to_front(clock, list_root) def delete_node(node: _Node[KT, VT]) -> int: node.drop_from_lists() diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py index 03bf5a94a7c..6af51931633 100644 --- a/synmark/suites/logging.py +++ b/synmark/suites/logging.py @@ -86,7 +86,9 @@ class _logging: hs_config = Config() # To be able to sleep. - clock = Clock(reactor) + # Ignore linter error here since we are running outside of the context of a + # `HomeServer`. + clock = Clock(reactor) # type: ignore[multiple-internal-clocks] errors = StringIO() publisher = LogPublisher() diff --git a/synmark/suites/lrucache.py b/synmark/suites/lrucache.py index d109441e551..f83f94371c4 100644 --- a/synmark/suites/lrucache.py +++ b/synmark/suites/lrucache.py @@ -23,13 +23,14 @@ from synapse.types import ISynapseReactor from synapse.util.caches.lrucache import LruCache +from synapse.util.clock import Clock async def main(reactor: ISynapseReactor, loops: int) -> float: """ Benchmark `loops` number of insertions into LruCache without eviction. """ - cache: LruCache[int, bool] = LruCache(max_size=loops) + cache: LruCache[int, bool] = LruCache(max_size=loops, clock=Clock(reactor)) # type: ignore[multiple-internal-clocks] start = perf_counter() diff --git a/synmark/suites/lrucache_evict.py b/synmark/suites/lrucache_evict.py index 00cfdd04471..30c5f46c847 100644 --- a/synmark/suites/lrucache_evict.py +++ b/synmark/suites/lrucache_evict.py @@ -23,6 +23,7 @@ from synapse.types import ISynapseReactor from synapse.util.caches.lrucache import LruCache +from synapse.util.clock import Clock async def main(reactor: ISynapseReactor, loops: int) -> float: @@ -30,7 +31,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: Benchmark `loops` number of insertions into LruCache where half of them are evicted. """ - cache: LruCache[int, bool] = LruCache(max_size=loops // 2) + cache: LruCache[int, bool] = LruCache(max_size=loops // 2, clock=Clock(reactor)) # type: ignore[multiple-internal-clocks] start = perf_counter() diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py index deb6bade461..4dce9aae013 100644 --- a/tests/config/test_cache.py +++ b/tests/config/test_cache.py @@ -24,6 +24,7 @@ from synapse.types import JsonDict from synapse.util.caches.lrucache import LruCache +from tests.server import get_clock from tests.unittest import TestCase @@ -32,6 +33,7 @@ def setUp(self) -> None: # Reset caches before each test since there's global state involved. self.config = CacheConfig(RootConfig()) self.config.reset() + _, self.clock = get_clock() def tearDown(self) -> None: # Also reset the caches after each test to leave state pristine. @@ -75,7 +77,7 @@ def test_individual_instantiated_before_config_load(self) -> None: the default cache size in the interim, and then resized once the config is loaded. """ - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache(max_size=100, clock=self.clock) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -96,7 +98,7 @@ def test_individual_instantiated_after_config_load(self) -> None: self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache(max_size=100, clock=self.clock) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 200) @@ -106,7 +108,7 @@ def test_global_instantiated_before_config_load(self) -> None: the default cache size in the interim, and then resized to the new default cache size once the config is loaded. """ - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache(max_size=100, clock=self.clock) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -126,7 +128,7 @@ def test_global_instantiated_after_config_load(self) -> None: self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(max_size=100) + cache: LruCache = LruCache(max_size=100, clock=self.clock) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 150) @@ -145,15 +147,15 @@ def test_cache_with_asterisk_in_name(self) -> None: self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache_a: LruCache = LruCache(max_size=100) + cache_a: LruCache = LruCache(max_size=100, clock=self.clock) add_resizable_cache("*cache_a*", cache_resize_callback=cache_a.set_cache_factor) self.assertEqual(cache_a.max_size, 200) - cache_b: LruCache = LruCache(max_size=100) + cache_b: LruCache = LruCache(max_size=100, clock=self.clock) add_resizable_cache("*Cache_b*", cache_resize_callback=cache_b.set_cache_factor) self.assertEqual(cache_b.max_size, 300) - cache_c: LruCache = LruCache(max_size=100) + cache_c: LruCache = LruCache(max_size=100, clock=self.clock) add_resizable_cache("*cache_c*", cache_resize_callback=cache_c.set_cache_factor) self.assertEqual(cache_c.max_size, 200) @@ -168,6 +170,7 @@ def test_apply_cache_factor_from_config(self) -> None: cache: LruCache = LruCache( max_size=self.config.event_cache_size, + clock=self.clock, apply_cache_factor_from_config=False, ) add_resizable_cache("event_cache", cache_resize_callback=cache.set_cache_factor) diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 832e9917305..b3f42c76f18 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -164,7 +164,10 @@ def test_cache_metric(self) -> None: """ CACHE_NAME = "cache_metrics_test_fgjkbdfg" cache: DeferredCache[str, str] = DeferredCache( - name=CACHE_NAME, server_name=self.hs.hostname, max_entries=777 + name=CACHE_NAME, + clock=self.hs.get_clock(), + server_name=self.hs.hostname, + max_entries=777, ) metrics_map = get_latest_metrics() @@ -212,10 +215,10 @@ def test_cache_metric_multiple_servers(self) -> None: """ CACHE_NAME = "cache_metric_multiple_servers_test" cache1: DeferredCache[str, str] = DeferredCache( - name=CACHE_NAME, server_name="hs1", max_entries=777 + name=CACHE_NAME, clock=self.clock, server_name="hs1", max_entries=777 ) cache2: DeferredCache[str, str] = DeferredCache( - name=CACHE_NAME, server_name="hs2", max_entries=777 + name=CACHE_NAME, clock=self.clock, server_name="hs2", max_entries=777 ) metrics_map = get_latest_metrics() diff --git a/tests/replication/test_module_cache_invalidation.py b/tests/replication/test_module_cache_invalidation.py index 8d5d0cce9a6..1cb898673bc 100644 --- a/tests/replication/test_module_cache_invalidation.py +++ b/tests/replication/test_module_cache_invalidation.py @@ -24,6 +24,7 @@ from synapse.module_api import cached from tests.replication._base import BaseMultiWorkerStreamTestCase +from tests.server import get_clock logger = logging.getLogger(__name__) @@ -36,6 +37,7 @@ class TestCache: current_value = FIRST_VALUE server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() async def cached_function(self, user_id: str) -> str: diff --git a/tests/util/caches/test_deferred_cache.py b/tests/util/caches/test_deferred_cache.py index 7017d6d70ac..f0deb1554ef 100644 --- a/tests/util/caches/test_deferred_cache.py +++ b/tests/util/caches/test_deferred_cache.py @@ -26,20 +26,26 @@ from synapse.util.caches.deferred_cache import DeferredCache +from tests.server import get_clock from tests.unittest import TestCase class DeferredCacheTestCase(TestCase): + def setUp(self) -> None: + super().setUp() + + _, self.clock = get_clock() + def test_empty(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) with self.assertRaises(KeyError): cache.get("foo") def test_hit(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) cache.prefill("foo", 123) @@ -47,7 +53,7 @@ def test_hit(self) -> None: def test_hit_deferred(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) origin_d: "defer.Deferred[int]" = defer.Deferred() set_d = cache.set("k1", origin_d) @@ -72,7 +78,7 @@ def check1(r: str) -> str: def test_callbacks(self) -> None: """Invalidation callbacks are called at the right time""" cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) callbacks = set() @@ -107,7 +113,7 @@ def test_callbacks(self) -> None: def test_set_fail(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) callbacks = set() @@ -146,7 +152,7 @@ def test_set_fail(self) -> None: def test_get_immediate(self) -> None: cache: DeferredCache[str, int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) d1: "defer.Deferred[int]" = defer.Deferred() cache.set("key1", d1) @@ -164,7 +170,7 @@ def test_get_immediate(self) -> None: def test_invalidate(self) -> None: cache: DeferredCache[Tuple[str], int] = DeferredCache( - name="test", server_name="test_server" + name="test", clock=self.clock, server_name="test_server" ) cache.prefill(("foo",), 123) cache.invalidate(("foo",)) @@ -174,7 +180,7 @@ def test_invalidate(self) -> None: def test_invalidate_all(self) -> None: cache: DeferredCache[str, str] = DeferredCache( - name="testcache", server_name="test_server" + name="testcache", clock=self.clock, server_name="test_server" ) callback_record = [False, False] @@ -220,6 +226,7 @@ def record_callback(idx: int) -> None: def test_eviction(self) -> None: cache: DeferredCache[int, str] = DeferredCache( name="test", + clock=self.clock, server_name="test_server", max_entries=2, apply_cache_factor_from_config=False, @@ -238,6 +245,7 @@ def test_eviction(self) -> None: def test_eviction_lru(self) -> None: cache: DeferredCache[int, str] = DeferredCache( name="test", + clock=self.clock, server_name="test_server", max_entries=2, apply_cache_factor_from_config=False, @@ -260,6 +268,7 @@ def test_eviction_lru(self) -> None: def test_eviction_iterable(self) -> None: cache: DeferredCache[int, List[str]] = DeferredCache( name="test", + clock=self.clock, server_name="test_server", max_entries=3, apply_cache_factor_from_config=False, diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py index 3dc0224adf1..d93b97ddd7a 100644 --- a/tests/util/caches/test_descriptors.py +++ b/tests/util/caches/test_descriptors.py @@ -49,6 +49,7 @@ from synapse.util.caches.descriptors import _CacheContext, cached, cachedList from tests import unittest +from tests.server import get_clock from tests.test_utils import get_awaitable_result logger = logging.getLogger(__name__) @@ -70,6 +71,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int) -> str: @@ -105,6 +107,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached(num_args=1) def fn(self, arg1: int, arg2: int) -> str: @@ -151,6 +154,7 @@ def fn(self, arg1: int, arg2: int, arg3: int) -> str: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached obj = Cls() obj.mock.return_value = "fish" @@ -182,6 +186,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, kwarg1: int = 2) -> str: @@ -217,6 +222,7 @@ def test_cache_with_sync_exception(self) -> None: class Cls: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> NoReturn: @@ -242,6 +248,7 @@ class Cls: result: Optional[Deferred] = None call_count = 0 server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> Deferred: @@ -296,6 +303,7 @@ def test_cache_logcontexts(self) -> Deferred: class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int) -> "Deferred[int]": @@ -340,6 +348,7 @@ def test_cache_logcontexts_with_exception(self) -> "Deferred[None]": class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int) -> Deferred: @@ -384,6 +393,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int = 2, arg3: int = 3) -> str: @@ -422,6 +432,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached(iterable=True) def fn(self, arg1: int, arg2: int) -> Tuple[str, ...]: @@ -456,6 +467,7 @@ def test_cache_iterable_with_sync_exception(self) -> None: class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached(iterable=True) def fn(self, arg1: int) -> NoReturn: @@ -479,6 +491,7 @@ def test_invalidate_cascade(self) -> None: class Cls: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached(cache_context=True) async def func1(self, key: str, cache_context: _CacheContext) -> int: @@ -507,6 +520,7 @@ def test_cancel(self) -> None: class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @cached() async def fn(self, arg1: int) -> str: @@ -540,6 +554,7 @@ def test_cancel_logcontexts(self) -> None: class Cls: inner_context_was_finished = False server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() async def fn(self, arg1: int) -> str: @@ -586,6 +601,7 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): def test_passthrough(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -602,6 +618,7 @@ def test_hit(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -622,6 +639,7 @@ def test_invalidate(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -642,6 +660,7 @@ def func(self, key: str) -> str: def test_invalidate_missing(self) -> None: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -655,6 +674,7 @@ def test_max_entries(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached(max_entries=10) def func(self, key: int) -> int: @@ -684,6 +704,7 @@ def test_prefill(self) -> None: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> "Deferred[int]": @@ -704,6 +725,7 @@ def test_invalidate_context(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -739,6 +761,7 @@ def test_eviction_context(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached(max_entries=2) def func(self, key: str) -> str: @@ -778,6 +801,7 @@ def test_double_get(self) -> Generator["Deferred[Any]", object, None]: class A: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def func(self, key: str) -> str: @@ -827,6 +851,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -893,6 +918,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int) -> None: @@ -937,6 +963,7 @@ class Cls: def __init__(self) -> None: self.mock = mock.Mock() self.server_name = "test_server" + _, self.clock = get_clock() # nb must be called this for @cached @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -978,6 +1005,7 @@ def test_cancel(self) -> None: class Cls: server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> None: @@ -1014,6 +1042,7 @@ def test_cancel_logcontexts(self) -> None: class Cls: inner_context_was_finished = False server_name = "test_server" # nb must be called this for @cached + _, clock = get_clock() # nb must be called this for @cached @cached() def fn(self, arg1: int) -> None: @@ -1058,6 +1087,7 @@ def test_num_args_mismatch(self) -> None: class Cls: server_name = "test_server" + _, clock = get_clock() # nb must be called this for @cached @descriptors.cached(tree=True) def fn(self, room_id: str, event_id: str) -> None: diff --git a/tests/util/test_dict_cache.py b/tests/util/test_dict_cache.py index 246e18fd155..16e096a4b25 100644 --- a/tests/util/test_dict_cache.py +++ b/tests/util/test_dict_cache.py @@ -23,12 +23,14 @@ from synapse.util.caches.dictionary_cache import DictionaryCache from tests import unittest +from tests.server import get_clock class DictCacheTestCase(unittest.TestCase): def setUp(self) -> None: + _, clock = get_clock() self.cache: DictionaryCache[str, str, str] = DictionaryCache( - name="foobar", server_name="test_server", max_entries=10 + name="foobar", clock=clock, server_name="test_server", max_entries=10 ) def test_simple_cache_hit_full(self) -> None: diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index b7acf586904..4c198366083 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -29,18 +29,24 @@ from synapse.util.caches.treecache import TreeCache from tests import unittest +from tests.server import get_clock from tests.unittest import override_config class LruCacheTestCase(unittest.HomeserverTestCase): + def setUp(self) -> None: + super().setUp() + + _, self.clock = get_clock() + def test_get_set(self) -> None: - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache(max_size=1, clock=self.clock) cache["key"] = "value" self.assertEqual(cache.get("key"), "value") self.assertEqual(cache["key"], "value") def test_eviction(self) -> None: - cache: LruCache[int, int] = LruCache(max_size=2) + cache: LruCache[int, int] = LruCache(max_size=2, clock=self.clock) cache[1] = 1 cache[2] = 2 @@ -54,7 +60,7 @@ def test_eviction(self) -> None: self.assertEqual(cache.get(3), 3) def test_setdefault(self) -> None: - cache: LruCache[str, int] = LruCache(max_size=1) + cache: LruCache[str, int] = LruCache(max_size=1, clock=self.clock) self.assertEqual(cache.setdefault("key", 1), 1) self.assertEqual(cache.get("key"), 1) self.assertEqual(cache.setdefault("key", 2), 1) @@ -63,7 +69,7 @@ def test_setdefault(self) -> None: self.assertEqual(cache.get("key"), 2) def test_pop(self) -> None: - cache: LruCache[str, int] = LruCache(max_size=1) + cache: LruCache[str, int] = LruCache(max_size=1, clock=self.clock) cache["key"] = 1 self.assertEqual(cache.pop("key"), 1) self.assertEqual(cache.pop("key"), None) @@ -71,7 +77,7 @@ def test_pop(self) -> None: def test_del_multi(self) -> None: # The type here isn't quite correct as they don't handle TreeCache well. cache: LruCache[Tuple[str, str], str] = LruCache( - max_size=4, cache_type=TreeCache + max_size=4, clock=self.clock, cache_type=TreeCache ) cache[("animal", "cat")] = "mew" cache[("animal", "dog")] = "woof" @@ -91,7 +97,7 @@ def test_del_multi(self) -> None: # Man from del_multi say "Yes". def test_clear(self) -> None: - cache: LruCache[str, int] = LruCache(max_size=1) + cache: LruCache[str, int] = LruCache(max_size=1, clock=self.clock) cache["key"] = 1 cache.clear() self.assertEqual(len(cache), 0) @@ -99,7 +105,10 @@ def test_clear(self) -> None: @override_config({"caches": {"per_cache_factors": {"mycache": 10}}}) def test_special_size(self) -> None: cache: LruCache = LruCache( - max_size=10, server_name="test_server", cache_name="mycache" + max_size=10, + clock=self.clock, + server_name="test_server", + cache_name="mycache", ) self.assertEqual(cache.max_size, 100) @@ -107,7 +116,7 @@ def test_special_size(self) -> None: class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache(max_size=1, clock=self.clock) cache.set("key", "value") self.assertFalse(m.called) @@ -126,7 +135,7 @@ def test_get(self) -> None: def test_multi_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache(max_size=1, clock=self.clock) cache.set("key", "value") self.assertFalse(m.called) @@ -145,7 +154,7 @@ def test_multi_get(self) -> None: def test_set(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache(max_size=1, clock=self.clock) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -161,7 +170,7 @@ def test_set(self) -> None: def test_pop(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(max_size=1) + cache: LruCache[str, str] = LruCache(max_size=1, clock=self.clock) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -182,7 +191,7 @@ def test_del_multi(self) -> None: m4 = Mock() # The type here isn't quite correct as they don't handle TreeCache well. cache: LruCache[Tuple[str, str], str] = LruCache( - max_size=4, cache_type=TreeCache + max_size=4, clock=self.clock, cache_type=TreeCache ) cache.set(("a", "1"), "value", callbacks=[m1]) @@ -205,7 +214,7 @@ def test_del_multi(self) -> None: def test_clear(self) -> None: m1 = Mock() m2 = Mock() - cache: LruCache[str, str] = LruCache(max_size=5) + cache: LruCache[str, str] = LruCache(max_size=5, clock=self.clock) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -222,7 +231,7 @@ def test_eviction(self) -> None: m1 = Mock(name="m1") m2 = Mock(name="m2") m3 = Mock(name="m3") - cache: LruCache[str, str] = LruCache(max_size=2) + cache: LruCache[str, str] = LruCache(max_size=2, clock=self.clock) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -258,7 +267,9 @@ def test_eviction(self) -> None: class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_evict(self) -> None: - cache: LruCache[str, List[int]] = LruCache(max_size=5, size_callback=len) + cache: LruCache[str, List[int]] = LruCache( + max_size=5, clock=self.clock, size_callback=len + ) cache["key1"] = [0] cache["key2"] = [1, 2] cache["key3"] = [3] @@ -282,7 +293,7 @@ def test_evict(self) -> None: def test_zero_size_drop_from_cache(self) -> None: """Test that `drop_from_cache` works correctly with 0-sized entries.""" cache: LruCache[str, List[int]] = LruCache( - max_size=5, size_callback=lambda x: 0 + max_size=5, clock=self.clock, size_callback=lambda x: 0 ) cache["key1"] = [] @@ -396,7 +407,7 @@ def test_evict_memory(self, jemalloc_interface: Mock) -> None: class ExtraIndexLruCacheTestCase(unittest.HomeserverTestCase): def test_invalidate_simple(self) -> None: cache: LruCache[str, int] = LruCache( - max_size=10, extra_index_cb=lambda k, v: str(v) + max_size=10, clock=self.hs.get_clock(), extra_index_cb=lambda k, v: str(v) ) cache["key1"] = 1 cache["key2"] = 2 @@ -411,7 +422,7 @@ def test_invalidate_simple(self) -> None: def test_invalidate_multi(self) -> None: cache: LruCache[str, int] = LruCache( - max_size=10, extra_index_cb=lambda k, v: str(v) + max_size=10, clock=self.hs.get_clock(), extra_index_cb=lambda k, v: str(v) ) cache["key1"] = 1 cache["key2"] = 1 From 92dba449f8f1234fba3c8c901988de3914e5f3f5 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Sat, 27 Sep 2025 19:34:36 -0600 Subject: [PATCH 162/181] Switch to using hs.run_as_background_process --- scripts-dev/mypy_synapse_plugin.py | 31 ++++++++++++++ synapse/app/_base.py | 6 ++- synapse/app/phone_stats_home.py | 12 ++---- synapse/appservice/scheduler.py | 16 ++++---- synapse/federation/federation_client.py | 2 + synapse/federation/sender/__init__.py | 7 +--- .../sender/per_destination_queue.py | 5 +-- synapse/handlers/appservice.py | 6 +-- synapse/handlers/deactivate_account.py | 5 +-- synapse/handlers/delayed_events.py | 18 +++------ synapse/handlers/device.py | 12 +++--- synapse/handlers/federation.py | 13 ++---- synapse/handlers/federation_event.py | 8 ++-- synapse/handlers/message.py | 16 ++++---- synapse/handlers/pagination.py | 10 ++--- synapse/handlers/presence.py | 5 +-- synapse/handlers/room_member.py | 5 +-- synapse/handlers/stats.py | 3 +- synapse/handlers/sync.py | 1 + synapse/handlers/user_directory.py | 12 ++---- synapse/http/matrixfederationclient.py | 1 + synapse/media/media_repository.py | 7 +--- synapse/media/url_previewer.py | 7 ++-- synapse/metrics/background_process_metrics.py | 2 +- synapse/module_api/__init__.py | 18 ++++++--- synapse/push/emailpusher.py | 3 +- synapse/push/httppusher.py | 7 ++-- synapse/push/pusherpool.py | 5 +-- synapse/replication/tcp/client.py | 5 +-- synapse/replication/tcp/handler.py | 5 +-- synapse/replication/tcp/protocol.py | 21 ++++++---- synapse/replication/tcp/redis.py | 15 ++++--- synapse/replication/tcp/resource.py | 9 +++-- synapse/rest/client/room.py | 5 +-- synapse/server.py | 40 ++++++++++++------- synapse/state/__init__.py | 1 + synapse/storage/controllers/persist_events.py | 7 ++-- synapse/storage/databases/main/deviceinbox.py | 1 + synapse/storage/databases/main/lock.py | 10 +++-- synapse/util/caches/expiringcache.py | 10 +++-- synapse/util/caches/lrucache.py | 8 ++-- synapse/util/clock.py | 6 ++- synapse/util/distributor.py | 14 ++++--- synapse/util/retryutils.py | 11 +++-- synapse/util/task_scheduler.py | 5 +-- tests/appservice/test_scheduler.py | 22 +++++----- tests/test_distributor.py | 5 ++- tests/util/test_expiring_cache.py | 4 ++ tests/util/test_retryutils.py | 10 +++++ 49 files changed, 252 insertions(+), 205 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 32d65776087..49008ecff22 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -98,6 +98,12 @@ category="synapse-reactor-clock", ) +UNTRACKED_BACKGROUND_PROCESS = ErrorCode( + "untracked-background-process", + "All calls to `run_as_background_process` should use the `HomeServer` method", + category="synapse-tracked-calls", +) + class Sentinel(enum.Enum): # defining a sentinel in this way allows mypy to correctly handle the @@ -246,6 +252,12 @@ def get_function_signature_hook( if fullname == "synapse.util.clock.Clock": return check_clock_creation + if ( + fullname + == "synapse.metrics.background_process_metrics.run_as_background_process" + ): + return check_background_process + return None def get_method_signature_hook( @@ -359,6 +371,25 @@ def check_clock_creation(ctx: FunctionSigContext) -> CallableType: return signature +def check_background_process(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that calls to `run_as_background_process` use the `HomeServer` method. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected all calls to `run_as_background_process` to use the `HomeServer` method. " + "This is so that the `HomeServer` can cancel any background processes " + "during server shutdown", + ctx.context, + code=UNTRACKED_BACKGROUND_PROCESS, + ) + + return signature + + def check_call_when_running(ctx: MethodSigContext) -> CallableType: """ Ensure that the `reactor.callWhenRunning` callsites aren't used. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 0716a988701..d4ec57f4db4 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -589,7 +589,11 @@ async def _handle_sighup(*args: Any, **kwargs: Any) -> None: sdnotify(b"READY=1") - return run_as_background_process( + # It's okay to ignore the linter error here and call + # `run_as_background_process` directly because `_handle_sighup` operates + # outside of the scope of a specific `HomeServer` instance and holds no + # references to it which would prevent a clean shutdown. + return run_as_background_process( # type: ignore[untracked-background-process] "sighup", server_name, _handle_sighup, diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index faff293d0bc..52b29b9586e 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -29,9 +29,6 @@ from twisted.internet import defer from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import ( - run_as_background_process, -) from synapse.types import JsonDict from synapse.util.constants import ONE_HOUR_SECONDS, ONE_MINUTE_SECONDS @@ -85,8 +82,6 @@ def phone_stats_home( stats: JsonDict, stats_process: List[Tuple[int, "resource.struct_rusage"]] = _stats_process, ) -> "defer.Deferred[None]": - server_name = hs.hostname - async def _phone_stats_home( hs: "HomeServer", stats: JsonDict, @@ -200,8 +195,8 @@ async def _phone_stats_home( except Exception as e: logger.warning("Error reporting stats: %s", e) - return run_as_background_process( - "phone_stats_home", server_name, _phone_stats_home, hs, stats, stats_process + return hs.run_as_background_process( + "phone_stats_home", _phone_stats_home, hs, stats, stats_process ) @@ -263,9 +258,8 @@ async def _generate_monthly_active_users() -> None: float(hs.config.server.max_mau_value) ) - return run_as_background_process( + return hs.run_as_background_process( "generate_monthly_active_users", - server_name, _generate_monthly_active_users, ) diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 73985c01c2c..6eeed976708 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -81,7 +81,6 @@ from synapse.appservice.api import ApplicationServiceApi from synapse.events import EventBase from synapse.logging.context import run_in_background -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main import DataStore from synapse.types import DeviceListUpdates, JsonMapping from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock @@ -200,6 +199,7 @@ def __init__(self, txn_ctrl: "_TransactionController", hs: "HomeServer"): ) self.server_name = hs.hostname self.clock = hs.get_clock() + self.hs = hs self._store = hs.get_datastores().main def start_background_request(self, service: ApplicationService) -> None: @@ -207,9 +207,7 @@ def start_background_request(self, service: ApplicationService) -> None: if service.id in self.requests_in_flight: return - run_as_background_process( - "as-sender", self.server_name, self._send_request, service - ) + self.hs.run_as_background_process("as-sender", self._send_request, service) async def _send_request(self, service: ApplicationService) -> None: # sanity-check: we shouldn't get here if this service already has a sender @@ -361,6 +359,7 @@ class _TransactionController: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname self.clock = hs.get_clock() + self.hs = hs self.store = hs.get_datastores().main self.as_api = hs.get_application_service_api() @@ -448,6 +447,7 @@ def start_recoverer(self, service: ApplicationService) -> None: recoverer = self.RECOVERER_CLASS( self.server_name, self.clock, + self.hs, self.store, self.as_api, service, @@ -494,6 +494,7 @@ def __init__( self, server_name: str, clock: Clock, + hs: "HomeServer", store: DataStore, as_api: ApplicationServiceApi, service: ApplicationService, @@ -501,6 +502,7 @@ def __init__( ): self.server_name = server_name self.clock = clock + self.hs = hs self.store = store self.as_api = as_api self.service = service @@ -513,9 +515,8 @@ def recover(self) -> None: logger.info("Scheduling retries on %s in %fs", self.service.id, delay) self.scheduled_recovery = self.clock.call_later( delay, - run_as_background_process, + self.hs.run_as_background_process, "as-recoverer", - self.server_name, self.retry, # Only track this call if it would delay shutdown by a substantial amount call_later_cancel_on_shutdown=True @@ -539,9 +540,8 @@ def force_retry(self) -> None: if self.scheduled_recovery: self.clock.cancel_call_later(self.scheduled_recovery) # Run a retry, which will resechedule a recovery if it fails. - run_as_background_process( + self.hs.run_as_background_process( "retry", - self.server_name, self.retry, ) diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 41595043d11..8c91336dbc1 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -148,6 +148,7 @@ def __init__(self, hs: "HomeServer"): self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache( cache_name="get_pdu_cache", server_name=self.server_name, + hs=self.hs, clock=self._clock, max_len=1000, expiry_ms=120 * 1000, @@ -167,6 +168,7 @@ def __init__(self, hs: "HomeServer"): ] = ExpiringCache( cache_name="get_room_hierarchy_cache", server_name=self.server_name, + hs=self.hs, clock=self._clock, max_len=1000, expiry_ms=5 * 60 * 1000, diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 4f2445cd920..caac495d921 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -168,7 +168,6 @@ events_processed_counter, ) from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.types import ( @@ -467,10 +466,9 @@ def __init__(self, hs: "HomeServer"): # Regularly wake up destinations that have outstanding PDUs to be caught up self.clock.looping_call_now( - run_as_background_process, + self.hs.run_as_background_process, WAKEUP_RETRY_PERIOD_SEC * 1000.0, "wake_destinations_needing_catchup", - self.server_name, self._wake_destinations_needing_catchup, ) @@ -517,9 +515,8 @@ def notify_new_events(self, max_token: RoomStreamToken) -> None: return # fire off a processing loop in the background - run_as_background_process( + self.hs.run_as_background_process( "process_event_queue_for_federation", - self.server_name, self._process_event_queue_loop, ) diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 12c2d91a012..845af92facf 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -43,7 +43,6 @@ from synapse.logging import issue9533_logger from synapse.logging.opentracing import SynapseTags, set_tag from synapse.metrics import SERVER_NAME_LABEL, sent_transactions_counter -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import JsonDict, ReadReceipt from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter from synapse.visibility import filter_events_for_server @@ -335,9 +334,8 @@ def attempt_new_transaction(self) -> None: logger.debug("TX [%s] Starting transaction loop", self._destination) - self.active_transmission_loop = run_as_background_process( + self.active_transmission_loop = self._hs.run_as_background_process( "federation_transaction_transmission_loop", - self.server_name, self._transaction_transmission_loop, ) @@ -351,6 +349,7 @@ async def _transaction_transmission_loop(self) -> None: await get_retry_limiter( destination=self._destination, our_server_name=self.server_name, + hs=self._hs, clock=self._clock, store=self._store, ) diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index bf36cf39a19..e6cbdb587c2 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -47,7 +47,6 @@ event_processing_loop_room_count, ) from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.storage.databases.main.directory import RoomAliasMapping @@ -79,6 +78,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = ( hs.hostname ) # nb must be called this for @wrap_as_background_process + self.hs = hs self.store = hs.get_datastores().main self.is_mine_id = hs.is_mine_id self.appservice_api = hs.get_application_service_api() @@ -171,8 +171,8 @@ async def start_scheduler() -> None: except Exception: logger.error("Application Services Failure") - run_as_background_process( - "as_scheduler", self.server_name, start_scheduler + self.hs.run_as_background_process( + "as_scheduler", start_scheduler ) self.started_scheduler = True diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index c0684380a70..204dffd2882 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -24,7 +24,6 @@ from synapse.api.constants import Membership from synapse.api.errors import SynapseError -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.deactivate_account import ( ReplicationNotifyAccountDeactivatedServlet, ) @@ -272,8 +271,8 @@ def _start_user_parting(self) -> None: pending deactivation, if it isn't already running. """ if not self._user_parter_running: - run_as_background_process( - "user_parter_loop", self.server_name, self._user_parter_loop + self.hs.run_as_background_process( + "user_parter_loop", self._user_parter_loop ) async def _user_parter_loop(self) -> None: diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 51ff44a3a84..73c466a60e9 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -24,9 +24,6 @@ from synapse.logging.context import make_deferred_yieldable from synapse.logging.opentracing import set_tag from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions -from synapse.metrics.background_process_metrics import ( - run_as_background_process, -) from synapse.replication.http.delayed_events import ( ReplicationAddedDelayedEventRestServlet, ) @@ -59,6 +56,7 @@ class DelayedEventsHandler: def __init__(self, hs: "HomeServer"): + self.hs = hs self.server_name = hs.hostname self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -117,15 +115,14 @@ async def _schedule_db_events() -> None: self._schedule_next_at(next_send_ts) # Can send the events in background after having awaited on marking them as processed - run_as_background_process( + self.hs.run_as_background_process( "_send_events", - self.server_name, self._send_events, events, ) - self._initialized_from_db = run_as_background_process( - "_schedule_db_events", self.server_name, _schedule_db_events + self._initialized_from_db = self.hs.run_as_background_process( + "_schedule_db_events", _schedule_db_events ) else: self._repl_client = ReplicationAddedDelayedEventRestServlet.make_client(hs) @@ -150,9 +147,7 @@ async def process() -> None: finally: self._event_processing = False - run_as_background_process( - "delayed_events.notify_new_event", self.server_name, process - ) + self.hs.run_as_background_process("delayed_events.notify_new_event", process) async def _unsafe_process_new_event(self) -> None: # We purposefully fetch the current max room stream ordering before @@ -547,9 +542,8 @@ def _schedule_next_at(self, next_send_ts: Timestamp) -> None: if self._next_delayed_event_call is None: self._next_delayed_event_call = self._clock.call_later( delay_sec, - run_as_background_process, + self.hs.run_as_background_process, "_send_on_timeout", - self.server_name, self._send_on_timeout, # Only track this call if it would delay shutdown by a substantial amount call_later_cancel_on_shutdown=True diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 9509ac422ec..39e2104899d 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -47,7 +47,6 @@ ) from synapse.logging.opentracing import log_kv, set_tag, trace from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.http.devices import ( @@ -191,10 +190,9 @@ def __init__(self, hs: "HomeServer"): and self._delete_stale_devices_after is not None ): self.clock.looping_call( - run_as_background_process, + self.hs.run_as_background_process, DELETE_STALE_DEVICES_INTERVAL_MS, desc="delete_stale_devices", - server_name=self.server_name, func=self._delete_stale_devices, ) @@ -1444,6 +1442,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): super().__init__(hs) + self.hs = hs self.server_name = hs.hostname self.federation = hs.get_federation_client() self.server_name = hs.hostname # nb must be called this for @measure_func @@ -1468,6 +1467,7 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache( cache_name="device_update_edu", server_name=self.server_name, + hs=self.hs, clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, @@ -1477,9 +1477,8 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): # Attempt to resync out of sync device lists every 30s. self._resync_retry_lock = Lock() self.clock.looping_call( - run_as_background_process, + self.hs.run_as_background_process, 30 * 1000, - server_name=self.server_name, func=self._maybe_retry_device_resync, desc="_maybe_retry_device_resync", ) @@ -1599,9 +1598,8 @@ async def _handle_device_updates(self, user_id: str) -> None: if resync: # We mark as stale up front in case we get restarted. await self.store.mark_remote_users_device_caches_as_stale([user_id]) - run_as_background_process( + self.hs.run_as_background_process( "_maybe_retry_device_resync", - self.server_name, self.multi_user_device_resync, [user_id], False, diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 41fb3076c36..adc20f4ad02 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -72,7 +72,6 @@ from synapse.logging.context import nested_logging_context from synapse.logging.opentracing import SynapseTags, set_tag, tag_args, trace from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.module_api import NOT_SPAM from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.storage.invite_rule import InviteRule @@ -188,9 +187,8 @@ def __init__(self, hs: "HomeServer"): # any partial-state-resync operations which were in flight when we # were shut down. if not hs.config.worker.worker_app: - run_as_background_process( + self.hs.run_as_background_process( "resume_sync_partial_state_room", - self.server_name, self._resume_partial_state_room_sync, ) @@ -318,9 +316,8 @@ async def _maybe_backfill_inner( logger.debug( "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points." ) - run_as_background_process( + self.hs.run_as_background_process( "_maybe_backfill_inner_anyway_with_max_depth", - self.server_name, self.maybe_backfill, room_id=room_id, # We use `MAX_DEPTH` so that we find all backfill points next @@ -802,9 +799,8 @@ async def do_invite_join( # lots of requests for missing prev_events which we do actually # have. Hence we fire off the background task, but don't wait for it. - run_as_background_process( + self.hs.run_as_background_process( "handle_queued_pdus", - self.server_name, self._handle_queued_pdus, room_queue, ) @@ -1877,9 +1873,8 @@ async def _sync_partial_state_room_wrapper() -> None: room_id=room_id, ) - run_as_background_process( + self.hs.run_as_background_process( desc="sync_partial_state_room", - server_name=self.server_name, func=_sync_partial_state_room_wrapper, ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index f117ff5f246..d6390b79c7b 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -81,7 +81,6 @@ trace, ) from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.federation import ( ReplicationFederationSendEventsRestServlet, ) @@ -153,6 +152,7 @@ class FederationEventHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self._clock = hs.get_clock() self._store = hs.get_datastores().main self._state_store = hs.get_datastores().state @@ -975,9 +975,8 @@ async def _process_new_pulled_events(new_events: Collection[EventBase]) -> None: # Process previously failed backfill events in the background to not waste # time on something that is likely to fail again. if len(events_with_failed_pull_attempts) > 0: - run_as_background_process( + self.hs.run_as_background_process( "_process_new_pulled_events_with_failed_pull_attempts", - self.server_name, _process_new_pulled_events, events_with_failed_pull_attempts, ) @@ -1569,9 +1568,8 @@ async def _process_received_pdu( resync = True if resync: - run_as_background_process( + self.hs.run_as_background_process( "resync_device_due_to_pdu", - self.server_name, self._resync_device, event.sender, ) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7068cacf223..66ff3a472aa 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -67,7 +67,6 @@ from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging import opentracing from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.send_events import ReplicationSendEventsRestServlet from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.types import ( @@ -100,6 +99,7 @@ class MessageHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self.auth = hs.get_auth() self.clock = hs.get_clock() self.state = hs.get_state_handler() @@ -114,8 +114,8 @@ def __init__(self, hs: "HomeServer"): self._scheduled_expiry: Optional[IDelayedCall] = None if not hs.config.worker.worker_app: - run_as_background_process( - "_schedule_next_expiry", self.server_name, self._schedule_next_expiry + self.hs.run_as_background_process( + "_schedule_next_expiry", self._schedule_next_expiry ) async def get_room_data( @@ -445,9 +445,8 @@ def _schedule_expiry_for_event(self, event_id: str, expiry_ts: int) -> None: self._scheduled_expiry = self.clock.call_later( delay, - run_as_background_process, + self.hs.run_as_background_process, "_expire_event", - self.server_name, self._expire_event, event_id, # Only track this call if it would delay shutdown by a substantial amount @@ -553,9 +552,8 @@ def __init__(self, hs: "HomeServer"): and self.config.server.cleanup_extremities_with_dummy_events ): self.clock.looping_call( - lambda: run_as_background_process( + lambda: self.hs.run_as_background_process( "send_dummy_events_to_fill_extremities", - self.server_name, self._send_dummy_events_to_fill_extremities, ), 5 * 60 * 1000, @@ -575,6 +573,7 @@ def __init__(self, hs: "HomeServer"): self._external_cache_joined_hosts_updates = ExpiringCache( cache_name="_external_cache_joined_hosts_updates", server_name=self.server_name, + hs=self.hs, clock=self.clock, expiry_ms=30 * 60 * 1000, ) @@ -2118,9 +2117,8 @@ async def persist_and_notify_client_events( if event.type == EventTypes.Message: # We don't want to block sending messages on any presence code. This # matters as sometimes presence code can take a while. - run_as_background_process( + self.hs.run_as_background_process( "bump_presence_active_time", - self.server_name, self._bump_active_time, requester.user, requester.device_id, diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index df1a7e714ce..02a67581e75 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -29,7 +29,6 @@ from synapse.events.utils import SerializeEventConfig from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging.opentracing import trace -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.admin._base import assert_user_is_admin from synapse.streams.config import PaginationConfig from synapse.types import ( @@ -116,10 +115,9 @@ def __init__(self, hs: "HomeServer"): logger.info("Setting up purge job with config: %s", job) self.clock.looping_call( - run_as_background_process, + self.hs.run_as_background_process, job.interval, "purge_history_for_rooms_in_range", - self.server_name, self.purge_history_for_rooms_in_range, job.shortest_max_lifetime, job.longest_max_lifetime, @@ -244,9 +242,8 @@ async def purge_history_for_rooms_in_range( # We want to purge everything, including local events, and to run the purge in # the background so that it's not blocking any other operation apart from # other purges in the same room. - run_as_background_process( + self.hs.run_as_background_process( PURGE_HISTORY_ACTION_NAME, - self.server_name, self.purge_history, room_id, token, @@ -604,9 +601,8 @@ async def get_messages( # Otherwise, we can backfill in the background for eventual # consistency's sake but we don't need to block the client waiting # for a costly federation call and processing. - run_as_background_process( + self.hs.run_as_background_process( "maybe_backfill_in_the_background", - self.server_name, self.hs.get_federation_handler().maybe_backfill, room_id, curr_topo, diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index bf697a812f0..7656abfaab7 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -107,7 +107,6 @@ from synapse.logging.context import run_in_background from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.replication.http.presence import ( @@ -1539,8 +1538,8 @@ async def _process_presence() -> None: finally: self._event_processing = False - run_as_background_process( - "presence.notify_new_event", self.server_name, _process_presence + self.hs.run_as_background_process( + "presence.notify_new_event", _process_presence ) async def _unsafe_process(self) -> None: diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index b426db07903..2c94ec600b9 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -50,7 +50,6 @@ from synapse.handlers.worker_lock import NEW_EVENT_DURING_PURGE_LOCK_NAME from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.http.push import ReplicationCopyPusherRestServlet from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.invite_rule import InviteRule @@ -2209,9 +2208,7 @@ async def process() -> None: finally: self._is_processing = False - run_as_background_process( - "room_forgetter.notify_new_event", self.server_name, process - ) + self._hs.run_as_background_process("room_forgetter.notify_new_event", process) async def _unsafe_process(self) -> None: # If self.pos is None then means we haven't fetched it from DB diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index bc9c24a1b0f..5221af725d8 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -33,7 +33,6 @@ from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.state_deltas import StateDelta from synapse.types import JsonDict from synapse.util.events import get_plain_text_topic_from_event_content @@ -94,7 +93,7 @@ async def process() -> None: finally: self._is_processing = False - run_as_background_process("stats.notify_new_event", self.server_name, process) + self.hs.run_as_background_process("stats.notify_new_event", process) async def _unsafe_process(self) -> None: # If self.pos is None then means we haven't fetched it from DB diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 86f7c084079..6f0522d5bba 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -323,6 +323,7 @@ def __init__(self, hs: "HomeServer"): ] = ExpiringCache( cache_name="lazy_loaded_members_cache", server_name=self.server_name, + hs=hs, clock=self.clock, max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 5e5993943dd..d467f1c547e 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -36,7 +36,6 @@ from synapse.api.errors import Codes, SynapseError from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.state_deltas import StateDelta from synapse.storage.databases.main.user_directory import SearchResult from synapse.storage.roommember import ProfileInfo @@ -200,9 +199,7 @@ async def process() -> None: self._is_processing = False self._is_processing = True - run_as_background_process( - "user_directory.notify_new_event", self.server_name, process - ) + self._hs.run_as_background_process("user_directory.notify_new_event", process) async def handle_local_profile_change( self, user_id: str, profile: ProfileInfo @@ -618,8 +615,8 @@ async def process() -> None: self._is_refreshing_remote_profiles = False self._is_refreshing_remote_profiles = True - run_as_background_process( - "user_directory.refresh_remote_profiles", self.server_name, process + self._hs.run_as_background_process( + "user_directory.refresh_remote_profiles", process ) async def _unsafe_refresh_remote_profiles(self) -> None: @@ -708,9 +705,8 @@ async def process() -> None: self._is_refreshing_remote_profiles_for_servers.remove(server_name) self._is_refreshing_remote_profiles_for_servers.add(server_name) - run_as_background_process( + self._hs.run_as_background_process( "user_directory.refresh_remote_profiles_for_remote_server", - self.server_name, process, ) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index c89cd205f03..617d6fd57d8 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -641,6 +641,7 @@ async def _send_request( limiter = await synapse.util.retryutils.get_retry_limiter( destination=request.destination, our_server_name=self.server_name, + hs=self.hs, clock=self.clock, store=self._store, backoff_on_404=backoff_on_404, diff --git a/synapse/media/media_repository.py b/synapse/media/media_repository.py index 436d9b7e35f..238dc6cb2f3 100644 --- a/synapse/media/media_repository.py +++ b/synapse/media/media_repository.py @@ -67,7 +67,6 @@ from synapse.media.storage_provider import StorageProviderWrapper from synapse.media.thumbnailer import Thumbnailer, ThumbnailError from synapse.media.url_previewer import UrlPreviewer -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia from synapse.types import UserID from synapse.util.async_helpers import Linearizer @@ -187,16 +186,14 @@ def __init__(self, hs: "HomeServer"): self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository def _start_update_recently_accessed(self) -> Deferred: - return run_as_background_process( + return self.hs.run_as_background_process( "update_recently_accessed_media", - self.server_name, self._update_recently_accessed, ) def _start_apply_media_retention_rules(self) -> Deferred: - return run_as_background_process( + return self.hs.run_as_background_process( "apply_media_retention_rules", - self.server_name, self._apply_media_retention_rules, ) diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index bb5c0920a71..1a82cc46e3e 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -44,7 +44,6 @@ from synapse.media.media_storage import MediaStorage, SHA256TransparentIOWriter from synapse.media.oembed import OEmbedProvider from synapse.media.preview_html import decode_body, parse_html_to_open_graph -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import JsonDict, UserID from synapse.util.async_helpers import ObservableDeferred from synapse.util.caches.expiringcache import ExpiringCache @@ -167,6 +166,7 @@ def __init__( media_storage: MediaStorage, ): self.clock = hs.get_clock() + self.hs = hs self.filepaths = media_repo.filepaths self.max_spider_size = hs.config.media.max_spider_size self.server_name = hs.hostname @@ -201,6 +201,7 @@ def __init__( self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache( cache_name="url_previews", server_name=self.server_name, + hs=self.hs, clock=self.clock, # don't spider URLs more often than once an hour expiry_ms=ONE_HOUR, @@ -737,8 +738,8 @@ async def _handle_oembed_response( return open_graph_result, oembed_response.author_name, expiration_ms def _start_expire_url_cache_data(self) -> Deferred: - return run_as_background_process( - "expire_url_cache_data", self.server_name, self._expire_url_cache_data + return self.hs.run_as_background_process( + "expire_url_cache_data", self._expire_url_cache_data ) async def _expire_url_cache_data(self) -> None: diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 6cdfa03f4e6..32d6fb85d22 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -442,7 +442,7 @@ def wrapped_func( "The `server_name` attribute must be set on the object where `@wrap_as_background_process` decorator is used." ) - return run_as_background_process( + return run_as_background_process( # type: ignore[untracked-background-process] desc, self.server_name, func, diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index d3068ba053c..65c07dd06da 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -275,7 +275,15 @@ def run_as_background_process( # function instead. stub_server_name = "synapse_module_running_from_unknown_server" - return _run_as_background_process( + # Ignore the linter error here. Since this is leveraging the + # `run_as_background_process` function directly and we don't want to break the + # module api, we need to keep the function signature the same. This means we don't + # have access to the running `HomeServer` and cannot track this background process + # for cleanup during shutdown. + # This is not an issue during runtime and is only potentially problematic if the + # application cares about being able to garbage collect `HomeServer` instances + # during runtime. + return _run_as_background_process( # type: ignore[untracked-background-process] desc, stub_server_name, func, @@ -1402,7 +1410,7 @@ def looping_background_call( if self._hs.config.worker.run_background_tasks or run_on_all_instances: self._clock.looping_call( - self.run_as_background_process, + self._hs.run_as_background_process, msec, desc, lambda: maybe_awaitable(f(*args, **kwargs)), @@ -1460,7 +1468,7 @@ def delayed_background_call( return self._clock.call_later( # convert ms to seconds as needed by call_later. msec * 0.001, - self.run_as_background_process, + self._hs.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), call_later_cancel_on_shutdown=False, # We don't track calls in the module api since we don't know their purpose @@ -1702,8 +1710,8 @@ def run_as_background_process( Note that the returned Deferred does not follow the synapse logcontext rules. """ - return _run_as_background_process( - desc, self.server_name, func, *args, bg_start_span=bg_start_span, **kwargs + return self._hs.run_as_background_process( + desc, func, *args, bg_start_span=bg_start_span, **kwargs ) async def defer_to_thread( diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index b73bd226648..98da43727a9 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -25,7 +25,6 @@ from twisted.internet.error import AlreadyCalled, AlreadyCancelled from twisted.internet.interfaces import IDelayedCall -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.push import Pusher, PusherConfig, PusherConfigException, ThrottleParams from synapse.push.mailer import Mailer from synapse.push.push_types import EmailReason @@ -119,7 +118,7 @@ def _start_processing(self) -> None: if self._is_processing: return - run_as_background_process("emailpush.process", self.server_name, self._process) + self.hs.run_as_background_process("emailpush.process", self._process) def _pause_processing(self) -> None: """Used by tests to temporarily pause processing of events. diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 1ec085953ea..801f157b825 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -32,7 +32,6 @@ from synapse.events import EventBase from synapse.logging import opentracing from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.push import Pusher, PusherConfig, PusherConfigException from synapse.storage.databases.main.event_push_actions import HttpPushAction from synapse.types import JsonDict, JsonMapping @@ -183,8 +182,8 @@ def on_new_receipts(self) -> None: # We could check the receipts are actually m.read receipts here, # but currently that's the only type of receipt anyway... - run_as_background_process( - "http_pusher.on_new_receipts", self.server_name, self._update_badge + self.hs.run_as_background_process( + "http_pusher.on_new_receipts", self._update_badge ) async def _update_badge(self) -> None: @@ -220,7 +219,7 @@ def _start_processing(self) -> None: if self.failing_since and self.timed_call and self.timed_call.active(): return - run_as_background_process("httppush.process", self.server_name, self._process) + self.hs.run_as_background_process("httppush.process", self._process) async def _process(self) -> None: # we should never get here if we are already processing diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index d1f79ec9995..39e7530751b 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -27,7 +27,6 @@ from synapse.api.errors import Codes, SynapseError from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.push import Pusher, PusherConfig, PusherConfigException @@ -112,9 +111,7 @@ def start(self) -> None: if not self._should_start_pushers: logger.info("Not starting pushers because they are disabled in the config") return - run_as_background_process( - "start_pushers", self.server_name, self._start_pushers - ) + self.hs.run_as_background_process("start_pushers", self._start_pushers) async def add_or_update_pusher( self, diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 01d624d49ed..576619deb8b 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -32,7 +32,6 @@ from synapse.federation import send_queue from synapse.federation.sender import FederationSender from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.streams import ( AccountDataStream, DeviceListsStream, @@ -516,8 +515,8 @@ async def update_token(self, token: int) -> None: # no need to queue up another task. return - run_as_background_process( - "_save_and_send_ack", self.server_name, self._save_and_send_ack + self._hs.run_as_background_process( + "_save_and_send_ack", self._save_and_send_ack ) async def _save_and_send_ack(self) -> None: diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index dd7e38dd781..4d0d3d44abc 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -41,7 +41,6 @@ from twisted.internet.protocol import ReconnectingClientFactory from synapse.metrics import SERVER_NAME_LABEL, LaterGauge -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.commands import ( ClearUserSyncsCommand, Command, @@ -132,6 +131,7 @@ class ReplicationCommandHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self._replication_data_handler = hs.get_replication_data_handler() self._presence_handler = hs.get_presence_handler() self._store = hs.get_datastores().main @@ -361,9 +361,8 @@ def _add_command_to_stream_queue( return # fire off a background process to start processing the queue. - run_as_background_process( + self.hs.run_as_background_process( "process-replication-data", - self.server_name, self._unsafe_process_queue, stream_name, ) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 25a7868cd7b..bcfc65c2c0d 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -42,7 +42,6 @@ from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( BackgroundProcessLoggingContext, - run_as_background_process, ) from synapse.replication.tcp.commands import ( VALID_CLIENT_COMMANDS, @@ -140,9 +139,14 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver): max_line_buffer = 10000 def __init__( - self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler" + self, + hs: "HomeServer", + server_name: str, + clock: Clock, + handler: "ReplicationCommandHandler", ): self.server_name = server_name + self.hs = hs self.clock = clock self.command_handler = handler @@ -290,9 +294,8 @@ def handle_command(self, cmd: Command) -> None: # if so. if isawaitable(res): - run_as_background_process( + self.hs.run_as_background_process( "replication-" + cmd.get_logcontext_id(), - self.server_name, lambda: res, ) @@ -470,9 +473,13 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol): VALID_OUTBOUND_COMMANDS = VALID_SERVER_COMMANDS def __init__( - self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler" + self, + hs: "HomeServer", + server_name: str, + clock: Clock, + handler: "ReplicationCommandHandler", ): - super().__init__(server_name, clock, handler) + super().__init__(hs, server_name, clock, handler) self.server_name = server_name @@ -497,7 +504,7 @@ def __init__( clock: Clock, command_handler: "ReplicationCommandHandler", ): - super().__init__(server_name, clock, command_handler) + super().__init__(hs, server_name, clock, command_handler) self.client_name = client_name self.server_name = server_name diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 0b1be033b11..7f013c56566 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -40,7 +40,6 @@ from synapse.metrics import SERVER_NAME_LABEL from synapse.metrics.background_process_metrics import ( BackgroundProcessLoggingContext, - run_as_background_process, wrap_as_background_process, ) from synapse.replication.tcp.commands import ( @@ -109,6 +108,7 @@ class RedisSubscriber(SubscriberProtocol): """ server_name: str + hs: "HomeServer" synapse_handler: "ReplicationCommandHandler" synapse_stream_prefix: str synapse_channel_names: List[str] @@ -146,9 +146,7 @@ def _get_logging_context(self) -> BackgroundProcessLoggingContext: def connectionMade(self) -> None: logger.info("Connected to redis") super().connectionMade() - run_as_background_process( - "subscribe-replication", self.server_name, self._send_subscribe - ) + self.hs.run_as_background_process("subscribe-replication", self._send_subscribe) async def _send_subscribe(self) -> None: # it's important to make sure that we only send the REPLICATE command once we @@ -223,8 +221,8 @@ def handle_command(self, cmd: Command) -> None: # if so. if isawaitable(res): - run_as_background_process( - "replication-" + cmd.get_logcontext_id(), self.server_name, lambda: res + self.hs.run_as_background_process( + "replication-" + cmd.get_logcontext_id(), lambda: res ) def connectionLost(self, reason: Failure) -> None: # type: ignore[override] @@ -245,9 +243,8 @@ def send_command(self, cmd: Command) -> None: Args: cmd: The command to send """ - run_as_background_process( + self.hs.run_as_background_process( "send-cmd", - self.server_name, self._async_send_command, cmd, # We originally started tracing background processes to avoid `There was no @@ -397,6 +394,7 @@ def __init__( ) self.server_name = hs.hostname + self.hs = hs self.synapse_handler = hs.get_replication_command_handler() self.synapse_stream_prefix = hs.hostname self.synapse_channel_names = channel_names @@ -412,6 +410,7 @@ def buildProtocol(self, addr: IAddress) -> RedisSubscriber: # the base method does some other things than just instantiating the # protocol. p.server_name = self.server_name + p.hs = self.hs p.synapse_handler = self.synapse_handler p.synapse_outbound_redis_connection = self.synapse_outbound_redis_connection p.synapse_stream_prefix = self.synapse_stream_prefix diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index d800cfe6f60..ef72a0a5325 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -30,7 +30,6 @@ from twisted.internet.protocol import ServerFactory from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.replication.tcp.commands import PositionCommand from synapse.replication.tcp.protocol import ServerReplicationStreamProtocol from synapse.replication.tcp.streams import EventsStream @@ -55,6 +54,7 @@ class ReplicationStreamProtocolFactory(ServerFactory): def __init__(self, hs: "HomeServer"): self.command_handler = hs.get_replication_command_handler() self.clock = hs.get_clock() + self.hs = hs self.server_name = hs.config.server.server_name # If we've created a `ReplicationStreamProtocolFactory` then we're @@ -69,7 +69,7 @@ def __init__(self, hs: "HomeServer"): def buildProtocol(self, addr: IAddress) -> ServerReplicationStreamProtocol: return ServerReplicationStreamProtocol( - self.server_name, self.clock, self.command_handler + self.hs, self.server_name, self.clock, self.command_handler ) @@ -82,6 +82,7 @@ class ReplicationStreamer: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname + self.hs = hs self.store = hs.get_datastores().main self.clock = hs.get_clock() self.notifier = hs.get_notifier() @@ -147,8 +148,8 @@ def on_notifier_poke(self) -> None: logger.debug("Notifier poke loop already running") return - run_as_background_process( - "replication_notifier", self.server_name, self._run_notifier_loop + self.hs.run_as_background_process( + "replication_notifier", self._run_notifier_loop ) async def _run_notifier_loop(self) -> None: diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 64deae76507..1084139df0f 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -66,7 +66,6 @@ from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import set_tag from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.client._base import client_patterns from synapse.rest.client.transactions import HttpTransactionCache from synapse.state import CREATE_KEY, POWER_KEY @@ -1225,6 +1224,7 @@ class RoomRedactEventRestServlet(TransactionRestServlet): def __init__(self, hs: "HomeServer"): super().__init__(hs) self.server_name = hs.hostname + self.hs = hs self.event_creation_handler = hs.get_event_creation_handler() self.auth = hs.get_auth() self._store = hs.get_datastores().main @@ -1307,9 +1307,8 @@ async def _do( ) if with_relations: - run_as_background_process( + self.hs.run_as_background_process( "redact_related_events", - self.server_name, self._relation_handler.redact_events_related_to, requester=requester, event_id=event_id, diff --git a/synapse/server.py b/synapse/server.py index f4579ba9c29..a1e65a06600 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -359,6 +359,7 @@ def __init__( # This attribute is set by the free function `refresh_certificate`. self.tls_server_context_factory: Optional[IOpenSSLContextFactory] = None + self._is_shutdown = False self._async_shutdown_handlers: List[ShutdownInfo] = [] self._sync_shutdown_handlers: List[ShutdownInfo] = [] self._background_processes: set[defer.Deferred[Optional[Any]]] = set() @@ -401,11 +402,21 @@ def run_as_background_process( Note that the returned Deferred does not follow the synapse logcontext rules. """ - deferred = run_as_background_process(desc, self.hostname, func, *args, **kwargs) + if self._is_shutdown: + raise Exception( + f"Cannot start background process. HomeServer has been shutdown {len(self._background_processes)} {len(self.get_clock()._looping_calls)} {len(self.get_clock()._call_id_to_delayed_call)}" + ) + + # Ignore linter error as this is the one location this should be called. + deferred = run_as_background_process(desc, self.hostname, func, *args, **kwargs) # type: ignore[untracked-background-process] self._background_processes.add(deferred) def on_done(res: R) -> R: - self._background_processes.remove(deferred) + try: + self._background_processes.remove(deferred) + except KeyError: + # If the background process isn't being tracked anymore we can just move on. + pass return res deferred.addBoth(on_done) @@ -422,6 +433,8 @@ async def shutdown(self) -> None: object in the garbage collector. """ + self._is_shutdown = True + logger.info( "Received shutdown request for %s (%s).", self.hostname, @@ -479,15 +492,6 @@ async def shutdown(self) -> None: except Exception: pass - self.get_clock().shutdown() - - for background_process in list(self._background_processes): - try: - background_process.cancel() - except Exception: - pass - self._background_processes.clear() - for shutdown_handler in self._async_shutdown_handlers: try: self.get_reactor().removeSystemEventTrigger(shutdown_handler.trigger_id) @@ -504,6 +508,15 @@ async def shutdown(self) -> None: logger.error("Error calling shutdown sync handler: %s", e) self._sync_shutdown_handlers.clear() + self.get_clock().shutdown() + + for background_process in list(self._background_processes): + try: + background_process.cancel() + except Exception: + pass + self._background_processes.clear() + for db in self.get_datastores().databases: db._db_pool.close() @@ -523,9 +536,8 @@ def register_async_shutdown_handler( id = self.get_clock().add_system_event_trigger( phase, eventType, - run_as_background_process, + self.run_as_background_process, desc, - self.config.server.server_name, shutdown_func, kwargs, ) @@ -683,7 +695,7 @@ def get_datastores(self) -> Databases: @cache_in_self def get_distributor(self) -> Distributor: - return Distributor(server_name=self.hostname) + return Distributor(hs=self) @cache_in_self def get_registration_ratelimiter(self) -> Ratelimiter: diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index dd8d7135ba3..394dc72fa69 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -651,6 +651,7 @@ def __init__(self, hs: "HomeServer"): ExpiringCache( cache_name="state_cache", server_name=self.server_name, + hs=hs, clock=self.clock, max_len=100000, expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index 120934af578..646e2cf1151 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -62,7 +62,6 @@ trace, ) from synapse.metrics import SERVER_NAME_LABEL -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.controllers.state import StateStorageController from synapse.storage.databases import Databases from synapse.storage.databases.main.events import DeltaState @@ -195,6 +194,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]): def __init__( self, + hs: "HomeServer", server_name: str, per_item_callback: Callable[ [str, _EventPersistQueueTask], @@ -207,6 +207,7 @@ def __init__( and its result will be returned via the Deferreds returned from add_to_queue. """ self.server_name = server_name + self.hs = hs self._event_persist_queues: Dict[str, Deque[_EventPersistQueueItem]] = {} self._currently_persisting_rooms: Set[str] = set() self._per_item_callback = per_item_callback @@ -311,7 +312,7 @@ async def handle_queue_loop() -> None: self._currently_persisting_rooms.discard(room_id) # set handle_queue_loop off in the background - run_as_background_process("persist_events", self.server_name, handle_queue_loop) + self.hs.run_as_background_process("persist_events", handle_queue_loop) def _get_drainining_queue( self, room_id: str @@ -354,7 +355,7 @@ def __init__( self._instance_name = hs.get_instance_name() self.is_mine_id = hs.is_mine_id self._event_persist_queue = _EventPeristenceQueue( - self.server_name, self._process_event_persist_queue_task + hs, self.server_name, self._process_event_persist_queue_task ) self._state_resolution_handler = hs.get_state_resolution_handler() self._state_controller = state_controller diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 7857939c461..a66e11f738c 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -96,6 +96,7 @@ def __init__( ] = ExpiringCache( cache_name="last_device_delete_cache", server_name=self.server_name, + hs=hs, clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index aa2de4fe06c..c3e1c8aa4ed 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -28,7 +28,6 @@ from twisted.internet.task import LoopingCall from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.storage._base import SQLBaseStore @@ -203,6 +202,7 @@ def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: lock = Lock( self.server_name, self._reactor, + self.hs, self.clock, self, read_write=False, @@ -271,6 +271,7 @@ def _try_acquire_read_write_lock_txn( lock = Lock( self.server_name, self._reactor, + self.hs, self.clock, self, read_write=True, @@ -375,6 +376,7 @@ def __init__( self, server_name: str, reactor: ISynapseReactor, + hs: "HomeServer", clock: Clock, store: LockStore, read_write: bool, @@ -388,6 +390,7 @@ def __init__( """ self._server_name = server_name self._reactor = reactor + self._hs = hs self._clock = clock self._store = store self._read_write = read_write @@ -411,6 +414,7 @@ def _setup_looping_call(self) -> None: _RENEWAL_INTERVAL_MS, self._server_name, self._store, + self._hs, self._clock, self._read_write, self._lock_name, @@ -422,6 +426,7 @@ def _setup_looping_call(self) -> None: def _renew( server_name: str, store: LockStore, + hs: "HomeServer", clock: Clock, read_write: bool, lock_name: str, @@ -458,9 +463,8 @@ async def _internal_renew( desc="renew_lock", ) - return run_as_background_process( + return hs.run_as_background_process( "Lock._renew", - server_name, _internal_renew, store, clock, diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 5479d274026..29ce6c0a776 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -22,6 +22,7 @@ import logging from collections import OrderedDict from typing import ( + TYPE_CHECKING, Any, Generic, Iterable, @@ -37,10 +38,12 @@ from twisted.internet import defer from synapse.config import cache as cache_config -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.util.caches import EvictionReason, register_cache from synapse.util.clock import Clock +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -58,6 +61,7 @@ def __init__( *, cache_name: str, server_name: str, + hs: "HomeServer", clock: Clock, max_len: int = 0, expiry_ms: int = 0, @@ -108,9 +112,7 @@ def __init__( return def f() -> "defer.Deferred[None]": - return run_as_background_process( - "prune_cache", server_name, self._prune_cache - ) + return hs.run_as_background_process("prune_cache", self._prune_cache) self._clock.looping_call(f, self._expiry_ms / 2) diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 7d4dc6284b3..324acb728ab 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -48,9 +48,6 @@ from twisted.internet import defer from synapse.config import cache as cache_config -from synapse.metrics.background_process_metrics import ( - run_as_background_process, -) from synapse.metrics.jemalloc import get_jemalloc_stats from synapse.util import caches from synapse.util.caches import CacheMetric, EvictionReason, register_cache @@ -122,6 +119,7 @@ def update_last_access(self, clock: Clock) -> None: def _expire_old_entries( server_name: str, + hs: "HomeServer", clock: Clock, expiry_seconds: float, autotune_config: Optional[dict], @@ -227,9 +225,8 @@ async def _internal_expire_old_entries( logger.info("Dropped %d items from caches", i) - return run_as_background_process( + return hs.run_as_background_process( "LruCache._expire_old_entries", - server_name, _internal_expire_old_entries, clock, expiry_seconds, @@ -260,6 +257,7 @@ def setup_expire_lru_cache_entries(hs: "HomeServer") -> None: _expire_old_entries, 30 * 1000, server_name, + hs, clock, expiry_time, hs.config.caches.cache_autotuning, diff --git a/synapse/util/clock.py b/synapse/util/clock.py index b45f69979bd..e421395d7a3 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -488,7 +488,11 @@ def cancel(self) -> None: underlying delayed_call. """ self.delayed_call.cancel() - self.clock._call_id_to_delayed_call.pop(self.call_id) + try: + self.clock._call_id_to_delayed_call.pop(self.call_id) + except KeyError: + # If the delayed call isn't being tracked anymore we can just move on. + pass def getTime(self) -> float: """Propagate the call to the underlying delayed_call.""" diff --git a/synapse/util/distributor.py b/synapse/util/distributor.py index f48ae3373ce..dec6536e4e1 100644 --- a/synapse/util/distributor.py +++ b/synapse/util/distributor.py @@ -20,6 +20,7 @@ # import logging from typing import ( + TYPE_CHECKING, Any, Awaitable, Callable, @@ -36,10 +37,13 @@ from twisted.internet import defer from synapse.logging.context import make_deferred_yieldable, run_in_background -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import UserID from synapse.util.async_helpers import maybe_awaitable +if TYPE_CHECKING: + from synapse.server import HomeServer + + logger = logging.getLogger(__name__) @@ -58,13 +62,13 @@ class Distributor: model will do for today. """ - def __init__(self, server_name: str) -> None: + def __init__(self, hs: "HomeServer") -> None: """ Args: server_name: The homeserver name of the server (used to label metrics) (this should be `hs.hostname`). """ - self.server_name = server_name + self.hs = hs self.signals: Dict[str, Signal] = {} self.pre_registration: Dict[str, List[Callable]] = {} @@ -97,8 +101,8 @@ def fire(self, name: str, *args: Any, **kwargs: Any) -> None: if name not in self.signals: raise KeyError("%r does not have a signal named %s" % (self, name)) - run_as_background_process( - name, self.server_name, self.signals[name].fire, *args, **kwargs + self.hs.run_as_background_process( + name, self.signals[name].fire, *args, **kwargs ) diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py index 42a0cc7aa81..96fe2bd5664 100644 --- a/synapse/util/retryutils.py +++ b/synapse/util/retryutils.py @@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Any, Optional, Type from synapse.api.errors import CodeMessageException -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage import DataStore from synapse.types import StrCollection from synapse.util.clock import Clock @@ -32,6 +31,7 @@ if TYPE_CHECKING: from synapse.notifier import Notifier from synapse.replication.tcp.handler import ReplicationCommandHandler + from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -62,6 +62,7 @@ async def get_retry_limiter( *, destination: str, our_server_name: str, + hs: "HomeServer", clock: Clock, store: DataStore, ignore_backoff: bool = False, @@ -124,6 +125,7 @@ async def get_retry_limiter( return RetryDestinationLimiter( destination=destination, our_server_name=our_server_name, + hs=hs, clock=clock, store=store, failure_ts=failure_ts, @@ -163,6 +165,7 @@ def __init__( *, destination: str, our_server_name: str, + hs: "HomeServer", clock: Clock, store: DataStore, failure_ts: Optional[int], @@ -181,6 +184,7 @@ def __init__( Args: destination our_server_name: Our homeserver name (used to label metrics) (`hs.hostname`) + hs: The homeserver instance clock store failure_ts: when this destination started failing (in ms since @@ -197,6 +201,7 @@ def __init__( error code. """ self.our_server_name = our_server_name + self.hs = hs self.clock = clock self.store = store self.destination = destination @@ -331,6 +336,4 @@ async def store_retry_timings() -> None: logger.exception("Failed to store destination_retry_timings") # we deliberately do this in the background. - run_as_background_process( - "store_retry_timings", self.our_server_name, store_retry_timings - ) + self.hs.run_as_background_process("store_retry_timings", store_retry_timings) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index e290c4ddc90..2269e344c71 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -32,7 +32,6 @@ ) from synapse.metrics import SERVER_NAME_LABEL, LaterGauge from synapse.metrics.background_process_metrics import ( - run_as_background_process, wrap_as_background_process, ) from synapse.types import JsonMapping, ScheduledTask, TaskStatus @@ -362,7 +361,7 @@ async def inner() -> None: finally: self._launching_new_tasks = False - run_as_background_process("launch_scheduled_tasks", self.server_name, inner) + self._hs.run_as_background_process("launch_scheduled_tasks", inner) @wrap_as_background_process("clean_scheduled_tasks") async def _clean_scheduled_tasks(self) -> None: @@ -497,4 +496,4 @@ async def wrapper() -> None: self._running_tasks.add(task.id) await self.update_task(task.id, status=TaskStatus.ACTIVE) - run_as_background_process(f"task-{task.action}", self.server_name, wrapper) + self._hs.run_as_background_process(f"task-{task.action}", wrapper) diff --git a/tests/appservice/test_scheduler.py b/tests/appservice/test_scheduler.py index c1084b9df35..7485a4f9f86 100644 --- a/tests/appservice/test_scheduler.py +++ b/tests/appservice/test_scheduler.py @@ -18,7 +18,7 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import List, Optional, Sequence, Tuple, cast +from typing import List, Optional, Sequence, Tuple from unittest.mock import AsyncMock, Mock from typing_extensions import TypeAlias @@ -168,17 +168,17 @@ def test_single_service_up_txn_not_sent(self) -> None: ) -class ApplicationServiceSchedulerRecovererTestCase(unittest.TestCase): +class ApplicationServiceSchedulerRecovererTestCase(unittest.HomeserverTestCase): def setUp(self) -> None: - self.clock = MockClock() - self.hs = Mock() + super().setUp() self.as_api = Mock() self.store = Mock() self.service = Mock() self.callback = AsyncMock() self.recoverer = _Recoverer( server_name="test_server", - clock=cast(Clock, self.clock), + hs=self.hs, + clock=self.clock, as_api=self.as_api, store=self.store, service=self.service, @@ -203,7 +203,7 @@ def take_txn( txn.send = AsyncMock(return_value=True) txn.complete = AsyncMock(return_value=None) # wait for exp backoff - self.clock.advance_time(2) + self.reactor.advance(2) self.assertEqual(1, txn.send.call_count) self.assertEqual(1, txn.complete.call_count) # 2 because it needs to get None to know there are no more txns @@ -230,21 +230,21 @@ def take_txn( self.assertEqual(0, self.store.get_oldest_unsent_txn.call_count) txn.send = AsyncMock(return_value=False) txn.complete = AsyncMock(return_value=None) - self.clock.advance_time(2) + self.reactor.advance(2) self.assertEqual(1, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) - self.clock.advance_time(4) + self.reactor.advance(4) self.assertEqual(2, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) - self.clock.advance_time(8) + self.reactor.advance(8) self.assertEqual(3, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) txn.send = AsyncMock(return_value=True) # successfully send the txn pop_txn = True # returns the txn the first time, then no more. - self.clock.advance_time(16) + self.reactor.advance(16) self.assertEqual(1, txn.send.call_count) # new mock reset call count self.assertEqual(1, txn.complete.call_count) self.callback.assert_called_once_with(self.recoverer) @@ -269,7 +269,7 @@ def take_txn( self.assertEqual(0, self.store.get_oldest_unsent_txn.call_count) txn.send = AsyncMock(return_value=False) txn.complete = AsyncMock(return_value=None) - self.clock.advance_time(2) + self.reactor.advance(2) self.assertEqual(1, txn.send.call_count) self.assertEqual(0, txn.complete.call_count) self.assertEqual(0, self.callback.call_count) diff --git a/tests/test_distributor.py b/tests/test_distributor.py index 19dafe64ed4..2dd26833c8d 100644 --- a/tests/test_distributor.py +++ b/tests/test_distributor.py @@ -26,9 +26,10 @@ from . import unittest -class DistributorTestCase(unittest.TestCase): +class DistributorTestCase(unittest.HomeserverTestCase): def setUp(self) -> None: - self.dist = Distributor(server_name="test_server") + super().setUp() + self.dist = Distributor(hs=self.hs) def test_signal_dispatch(self) -> None: self.dist.declare("alert") diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index bfcc6cd12f2..9dc4b5e91da 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -35,6 +35,7 @@ def test_get_set(self) -> None: cache: ExpiringCache[str, str] = ExpiringCache( cache_name="test", server_name="testserver", + hs=self.hs, clock=cast(Clock, clock), max_len=1, ) @@ -48,6 +49,7 @@ def test_eviction(self) -> None: cache: ExpiringCache[str, str] = ExpiringCache( cache_name="test", server_name="testserver", + hs=self.hs, clock=cast(Clock, clock), max_len=2, ) @@ -67,6 +69,7 @@ def test_iterable_eviction(self) -> None: cache: ExpiringCache[str, List[int]] = ExpiringCache( cache_name="test", server_name="testserver", + hs=self.hs, clock=cast(Clock, clock), max_len=5, iterable=True, @@ -91,6 +94,7 @@ def test_time_eviction(self) -> None: cache: ExpiringCache[str, int] = ExpiringCache( cache_name="test", server_name="testserver", + hs=self.hs, clock=cast(Clock, clock), expiry_ms=1000, ) diff --git a/tests/util/test_retryutils.py b/tests/util/test_retryutils.py index 82baff58837..593be93ea3a 100644 --- a/tests/util/test_retryutils.py +++ b/tests/util/test_retryutils.py @@ -35,6 +35,7 @@ def test_new_destination(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -57,6 +58,7 @@ def test_limiter(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -89,6 +91,7 @@ def test_limiter(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ), @@ -104,6 +107,7 @@ def test_limiter(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -139,6 +143,7 @@ def test_limiter(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -165,6 +170,7 @@ def test_notifier_replication(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, notifier=notifier, @@ -238,6 +244,7 @@ def test_max_retry_interval(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -261,6 +268,7 @@ def test_max_retry_interval(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ), @@ -273,6 +281,7 @@ def test_max_retry_interval(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ) @@ -297,6 +306,7 @@ def test_max_retry_interval(self) -> None: get_retry_limiter( destination="test_dest", our_server_name=self.hs.hostname, + hs=self.hs, clock=self.clock, store=store, ), From cb699a7eb514315b9d798afa756fe0cf9199df01 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Sat, 27 Sep 2025 20:33:33 -0600 Subject: [PATCH 163/181] Make call_later cancellation bool optional with default logic --- synapse/app/phone_stats_home.py | 1 - synapse/appservice/scheduler.py | 6 +----- synapse/handlers/delayed_events.py | 6 ------ synapse/handlers/message.py | 5 ----- synapse/handlers/presence.py | 2 -- synapse/handlers/room_member.py | 1 - synapse/handlers/stats.py | 1 - synapse/handlers/user_directory.py | 11 ----------- synapse/handlers/worker_lock.py | 11 +---------- synapse/http/client.py | 4 ---- synapse/metrics/background_process_metrics.py | 1 - synapse/push/emailpusher.py | 5 ----- synapse/push/httppusher.py | 5 ----- synapse/replication/tcp/client.py | 1 - synapse/storage/database.py | 2 -- synapse/storage/databases/main/registration.py | 1 - synapse/storage/databases/main/roommember.py | 1 - synapse/util/async_helpers.py | 15 +++++---------- synapse/util/clock.py | 18 +++++++++++++++--- synapse/util/ratelimitutils.py | 1 - synapse/util/task_scheduler.py | 1 - tests/util/test_async_helpers.py | 3 --- 22 files changed, 22 insertions(+), 80 deletions(-) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 52b29b9586e..bd88f711bbb 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -282,7 +282,6 @@ async def _generate_monthly_active_users() -> None: clock.call_later( 0, performance_stats_init, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # We wait 5 minutes to send the first set of stats as the server can diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index 6eeed976708..b4de759b675 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -83,7 +83,7 @@ from synapse.logging.context import run_in_background from synapse.storage.databases.main import DataStore from synapse.types import DeviceListUpdates, JsonMapping -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock +from synapse.util.clock import Clock if TYPE_CHECKING: from synapse.server import HomeServer @@ -518,10 +518,6 @@ def recover(self) -> None: self.hs.run_as_background_process, "as-recoverer", self.retry, - # Only track this call if it would delay shutdown by a substantial amount - call_later_cancel_on_shutdown=True - if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) def _backoff(self) -> None: diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 73c466a60e9..79dd3e84165 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -43,7 +43,6 @@ UserID, create_requester, ) -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from synapse.util.events import generate_fake_event_id from synapse.util.metrics import Measure from synapse.util.sentinel import Sentinel @@ -96,7 +95,6 @@ async def _schedule_db_events() -> None: self._clock.call_later( 0, self.notify_new_event, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Delayed events that are already marked as processed on startup might not have been @@ -545,10 +543,6 @@ def _schedule_next_at(self, next_send_ts: Timestamp) -> None: self.hs.run_as_background_process, "_send_on_timeout", self._send_on_timeout, - # Only track this call if it would delay shutdown by a substantial amount - call_later_cancel_on_shutdown=True - if delay_sec > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) else: self._next_delayed_event_call.reset(delay_sec) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 66ff3a472aa..e874b600008 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -83,7 +83,6 @@ from synapse.util import log_failure, unwrapFirstError from synapse.util.async_helpers import Linearizer, gather_results from synapse.util.caches.expiringcache import ExpiringCache -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from synapse.util.json import json_decoder, json_encoder from synapse.util.metrics import measure_func from synapse.visibility import get_effective_room_visibility_from_state @@ -449,10 +448,6 @@ def _schedule_expiry_for_event(self, event_id: str, expiry_ts: int) -> None: "_expire_event", self._expire_event, event_id, - # Only track this call if it would delay shutdown by a substantial amount - call_later_cancel_on_shutdown=True - if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) async def _expire_event(self, event_id: str) -> None: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 7656abfaab7..4e6055d0972 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -879,7 +879,6 @@ def __init__(self, hs: "HomeServer"): self.clock.looping_call, self._handle_timeouts, 5000, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Presence information is persisted, whether or not it is being tracked @@ -890,7 +889,6 @@ def __init__(self, hs: "HomeServer"): self.clock.looping_call, self._persist_unpersisted_changes, 60 * 1000, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) presence_wheel_timer_size_gauge.register_hook( diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 2c94ec600b9..2ab9b70f8c5 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -2192,7 +2192,6 @@ def __init__(self, hs: "HomeServer"): self._clock.call_later( 0, self.notify_new_event, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def notify_new_event(self) -> None: diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index 5221af725d8..5b4a2cc62dc 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -77,7 +77,6 @@ def __init__(self, hs: "HomeServer"): self.clock.call_later( 0, self.notify_new_event, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def notify_new_event(self) -> None: diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index d467f1c547e..28961f5925f 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -40,7 +40,6 @@ from synapse.storage.databases.main.user_directory import SearchResult from synapse.storage.roommember import ProfileInfo from synapse.types import UserID -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from synapse.util.metrics import Measure from synapse.util.retryutils import NotRetryingDestination from synapse.util.stringutils import non_null_str_or_none @@ -140,14 +139,12 @@ def __init__(self, hs: "HomeServer"): self.clock.call_later( 0, self.notify_new_event, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Kick off the profile refresh process on startup self._refresh_remote_profiles_call_later = self.clock.call_later( 10, self.kick_off_remote_profile_refresh_process, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) async def search_users( @@ -564,13 +561,11 @@ async def _handle_possible_remote_profile_change( USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, self.kick_off_remote_profile_refresh_process_for_remote_server, UserID.from_string(user_id).domain, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # Schedule a wake-up to handle any backoffs that may occur in the future. self.clock.call_later( 2 * USER_DIRECTORY_STALE_REFRESH_TIME_MS // 1000 + 1, self.kick_off_remote_profile_refresh_process, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) return @@ -630,7 +625,6 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, self.kick_off_remote_profile_refresh_process, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) return @@ -666,10 +660,6 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( delay, self.kick_off_remote_profile_refresh_process, - # Only track this call if it would delay shutdown by a substantial amount - call_later_cancel_on_shutdown=True - if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) return @@ -682,7 +672,6 @@ async def _unsafe_refresh_remote_profiles(self) -> None: self._refresh_remote_profiles_call_later = self.clock.call_later( INTERVAL_TO_ADD_MORE_SERVERS_TO_REFRESH_PROFILES, self.kick_off_remote_profile_refresh_process, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def kick_off_remote_profile_refresh_process_for_remote_server( diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index accbcba5517..4eaf9cdf425 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -43,7 +43,7 @@ from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage.databases.main.lock import Lock, LockStore from synapse.util.async_helpers import timeout_deferred -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock +from synapse.util.clock import Clock from synapse.util.constants import ONE_MINUTE_SECONDS if TYPE_CHECKING: @@ -201,7 +201,6 @@ def _wake_all_locks( 0, _wake_all_locks, locks, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) @wrap_as_background_process("_cleanup_locks") @@ -255,10 +254,6 @@ async def __aenter__(self) -> None: await timeout_deferred( deferred=self.deferred, timeout=timeout, - # Only track this call if it would delay shutdown substantially - cancel_on_shutdown=True - if timeout > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, clock=self.clock, ) except Exception: @@ -337,10 +332,6 @@ async def __aenter__(self) -> None: await timeout_deferred( deferred=self.deferred, timeout=timeout, - # Only track this call if it would delay shutdown substantially - cancel_on_shutdown=True - if timeout > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, clock=self.clock, ) except Exception: diff --git a/synapse/http/client.py b/synapse/http/client.py index 7ca9733afc3..370cdc3568b 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -175,7 +175,6 @@ def _scheduler(x: Callable[[], object]) -> IDelayedCall: return clock.call_later( _EPSILON, x, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) return _scheduler @@ -440,7 +439,6 @@ async def request( request_deferred = timeout_deferred( deferred=request_deferred, timeout=60, - cancel_on_shutdown=False, # We don't track this call since it's short clock=self.hs.get_clock(), ) @@ -769,7 +767,6 @@ async def get_file( d = timeout_deferred( deferred=d, timeout=30, - cancel_on_shutdown=False, # We don't track this call since it's short clock=self.hs.get_clock(), ) @@ -967,7 +964,6 @@ async def request( request_deferred = timeout_deferred( deferred=request_deferred, timeout=60, - cancel_on_shutdown=False, # We don't track this call since it's short clock=self.hs.get_clock(), ) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 32d6fb85d22..dc833ae0e4f 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -430,7 +430,6 @@ def func(*args): ... multiple places. """ - # TODO: ensure it has a hs object, not just servername def wrapper( func: Callable[Concatenate[HasServerName, P], Awaitable[Optional[R]]], ) -> Callable[P, "defer.Deferred[Optional[R]]"]: diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 98da43727a9..1484bc8fc01 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -29,7 +29,6 @@ from synapse.push.mailer import Mailer from synapse.push.push_types import EmailReason from synapse.storage.databases.main.event_push_actions import EmailPushAction -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from synapse.util.threepids import validate_email if TYPE_CHECKING: @@ -232,10 +231,6 @@ async def _unsafe_process(self) -> None: self.timed_call = self.hs.get_clock().call_later( delay, self.on_timer, - # Only track this call if it would delay shutdown substantially - call_later_cancel_on_shutdown=True - if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) async def save_last_stream_ordering_and_success( diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 801f157b825..5cac5de8cb4 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -35,7 +35,6 @@ from synapse.push import Pusher, PusherConfig, PusherConfigException from synapse.storage.databases.main.event_push_actions import HttpPushAction from synapse.types import JsonDict, JsonMapping -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from . import push_tools @@ -339,10 +338,6 @@ async def _unsafe_process(self) -> None: self.timed_call = self.hs.get_clock().call_later( self.backoff_delay, self.on_timer, - # Only track backoffs if they would delay shutdown substantially - call_later_cancel_on_shutdown=True - if self.backoff_delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) self.backoff_delay = min( self.backoff_delay * 2, self.MAX_BACKOFF_SEC diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 576619deb8b..f2561bc0c52 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -345,7 +345,6 @@ async def wait_for_stream_position( deferred = timeout_deferred( deferred=deferred, timeout=_WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, - cancel_on_shutdown=False, # We don't track this call since it's short clock=self._clock, ) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index 5bc33b24aa3..f2d24be749b 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -642,7 +642,6 @@ def __init__( "upsert_safety_check", self.server_name, self._check_safe_to_upsert, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def stop_background_updates(self) -> None: @@ -692,7 +691,6 @@ async def _check_safe_to_upsert(self) -> None: "upsert_safety_check", self.server_name, self._check_safe_to_upsert, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) def start_profiling(self) -> None: diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index cf4309ec645..906d1a91f68 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -215,7 +215,6 @@ def __init__( self.clock.call_later( 0.0, self._set_expiration_date_when_missing, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) # If support for MSC3866 is enabled and configured to require approval for new diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index a0f863c9155..65caf4b1eaa 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -122,7 +122,6 @@ def __init__( self.hs.get_clock().call_later( 1, self._count_known_servers, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) federation_known_servers_gauge.register_hook( homeserver_instance_id=hs.get_instance_id(), diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 180d042da25..990de297589 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -63,7 +63,7 @@ run_coroutine_in_background, run_in_background, ) -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S, Clock +from synapse.util.clock import Clock logger = logging.getLogger(__name__) @@ -773,7 +773,7 @@ def timeout_deferred( *, deferred: "defer.Deferred[_T]", timeout: float, - cancel_on_shutdown: bool, + cancel_on_shutdown: Optional[bool] = None, clock: Clock, ) -> "defer.Deferred[_T]": """The in built twisted `Deferred.addTimeout` fails to time out deferreds @@ -798,6 +798,9 @@ def timeout_deferred( tracking since the small delay after shutdown before they trigger is immaterial. It's not worth the overhead to track those calls as it blows up the tracking collection on large server instances. + If this value is `None` then the function will decide whether to track + the call for cleanup by comparing whether the `delay` is longer than + `CALL_LATER_DELAY_TRACKING_THRESHOLD_S`. clock: The `Clock` instance used to track delayed calls. @@ -994,10 +997,6 @@ async def sleep(self, name: str, delay_ms: int) -> None: delay_ms / 1000, sleep_deferred.callback, None, - # Only track this call if it would delay shutdown by a substantial amount - call_later_cancel_on_shutdown=True - if delay_ms / 1000 > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) # Create a deferred that will get called if `wake` is called with @@ -1057,10 +1056,6 @@ async def wait(self, timeout_seconds: float) -> bool: timeout_seconds, sleep_deferred.callback, None, - # Only track this call if it would delay shutdown by a substantial amount - call_later_cancel_on_shutdown=True - if timeout_seconds > CALL_LATER_DELAY_TRACKING_THRESHOLD_S - else False, ) try: diff --git a/synapse/util/clock.py b/synapse/util/clock.py index e421395d7a3..ac3a20341c3 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -19,6 +19,7 @@ Callable, Dict, List, + Optional, ) from typing_extensions import ParamSpec @@ -36,7 +37,7 @@ P = ParamSpec("P") -CALL_LATER_DELAY_TRACKING_THRESHOLD_S = 60 +CALL_LATER_DELAY_TRACKING_THRESHOLD_S = 30 """ The length of time (in seconds) the `delay` in a call to `Clock.call_later` must be before it is tracked for cancellation on shutdown. @@ -233,7 +234,7 @@ def call_later( delay: float, callback: Callable, *args: Any, - call_later_cancel_on_shutdown: bool = True, + call_later_cancel_on_shutdown: Optional[bool] = None, **kwargs: Any, ) -> IDelayedCall: """Call something later @@ -252,6 +253,9 @@ def call_later( tracking since the small delay after shutdown before they trigger is immaterial. It's not worth the overhead to track those calls as it blows up the tracking collection on large server instances. + If this value is `None` then the function will decide whether to track + the call for cleanup by comparing whether the `delay` is longer than + `CALL_LATER_DELAY_TRACKING_THRESHOLD_S`. Placed in between `*args` and `**kwargs` in order to be able to set a default value. **kwargs: Key arguments to pass to function. @@ -302,7 +306,15 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: return wrapped_callback - if call_later_cancel_on_shutdown: + cancel_on_shutdown = False + if call_later_cancel_on_shutdown is None: + cancel_on_shutdown = ( + True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False + ) + else: + cancel_on_shutdown = call_later_cancel_on_shutdown + + if cancel_on_shutdown: call_id = self._delayed_call_id self._delayed_call_id = self._delayed_call_id + 1 diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index 884e1931f65..756677fe6c4 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -422,5 +422,4 @@ def start_next_request() -> None: self.clock.call_later( 0.0, start_next_request, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 2269e344c71..40afdcb102a 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -475,7 +475,6 @@ async def wrapper() -> None: self._clock.call_later( 0.1, self._launch_scheduled_tasks, - call_later_cancel_on_shutdown=False, # We don't track this call since it's short ) if len(self._running_tasks) >= TaskScheduler.MAX_CONCURRENT_RUNNING_TASKS: diff --git a/tests/util/test_async_helpers.py b/tests/util/test_async_helpers.py index 243f7cbe46c..fd8d576aea8 100644 --- a/tests/util/test_async_helpers.py +++ b/tests/util/test_async_helpers.py @@ -167,7 +167,6 @@ def canceller(_d: Deferred) -> None: timing_out_d = timeout_deferred( deferred=non_completing_d, timeout=1.0, - cancel_on_shutdown=False, clock=self.clock, ) @@ -190,7 +189,6 @@ def canceller(_d: Deferred) -> None: timing_out_d = timeout_deferred( deferred=non_completing_d, timeout=1.0, - cancel_on_shutdown=False, clock=self.clock, ) @@ -236,7 +234,6 @@ def mark_was_cancelled(res: Failure) -> None: timing_out_d = timeout_deferred( deferred=incomplete_d, timeout=1.0, - cancel_on_shutdown=False, clock=self.clock, ) self.assertNoResult(timing_out_d) From 8d05718dda732a2a2e79eaddc700b395f86b27b7 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Sat, 27 Sep 2025 21:09:01 -0600 Subject: [PATCH 164/181] Fix appservice tests with wrap_as_background_process changes --- synapse/federation/sender/__init__.py | 7 ++++- synapse/handlers/appservice.py | 2 +- synapse/handlers/typing.py | 2 +- synapse/handlers/worker_lock.py | 5 ++-- synapse/metrics/background_process_metrics.py | 23 ++++++++------- synapse/replication/tcp/redis.py | 5 ++-- synapse/storage/controllers/purge_events.py | 1 + synapse/util/task_scheduler.py | 12 ++++---- tests/handlers/test_appservice.py | 29 ++++++++++++++++++- 9 files changed, 58 insertions(+), 28 deletions(-) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index caac495d921..4410ffc5c56 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -330,6 +330,7 @@ class _DestinationWakeupQueue: _MAX_TIME_IN_QUEUE = 30.0 sender: "FederationSender" = attr.ib() + hs: "HomeServer" = attr.ib() server_name: str = attr.ib() """ Our homeserver name (used to label metrics) (`hs.hostname`). @@ -457,7 +458,11 @@ def __init__(self, hs: "HomeServer"): 1.0 / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second ) self._destination_wakeup_queue = _DestinationWakeupQueue( - self, self.server_name, self.clock, max_delay_s=rr_txn_interval_per_room_s + self, + hs, + self.server_name, + self.clock, + max_delay_s=rr_txn_interval_per_room_s, ) # It is important for `_is_shutdown` to be instantiated before the looping call diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index e6cbdb587c2..575ae940494 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -78,7 +78,7 @@ def __init__(self, hs: "HomeServer"): self.server_name = ( hs.hostname ) # nb must be called this for @wrap_as_background_process - self.hs = hs + self.hs = hs # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main self.is_mine_id = hs.is_mine_id self.appservice_api = hs.get_application_service_api() diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 7e62df4f4c3..296e063f4c5 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -77,7 +77,7 @@ class FollowerTypingHandler: """ def __init__(self, hs: "HomeServer"): - self.hs = hs + self.hs = hs # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.server_name = ( diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index 4eaf9cdf425..ad20376dd40 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -66,9 +66,8 @@ class WorkerLocksHandler: """ def __init__(self, hs: "HomeServer") -> None: - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self._clock = hs.get_clock() self._store = hs.get_datastores().main self._clock = hs.get_clock() diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index dc833ae0e4f..3f400687cd0 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -66,6 +66,8 @@ # Old versions don't have `LiteralString` from typing_extensions import LiteralString + from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -397,11 +399,11 @@ def combined_context_manager() -> Generator[None, None, None]: P = ParamSpec("P") -class HasServerName(Protocol): - server_name: str +class HasHomeServer(Protocol): + hs: "HomeServer" """ - The homeserver name that this cache is associated with (used to label the metric) - (`hs.hostname`). + The homeserver that this cache is associated with (used to label the metric and + track backgroun processes for clean shutdown). """ @@ -431,19 +433,18 @@ def func(*args): ... """ def wrapper( - func: Callable[Concatenate[HasServerName, P], Awaitable[Optional[R]]], + func: Callable[Concatenate[HasHomeServer, P], Awaitable[Optional[R]]], ) -> Callable[P, "defer.Deferred[Optional[R]]"]: @wraps(func) def wrapped_func( - self: HasServerName, *args: P.args, **kwargs: P.kwargs + self: HasHomeServer, *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": - assert self.server_name is not None, ( - "The `server_name` attribute must be set on the object where `@wrap_as_background_process` decorator is used." + assert self.hs is not None, ( + "The `hs` attribute must be set on the object where `@wrap_as_background_process` decorator is used." ) - return run_as_background_process( # type: ignore[untracked-background-process] + return self.hs.run_as_background_process( desc, - self.server_name, func, self, *args, @@ -451,7 +452,7 @@ def wrapped_func( # Argument 4 to "run_as_background_process" has incompatible type # "**P.kwargs"; expected "bool" # See https://github.com/python/mypy/issues/8862 - **kwargs, # type: ignore[arg-type] + **kwargs, ) # There are some shenanigans here, because we're decorating a method but diff --git a/synapse/replication/tcp/redis.py b/synapse/replication/tcp/redis.py index 7f013c56566..caffb2913ea 100644 --- a/synapse/replication/tcp/redis.py +++ b/synapse/replication/tcp/redis.py @@ -314,9 +314,8 @@ def __init__( convertNumbers=convertNumbers, ) - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname hs.get_clock().looping_call(self._send_ping, 30 * 1000) diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 14b37ac543b..9a1f1b53062 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -46,6 +46,7 @@ class PurgeEventsStorageController: """High level interface for purging rooms and event history.""" def __init__(self, hs: "HomeServer", stores: Databases): + self.hs = hs # nb must be called this for @wrap_as_background_process self.server_name = ( hs.hostname ) # nb must be called this for @wrap_as_background_process diff --git a/synapse/util/task_scheduler.py b/synapse/util/task_scheduler.py index 40afdcb102a..7443d4e097e 100644 --- a/synapse/util/task_scheduler.py +++ b/synapse/util/task_scheduler.py @@ -106,10 +106,8 @@ class TaskScheduler: OCCASIONAL_REPORT_INTERVAL_MS = 5 * 60 * 1000 # 5 minutes def __init__(self, hs: "HomeServer"): - self._hs = hs - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self._store = hs.get_datastores().main self._clock = hs.get_clock() self._running_tasks: Set[str] = set() @@ -214,7 +212,7 @@ async def schedule_task( if self._run_background_tasks: self._launch_scheduled_tasks() else: - self._hs.get_replication_command_handler().send_new_active_task(task.id) + self.hs.get_replication_command_handler().send_new_active_task(task.id) return task.id @@ -361,7 +359,7 @@ async def inner() -> None: finally: self._launching_new_tasks = False - self._hs.run_as_background_process("launch_scheduled_tasks", inner) + self.hs.run_as_background_process("launch_scheduled_tasks", inner) @wrap_as_background_process("clean_scheduled_tasks") async def _clean_scheduled_tasks(self) -> None: @@ -495,4 +493,4 @@ async def wrapper() -> None: self._running_tasks.add(task.id) await self.update_task(task.id, status=TaskStatus.ACTIVE) - self._hs.run_as_background_process(f"task-{task.action}", wrapper) + self.hs.run_as_background_process(f"task-{task.action}", wrapper) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 999d7f5e6cd..b9e204c5d08 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -19,7 +19,17 @@ # # -from typing import Dict, Iterable, List, Optional +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Dict, + Iterable, + List, + Optional, + TypeVar, +) from unittest.mock import AsyncMock, Mock from parameterized import parameterized @@ -36,6 +46,7 @@ TransactionUnusedFallbackKeys, ) from synapse.handlers.appservice import ApplicationServicesHandler +from synapse.metrics.background_process_metrics import run_as_background_process from synapse.rest.client import login, receipts, register, room, sendtodevice from synapse.server import HomeServer from synapse.types import ( @@ -53,6 +64,11 @@ from tests.unittest import override_config from tests.utils import MockClock +if TYPE_CHECKING: + from typing_extensions import LiteralString + +R = TypeVar("R") + class AppServiceHandlerTestCase(unittest.TestCase): """Tests the ApplicationServicesHandler.""" @@ -62,6 +78,17 @@ def setUp(self) -> None: self.mock_as_api = AsyncMock() self.mock_scheduler = Mock() hs = Mock() + + def test_run_as_background_process( + desc: "LiteralString", + func: Callable[..., Awaitable[Optional[R]]], + *args: Any, + **kwargs: Any, + ) -> "defer.Deferred[Optional[R]]": + # Ignore linter error as this is used only for testing purposes. + return run_as_background_process(desc, "test_server", func, *args, **kwargs) # type: ignore[untracked-background-process] + + hs.run_as_background_process = test_run_as_background_process hs.get_datastores.return_value = Mock(main=self.mock_store) self.mock_store.get_appservice_last_pos = AsyncMock(return_value=None) self.mock_store.set_appservice_last_pos = AsyncMock(return_value=None) From 751cacd34a90e5f586842fd5fe49a5f58ad79b7e Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Sun, 28 Sep 2025 08:43:32 -0600 Subject: [PATCH 165/181] Fix registered shutdown handlers --- synapse/app/_base.py | 1 - synapse/handlers/presence.py | 3 +-- synapse/server.py | 15 ++++----------- synapse/storage/databases/main/client_ips.py | 1 - synapse/storage/databases/main/lock.py | 1 - synapse/util/async_helpers.py | 2 -- 6 files changed, 5 insertions(+), 18 deletions(-) diff --git a/synapse/app/_base.py b/synapse/app/_base.py index d4ec57f4db4..655f684ecf0 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -653,7 +653,6 @@ def log_shutdown() -> None: hs.register_sync_shutdown_handler( phase="before", eventType="shutdown", - desc="shutdown log entry", shutdown_func=log_shutdown, ) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 4e6055d0972..401cde2fc69 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -541,7 +541,6 @@ def __init__(self, hs: "HomeServer"): hs.register_async_shutdown_handler( phase="before", eventType="shutdown", - desc="generic_presence.on_shutdown", shutdown_func=self._on_shutdown, ) @@ -841,7 +840,6 @@ def __init__(self, hs: "HomeServer"): hs.register_async_shutdown_handler( phase="before", eventType="shutdown", - desc="presence.on_shutdown", shutdown_func=self._on_shutdown, ) @@ -905,6 +903,7 @@ def __init__(self, hs: "HomeServer"): self._event_pos = self.store.get_room_max_stream_ordering() self._event_processing = False + @wrap_as_background_process("PresenceHandler._on_shutdown") async def _on_shutdown(self) -> None: """Gets called when shutting down. This lets us persist any updates that we haven't yet persisted, e.g. updates that only changes some internal diff --git a/synapse/server.py b/synapse/server.py index a1e65a06600..cc0d3a427b4 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -268,14 +268,12 @@ class ShutdownInfo: """Information for callable functions called at time of shutdown. Attributes: - desc: a short description of the event trigger. func: the object to call before shutdown. trigger_id: an ID returned when registering this event trigger. args: the arguments to call the function with. kwargs: the keyword arguments to call the function with. """ - desc: str func: Callable[..., Any] trigger_id: _SystemEventID kwargs: Dict[str, object] @@ -525,7 +523,6 @@ def register_async_shutdown_handler( *, phase: str, eventType: str, - desc: "LiteralString", shutdown_func: Callable[..., Any], **kwargs: object, ) -> None: @@ -536,13 +533,11 @@ def register_async_shutdown_handler( id = self.get_clock().add_system_event_trigger( phase, eventType, - self.run_as_background_process, - desc, shutdown_func, - kwargs, + **kwargs, ) self._async_shutdown_handlers.append( - ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, kwargs=kwargs) + ShutdownInfo(func=shutdown_func, trigger_id=id, kwargs=kwargs) ) def register_sync_shutdown_handler( @@ -550,7 +545,6 @@ def register_sync_shutdown_handler( *, phase: str, eventType: str, - desc: "LiteralString", shutdown_func: Callable[..., Any], **kwargs: object, ) -> None: @@ -562,10 +556,10 @@ def register_sync_shutdown_handler( phase, eventType, shutdown_func, - kwargs, + **kwargs, ) self._sync_shutdown_handlers.append( - ShutdownInfo(desc=desc, func=shutdown_func, trigger_id=id, kwargs=kwargs) + ShutdownInfo(func=shutdown_func, trigger_id=id, kwargs=kwargs) ) def register_module_web_resource(self, path: str, resource: Resource) -> None: @@ -1253,7 +1247,6 @@ def get_media_sender_thread_pool(self) -> ThreadPool: self.register_sync_shutdown_handler( phase="during", eventType="shutdown", - desc="Homeserver media_threadpool.stop", shutdown_func=media_threadpool.stop, ) diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index c9522cef626..dc6ab99a6c7 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -457,7 +457,6 @@ def __init__( hs.register_async_shutdown_handler( phase="before", eventType="shutdown", - desc="ClientIpWorkerStore _update_client_ips_batch", shutdown_func=self._update_client_ips_batch, ) diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index c3e1c8aa4ed..e2b15eaf6a5 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -101,7 +101,6 @@ def __init__( hs.register_async_shutdown_handler( phase="before", eventType="shutdown", - desc="LockStore _on_shutdown", shutdown_func=self._on_shutdown, ) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 990de297589..6daa5d027b2 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -815,9 +815,7 @@ def time_it_out() -> None: timed_out[0] = True try: - logger.error("Cancelling deferred in timeout") deferred.cancel() - logger.error("Cancelled deferred in timeout") except Exception: # if we throw any exception it'll break time outs logger.exception("Canceller failed during timeout") From 54d2fe4d90515d65e76e98d6e510924160f814b7 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 11:54:53 -0600 Subject: [PATCH 166/181] Address mypy plugin review comments --- scripts-dev/mypy_synapse_plugin.py | 84 ++++++++++++++++-------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 49008ecff22..5a336e6d791 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -68,7 +68,7 @@ category="per-homeserver-tenant-metrics", ) -INTERNAL_CLOCK_CALL_LATER_NOT_USED = ErrorCode( +PREFER_SYNAPSE_CLOCK_CALL_LATER = ErrorCode( "call-later-not-tracked", "`synapse.util.Clock.call_later` should be used instead of `reactor.callLater`", category="synapse-reactor-clock", @@ -100,7 +100,7 @@ UNTRACKED_BACKGROUND_PROCESS = ErrorCode( "untracked-background-process", - "All calls to `run_as_background_process` should use the `HomeServer` method", + "Prefer using `HomeServer.run_as_background_process` method over the bare `run_as_background_process`", category="synapse-tracked-calls", ) @@ -301,6 +301,27 @@ def get_method_signature_hook( return None +def check_clock_creation(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that the only `clock.Clock` instance is the one used by the `HomeServer`. + This is so that the `HomeServer` can cancel any tracked delayed or looping calls + during server shutdown. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Expected the only `clock.Clock` instance to be the one used by the `HomeServer`. " + "This is so that the `HomeServer` can cancel any tracked delayed or looping calls " + "during server shutdown", + ctx.context, + code=MULTIPLE_INTERNAL_CLOCKS_CREATED, + ) + + return signature + + def check_call_later(ctx: MethodSigContext) -> CallableType: """ Ensure that the `reactor.callLater` callsites aren't used. @@ -352,44 +373,6 @@ def check_looping_call(ctx: FunctionSigContext) -> CallableType: return signature -def check_clock_creation(ctx: FunctionSigContext) -> CallableType: - """ - Ensure that the only `clock.Clock` is the one used by the `HomeServer`. - - Args: - ctx: The `FunctionSigContext` from mypy. - """ - signature: CallableType = ctx.default_signature - ctx.api.fail( - "Expected the only `clock.Clock` instance to be the one used by the `HomeServer`. " - "This is so that the `HomeServer` can cancel any tracked delayed or looping calls " - "during server shutdown", - ctx.context, - code=MULTIPLE_INTERNAL_CLOCKS_CREATED, - ) - - return signature - - -def check_background_process(ctx: FunctionSigContext) -> CallableType: - """ - Ensure that calls to `run_as_background_process` use the `HomeServer` method. - - Args: - ctx: The `FunctionSigContext` from mypy. - """ - signature: CallableType = ctx.default_signature - ctx.api.fail( - "Expected all calls to `run_as_background_process` to use the `HomeServer` method. " - "This is so that the `HomeServer` can cancel any background processes " - "during server shutdown", - ctx.context, - code=UNTRACKED_BACKGROUND_PROCESS, - ) - - return signature - - def check_call_when_running(ctx: MethodSigContext) -> CallableType: """ Ensure that the `reactor.callWhenRunning` callsites aren't used. @@ -444,6 +427,27 @@ def check_add_system_event_trigger(ctx: MethodSigContext) -> CallableType: return signature +def check_background_process(ctx: FunctionSigContext) -> CallableType: + """ + Ensure that calls to `run_as_background_process` use the `HomeServer` method. + This is so that the `HomeServer` can cancel any running background processes during + server shutdown. + + Args: + ctx: The `FunctionSigContext` from mypy. + """ + signature: CallableType = ctx.default_signature + ctx.api.fail( + "Prefer using `HomeServer.run_as_background_process` method over the bare " + "`run_as_background_process`. This is so that the `HomeServer` can cancel " + "any background processes during server shutdown", + ctx.context, + code=UNTRACKED_BACKGROUND_PROCESS, + ) + + return signature + + def analyze_prometheus_metric_classes(ctx: ClassDefContext) -> None: """ Cross-check the list of Prometheus metric classes against the From 89a133e03b3219acf5afdf39348d0ff1c198a6b3 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 11:56:30 -0600 Subject: [PATCH 167/181] Update lint ignore comment --- tests/handlers/test_appservice.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index b9e204c5d08..25a39904976 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -85,7 +85,7 @@ def test_run_as_background_process( *args: Any, **kwargs: Any, ) -> "defer.Deferred[Optional[R]]": - # Ignore linter error as this is used only for testing purposes. + # Ignore linter error as this is used only for testing purposes (i.e. outside of Synapse). return run_as_background_process(desc, "test_server", func, *args, **kwargs) # type: ignore[untracked-background-process] hs.run_as_background_process = test_run_as_background_process From baa066c58861ad4f509914b0b6314eac6070abd1 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 11:59:02 -0600 Subject: [PATCH 168/181] Apply wording to other mypy lint codes --- scripts-dev/mypy_synapse_plugin.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts-dev/mypy_synapse_plugin.py b/scripts-dev/mypy_synapse_plugin.py index 5a336e6d791..0b854cdba52 100644 --- a/scripts-dev/mypy_synapse_plugin.py +++ b/scripts-dev/mypy_synapse_plugin.py @@ -70,25 +70,25 @@ PREFER_SYNAPSE_CLOCK_CALL_LATER = ErrorCode( "call-later-not-tracked", - "`synapse.util.Clock.call_later` should be used instead of `reactor.callLater`", + "Prefer using `synapse.util.Clock.call_later` instead of `reactor.callLater`", category="synapse-reactor-clock", ) PREFER_SYNAPSE_CLOCK_LOOPING_CALL = ErrorCode( "prefer-synapse-clock-looping-call", - "`synapse.util.Clock.looping_call` should be used instead of `task.LoopingCall`", + "Prefer using `synapse.util.Clock.looping_call` instead of `task.LoopingCall`", category="synapse-reactor-clock", ) PREFER_SYNAPSE_CLOCK_CALL_WHEN_RUNNING = ErrorCode( "prefer-synapse-clock-call-when-running", - "`synapse.util.Clock.call_when_running` should be used instead of `reactor.callWhenRunning`", + "Prefer using `synapse.util.Clock.call_when_running` instead of `reactor.callWhenRunning`", category="synapse-reactor-clock", ) PREFER_SYNAPSE_CLOCK_ADD_SYSTEM_EVENT_TRIGGER = ErrorCode( "prefer-synapse-clock-add-system-event-trigger", - "`synapse.util.Clock.add_system_event_trigger` should be used instead of `reactor.addSystemEventTrigger`", + "Prefer using `synapse.util.Clock.add_system_event_trigger` instead of `reactor.addSystemEventTrigger`", category="synapse-reactor-clock", ) @@ -344,7 +344,7 @@ def check_call_later(ctx: MethodSigContext) -> CallableType: "instead. This is so that long lived calls can be tracked for cancellation during " "server shutdown", ctx.context, - code=INTERNAL_CLOCK_CALL_LATER_NOT_USED, + code=PREFER_SYNAPSE_CLOCK_CALL_LATER, ) return signature From 46e112b238f8bbb7c81fd814310645cebf2e2d8b Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 12:00:49 -0600 Subject: [PATCH 169/181] Make test comments clearer --- tests/app/test_homeserver_shutdown.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index ed592774152..5db96dfb03f 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -60,8 +60,8 @@ def test_clean_homeserver_shutdown(self) -> None: # Run the reactor so any `callWhenRunning` functions can be cleared out. self.reactor.run() - # Cleanup the registered system event triggers since the `MemoryReactor` doesn't - # do this on it's own. + # This would normally happen as part of `HomeServer.shutdown` but the `MemoryReactor` + # we use in tests doesn't handle this properly (see doc comment) cleanup_test_reactor_system_event_triggers(self.reactor) # Cleanup the homeserver. @@ -139,8 +139,8 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # Run the reactor so any `callWhenRunning` functions can be cleared out. self.reactor.run() - # Cleanup the registered system event triggers since the `MemoryReactor` doesn't - # do this on it's own. + # This would normally happen as part of `HomeServer.shutdown` but the `MemoryReactor` + # we use in tests doesn't handle this properly (see doc comment) cleanup_test_reactor_system_event_triggers(self.reactor) # Also advance the reactor by the delay tracking threshold to ensure all From f2f0baea84ea9c7bb0a0e92b62dfd6171425bb48 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 12:02:33 -0600 Subject: [PATCH 170/181] Address synmark review comments --- synmark/suites/logging.py | 2 +- synmark/suites/lrucache.py | 2 ++ synmark/suites/lrucache_evict.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/synmark/suites/logging.py b/synmark/suites/logging.py index ccfe506a0ca..cf9c836e062 100644 --- a/synmark/suites/logging.py +++ b/synmark/suites/logging.py @@ -87,7 +87,7 @@ class _logging: # To be able to sleep. # Ignore linter error here since we are running outside of the context of a - # `HomeServer`. + # Synapse `HomeServer`. clock = Clock(reactor, server_name=hs_config.server.server_name) # type: ignore[multiple-internal-clocks] errors = StringIO() diff --git a/synmark/suites/lrucache.py b/synmark/suites/lrucache.py index 425644ec623..830a3daa8fc 100644 --- a/synmark/suites/lrucache.py +++ b/synmark/suites/lrucache.py @@ -30,6 +30,8 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: """ Benchmark `loops` number of insertions into LruCache without eviction. """ + # Ignore linter error here since we are running outside of the context of a + # Synapse `HomeServer`. cache: LruCache[int, bool] = LruCache( max_size=loops, clock=Clock(reactor, server_name="synmark_benchmark"), # type: ignore[multiple-internal-clocks] diff --git a/synmark/suites/lrucache_evict.py b/synmark/suites/lrucache_evict.py index f7a6701d8c4..c67e0c90017 100644 --- a/synmark/suites/lrucache_evict.py +++ b/synmark/suites/lrucache_evict.py @@ -31,6 +31,8 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: Benchmark `loops` number of insertions into LruCache where half of them are evicted. """ + # Ignore linter error here since we are running outside of the context of a + # Synapse `HomeServer`. cache: LruCache[int, bool] = LruCache( max_size=loops // 2, clock=Clock(reactor, server_name="synmark_benchmark"), # type: ignore[multiple-internal-clocks] From 3ca4e98e781a65673525e33317cb4eadf2cdedb3 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 12:04:34 -0600 Subject: [PATCH 171/181] Remove unnecessary comment --- synapse/metrics/background_process_metrics.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 3f400687cd0..6dc2cbe1322 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -448,10 +448,6 @@ def wrapped_func( func, self, *args, - # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. - # Argument 4 to "run_as_background_process" has incompatible type - # "**P.kwargs"; expected "bool" - # See https://github.com/python/mypy/issues/8862 **kwargs, ) From 3e92bf70d42f8f926f18a29bb1c783a9c1d74155 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 12:32:04 -0600 Subject: [PATCH 172/181] Remove unnecessary comments --- synapse/handlers/account_validity.py | 6 ++---- synapse/handlers/appservice.py | 4 +--- synapse/handlers/presence.py | 4 +--- synapse/handlers/typing.py | 4 +--- synapse/handlers/worker_lock.py | 1 - synapse/push/pusherpool.py | 6 ++---- synapse/storage/controllers/purge_events.py | 4 +--- 7 files changed, 8 insertions(+), 21 deletions(-) diff --git a/synapse/handlers/account_validity.py b/synapse/handlers/account_validity.py index 39a22b8cbb1..eed50ef69a7 100644 --- a/synapse/handlers/account_validity.py +++ b/synapse/handlers/account_validity.py @@ -37,10 +37,8 @@ class AccountValidityHandler: def __init__(self, hs: "HomeServer"): - self.hs = hs - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.config = hs.config self.store = hs.get_datastores().main self.send_email_handler = hs.get_send_email_handler() diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 575ae940494..6536d9fe510 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -75,9 +75,7 @@ class ApplicationServicesHandler: def __init__(self, hs: "HomeServer"): - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.hs = hs # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main self.is_mine_id = hs.is_mine_id diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 401cde2fc69..1610683066a 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -774,9 +774,7 @@ async def bump_presence_active_time( class PresenceHandler(BasePresenceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.wheel_timer: WheelTimer[str] = WheelTimer() self.notifier = hs.get_notifier() diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 296e063f4c5..77c5b747c3f 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -80,9 +80,7 @@ def __init__(self, hs: "HomeServer"): self.hs = hs # nb must be called this for @wrap_as_background_process self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.clock = hs.get_clock() self.is_mine_id = hs.is_mine_id self.is_mine_server_name = hs.is_mine_server_name diff --git a/synapse/handlers/worker_lock.py b/synapse/handlers/worker_lock.py index ad20376dd40..ca1e2b166c3 100644 --- a/synapse/handlers/worker_lock.py +++ b/synapse/handlers/worker_lock.py @@ -67,7 +67,6 @@ class WorkerLocksHandler: def __init__(self, hs: "HomeServer") -> None: self.hs = hs # nb must be called this for @wrap_as_background_process - self.server_name = hs.hostname self._clock = hs.get_clock() self._store = hs.get_datastores().main self._clock = hs.get_clock() diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py index 39e7530751b..977c55b6836 100644 --- a/synapse/push/pusherpool.py +++ b/synapse/push/pusherpool.py @@ -69,10 +69,8 @@ class PusherPool: """ def __init__(self, hs: "HomeServer"): - self.hs = hs - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.hs = hs # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.pusher_factory = PusherFactory(hs) self.store = self.hs.get_datastores().main self.clock = self.hs.get_clock() diff --git a/synapse/storage/controllers/purge_events.py b/synapse/storage/controllers/purge_events.py index 9a1f1b53062..ded9cb0567e 100644 --- a/synapse/storage/controllers/purge_events.py +++ b/synapse/storage/controllers/purge_events.py @@ -47,9 +47,7 @@ class PurgeEventsStorageController: def __init__(self, hs: "HomeServer", stores: Databases): self.hs = hs # nb must be called this for @wrap_as_background_process - self.server_name = ( - hs.hostname - ) # nb must be called this for @wrap_as_background_process + self.server_name = hs.hostname self.stores = stores if hs.config.worker.run_background_tasks: From f39e2125a99f6cf4a7700a91f8e439c86eee9da6 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 12:35:06 -0600 Subject: [PATCH 173/181] Don't prevent module_api from cancelling background processes on shutdown --- synapse/module_api/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 65c07dd06da..12a31dd2abb 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -1471,7 +1471,6 @@ def delayed_background_call( self._hs.run_as_background_process, desc, lambda: maybe_awaitable(f(*args, **kwargs)), - call_later_cancel_on_shutdown=False, # We don't track calls in the module api since we don't know their purpose ) async def sleep(self, seconds: float) -> None: From d47839ee2d9821e9b3292f4be2607831c80b87b8 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Mon, 29 Sep 2025 16:56:13 -0600 Subject: [PATCH 174/181] Realy pull back untracked delayed calls --- synapse/http/matrixfederationclient.py | 3 --- synapse/http/proxy.py | 1 - synapse/util/async_helpers.py | 15 +++++------ synapse/util/clock.py | 37 ++++++-------------------- tests/app/test_homeserver_shutdown.py | 11 -------- 5 files changed, 14 insertions(+), 53 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 617d6fd57d8..4d72c72d018 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -304,7 +304,6 @@ async def _handle_response( d = timeout_deferred( deferred=d, timeout=timeout_sec, - cancel_on_shutdown=False, # We don't track this call since it's short clock=clock, ) @@ -748,7 +747,6 @@ async def _send_request( request_deferred = timeout_deferred( deferred=request_deferred, timeout=_sec_timeout, - cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self.clock, ) @@ -809,7 +807,6 @@ async def _send_request( d = timeout_deferred( deferred=d, timeout=_sec_timeout, - cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self.clock, ) diff --git a/synapse/http/proxy.py b/synapse/http/proxy.py index c8e6d414421..fa17432984a 100644 --- a/synapse/http/proxy.py +++ b/synapse/http/proxy.py @@ -166,7 +166,6 @@ async def _async_render(self, request: "SynapseRequest") -> Tuple[int, Any]: # so that it has enough time to complete and pass us the data before we give # up. timeout=90, - cancel_on_shutdown=False, # We don't track this call since it will typically be short clock=self._clock, ) diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index 6daa5d027b2..2a167f209cb 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -773,7 +773,7 @@ def timeout_deferred( *, deferred: "defer.Deferred[_T]", timeout: float, - cancel_on_shutdown: Optional[bool] = None, + cancel_on_shutdown: bool = True, clock: Clock, ) -> "defer.Deferred[_T]": """The in built twisted `Deferred.addTimeout` fails to time out deferreds @@ -793,14 +793,11 @@ def timeout_deferred( deferred: The Deferred to potentially timeout. timeout: Timeout in seconds cancel_on_shutdown: Whether this call should be tracked for cleanup during - shutdown. Any call with a long delay, or that is created infrequently, - should be tracked. Calls which are short or of 0 delay don't require - tracking since the small delay after shutdown before they trigger is - immaterial. It's not worth the overhead to track those calls as it blows up - the tracking collection on large server instances. - If this value is `None` then the function will decide whether to track - the call for cleanup by comparing whether the `delay` is longer than - `CALL_LATER_DELAY_TRACKING_THRESHOLD_S`. + shutdown. In general, all calls should be tracked. There may be a use case + not to track calls with a `timeout` of 0 (or similarly short) since tracking + them may result in rapid insertions and removals of tracked calls + unnecessarily. But unless a specific instance of tracking proves to be an + issue, we can just track all delayed calls. clock: The `Clock` instance used to track delayed calls. diff --git a/synapse/util/clock.py b/synapse/util/clock.py index ac3a20341c3..5e65cf32a4b 100644 --- a/synapse/util/clock.py +++ b/synapse/util/clock.py @@ -19,7 +19,6 @@ Callable, Dict, List, - Optional, ) from typing_extensions import ParamSpec @@ -37,13 +36,6 @@ P = ParamSpec("P") -CALL_LATER_DELAY_TRACKING_THRESHOLD_S = 30 -""" -The length of time (in seconds) the `delay` in a call to `Clock.call_later` must be -before it is tracked for cancellation on shutdown. -""" - - class Clock: """ A Clock wraps a Twisted reactor and provides utilities on top of it. @@ -234,7 +226,7 @@ def call_later( delay: float, callback: Callable, *args: Any, - call_later_cancel_on_shutdown: Optional[bool] = None, + call_later_cancel_on_shutdown: bool = True, **kwargs: Any, ) -> IDelayedCall: """Call something later @@ -247,17 +239,12 @@ def call_later( delay: How long to wait in seconds. callback: Function to call *args: Postional arguments to pass to function. - call_cater_cancel_on_shutdown: Whether this call should be tracked for cleanup during - shutdown. Any call with a long delay, or that is created infrequently, - should be tracked. Calls which are short or of 0 delay don't require - tracking since the small delay after shutdown before they trigger is - immaterial. It's not worth the overhead to track those calls as it blows up - the tracking collection on large server instances. - If this value is `None` then the function will decide whether to track - the call for cleanup by comparing whether the `delay` is longer than - `CALL_LATER_DELAY_TRACKING_THRESHOLD_S`. - Placed in between `*args` and `**kwargs` in order to be able to set a - default value. + call_later_cancel_on_shutdown: Whether this call should be tracked for cleanup during + shutdown. In general, all calls should be tracked. There may be a use case + not to track calls with a `timeout` of 0 (or similarly short) since tracking + them may result in rapid insertions and removals of tracked calls + unnecessarily. But unless a specific instance of tracking proves to be an + issue, we can just track all delayed calls. **kwargs: Key arguments to pass to function. """ @@ -306,15 +293,7 @@ def wrapped_callback(*args: Any, **kwargs: Any) -> None: return wrapped_callback - cancel_on_shutdown = False - if call_later_cancel_on_shutdown is None: - cancel_on_shutdown = ( - True if delay > CALL_LATER_DELAY_TRACKING_THRESHOLD_S else False - ) - else: - cancel_on_shutdown = call_later_cancel_on_shutdown - - if cancel_on_shutdown: + if call_later_cancel_on_shutdown: call_id = self._delayed_call_id self._delayed_call_id = self._delayed_call_id + 1 diff --git a/tests/app/test_homeserver_shutdown.py b/tests/app/test_homeserver_shutdown.py index 5db96dfb03f..cf1f06244db 100644 --- a/tests/app/test_homeserver_shutdown.py +++ b/tests/app/test_homeserver_shutdown.py @@ -23,7 +23,6 @@ from synapse.app.homeserver import SynapseHomeServer from synapse.storage.background_updates import UpdaterStatus -from synapse.util.clock import CALL_LATER_DELAY_TRACKING_THRESHOLD_S from tests.server import ( cleanup_test_reactor_system_event_triggers, @@ -70,10 +69,6 @@ def test_clean_homeserver_shutdown(self) -> None: # Cleanup the internal reference in our test case del self.hs - # Advance the reactor to allow for any outstanding calls to be run. - # A value of 1 is not enough, so a value of 2 is used. - self.reactor.advance(2) - # Force garbage collection. gc.collect() @@ -143,12 +138,6 @@ def test_clean_homeserver_shutdown_mid_background_updates(self) -> None: # we use in tests doesn't handle this properly (see doc comment) cleanup_test_reactor_system_event_triggers(self.reactor) - # Also advance the reactor by the delay tracking threshold to ensure all - # cancellable delayed calls have been scheduled. Must be done prior to - # `hs.shutdown()` otherwise they will be scheduled later during the test when we - # advance the reactor to wait out any non-tracked delayed calls. - self.reactor.advance(CALL_LATER_DELAY_TRACKING_THRESHOLD_S) - # Ensure the background updates are not complete. self.assertNotEqual(store.db_pool.updates.get_status(), UpdaterStatus.COMPLETE) From 6978f57a86148dec3324826930e04f79f223375a Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 30 Sep 2025 08:49:12 -0600 Subject: [PATCH 175/181] Add requirement of self.hs comment to device handlers --- synapse/handlers/device.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 39e2104899d..6e2b33bf262 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -124,7 +124,7 @@ class DeviceHandler: def __init__(self, hs: "HomeServer"): self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func - self.hs = hs + self.hs = hs # nb must be called this for @wrap_as_background_process self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() From 40ae3aef6ec951b4541c35805e305d36abdfb748 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 30 Sep 2025 10:04:32 -0600 Subject: [PATCH 176/181] Remove unnecessary assignment --- synapse/handlers/device.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 6e2b33bf262..1e5c180504a 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -962,9 +962,6 @@ class DeviceWriterHandler(DeviceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.server_name = ( - hs.hostname - ) # nb must be called this for @measure_func and @wrap_as_background_process # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by # `synapse.app.generic_worker.FederationSenderHandler` when it sees it From 8063d9fd0f49b739901f4f0faa18269a6696e1cc Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 30 Sep 2025 10:23:33 -0600 Subject: [PATCH 177/181] Shuffle variables --- synapse/handlers/device.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 1e5c180504a..c1b02c76cda 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -122,7 +122,6 @@ class DeviceHandler: store: "GenericWorkerStore" def __init__(self, hs: "HomeServer"): - self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self.hs = hs # nb must be called this for @wrap_as_background_process self.store = cast("GenericWorkerStore", hs.get_datastores().main) @@ -961,6 +960,7 @@ class DeviceWriterHandler(DeviceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname # nb must be called this for @measure_func # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by @@ -1440,7 +1440,6 @@ def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): super().__init__(hs) self.hs = hs - self.server_name = hs.hostname self.federation = hs.get_federation_client() self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func From c00454523902357b6f172cc8ed49f66b02c91528 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 30 Sep 2025 10:26:24 -0600 Subject: [PATCH 178/181] Shuffles --- synapse/handlers/device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index c1b02c76cda..a2e5755707b 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -961,6 +961,7 @@ class DeviceWriterHandler(DeviceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) self.server_name = hs.hostname # nb must be called this for @measure_func + self.hs = hs # nb must be called this for @wrap_as_background_process # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by From 13b940ed5cfdd357ed337deb34a791d94bea57da Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 30 Sep 2025 12:52:00 -0600 Subject: [PATCH 179/181] Remove server_name from run_as_background_process calls --- synapse/storage/database.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index f2d24be749b..fcc910cf45e 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -640,7 +640,6 @@ def __init__( 0.0, run_as_background_process, "upsert_safety_check", - self.server_name, self._check_safe_to_upsert, ) @@ -689,7 +688,6 @@ async def _check_safe_to_upsert(self) -> None: 15.0, self.hs.run_as_background_process, "upsert_safety_check", - self.server_name, self._check_safe_to_upsert, ) From adb63800ff601a7f90b1b2e7316a4af6ef464e72 Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 30 Sep 2025 13:25:28 -0600 Subject: [PATCH 180/181] Oops - use self.hs.run_as_background_process --- synapse/storage/database.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/storage/database.py b/synapse/storage/database.py index fcc910cf45e..a4b2b26795c 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -62,7 +62,6 @@ make_deferred_yieldable, ) from synapse.metrics import SERVER_NAME_LABEL, register_threadpool -from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.background_updates import BackgroundUpdater from synapse.storage.engines import BaseDatabaseEngine, PostgresEngine, Sqlite3Engine from synapse.storage.types import Connection, Cursor, SQLQueryParameters @@ -638,7 +637,7 @@ def __init__( # background updates of tables that aren't safe to update. self._clock.call_later( 0.0, - run_as_background_process, + self.hs.run_as_background_process, "upsert_safety_check", self._check_safe_to_upsert, ) From 699923ad33c9382004b010708eeeb355bfe758fd Mon Sep 17 00:00:00 2001 From: Devon Hudson Date: Tue, 30 Sep 2025 14:02:53 -0600 Subject: [PATCH 181/181] Don't remove necessary variable --- synapse/handlers/device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index a2e5755707b..c6024597b74 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -122,6 +122,7 @@ class DeviceHandler: store: "GenericWorkerStore" def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self.hs = hs # nb must be called this for @wrap_as_background_process self.store = cast("GenericWorkerStore", hs.get_datastores().main)