restart remote-init: retry on another platform if available

[oliver-sanders]

Spotted while reviewing #7379
Note: Lowish-priority, should probs go onto a minor branch

When a workflow is restarted, we re-remote-init all platforms in order to update auth keys and workflow files.

At present, we attempt to remote-init on one platform per install-target as extracted from the task pool.

If remote-init/file-install fails for any host in the platform, we will move onto the next host (provided there is more than one host for the platform). However, if we run out of hosts, we will not move onto the next platform (provided there is more than one platform for the install target in the task pool at restart-time).

This is similar to this comment/issue in cylc clean - #7290 (review)

Here's an untested diff built on #7379 (2026-07-29):

diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py
index 0982c143df..91a834b0d5 100644
--- a/cylc/flow/scheduler.py
+++ b/cylc/flow/scheduler.py
@@ -341,7 +341,7 @@ class Scheduler:
 
         # Map used to track incomplete remote inits for restart
         # {install_target: platform}
-        self.incomplete_ri_map: Dict[str, Dict] = {}
+        self.incomplete_ri_map: Dict[str, list[dict]] = {}
 
     async def install(self):
         """Get the filesystem in the right state to run the flow.
@@ -887,7 +887,7 @@ class Scheduler:
     def restart_remote_init(self):
         """Remote init for all submitted/running tasks in the pool."""
         self.task_job_mgr.task_remote_mgr.is_restart = True
-        distinct_install_target_platforms = []
+        install_target_map = {}
         for itask in self.pool.get_tasks():
             itask.platform['install target'] = (
                 get_install_target_from_platform(itask.platform))
@@ -895,26 +895,20 @@ class Scheduler:
                 # we don't need to remote-init for preparing tasks because
                 # they will be reset to waiting on restart
                 itask.state(*TASK_STATUSES_ACTIVE)
-                and not (
-                    is_platform_with_target_in_list(
-                        itask.platform['install target'],
-                        distinct_install_target_platforms
-                    )
-                )
+
             ):
-                distinct_install_target_platforms.append(itask.platform)
+                install_target_map.setdefault(itask.platform['install target'], set()).add(itask.platform)
 
-        for platform in distinct_install_target_platforms:
+        for install_target, platforms in install_target_map.items():
             # skip remote init for localhost
-            install_target = platform['install target']
             if install_target == get_localhost_install_target():
                 continue
             # set off remote init
-            self.task_job_mgr.task_remote_mgr.remote_init(platform)
+            self.task_job_mgr.task_remote_mgr.remote_init(platforms[0])
             # Remote init/file-install is done via process pool
             self.proc_pool.process()
             # add platform to map (to be picked up on main loop)
-            self.incomplete_ri_map[install_target] = platform
+            self.incomplete_ri_map[install_target] = list(platforms)
 
     def manage_remote_init(self):
         """Manage the remote init/file install process for restarts.
@@ -925,21 +919,26 @@ class Scheduler:
         * Removes complete or fatally failed installations.
         * The bad_hosts logic already handles unreachable hosts.
         """
-        for install_target, platform in list(self.incomplete_ri_map.items()):
+        for install_target, platforms in list(self.incomplete_ri_map.items()):
             remote_mgr = self.task_job_mgr.task_remote_mgr
             status = remote_mgr.remote_init_map[install_target]
             if status == REMOTE_INIT_DONE:
-                remote_mgr.file_install(platform)
+                remote_mgr.file_install(platforms[0])
             elif status == REMOTE_INIT_255:
                 # Remote init failed due to unreachable host, retry.
-                remote_mgr.remote_init(platform)
+                remote_mgr.remote_init(platforms[0])
             elif status == REMOTE_FILE_INSTALL_255:
                 # File install failed due to unreachable host, retry.
-                remote_mgr.file_install(platform)
-            elif status in [REMOTE_FILE_INSTALL_DONE,
-                            REMOTE_INIT_FAILED,
-                            REMOTE_FILE_INSTALL_FAILED]:
-                # Complete or fatally failed, remove install target.
+                remote_mgr.file_install(platforms[0])
+            elif status == REMOTE_INIT_FAILED and len(platforms) > 1:
+                # Remote init failed (no more hosts), try on another platform
+                platforms.pop(0)
+                remote_mgr.remote_init(platforms[0])
+            elif status == REMOTE_FILE_INSTALL_FAILED and len(platforms) > 1:
+                # File install failed (no more hosts), try on another platform
+                platforms.pop(0)
+                remote_mgr.file_install(platforms[0])
+            else:  # REMOTE_FILE_INSTALL_DONE or FAILED and len(platforms) == 1
                 self.incomplete_ri_map.pop(install_target)
 
     def _load_task_run_times(self, row_idx, row):