Merge 'upstream/master' into 8.3.x-sync

Author: MetRonnie

.github/workflows/2_auto_publish_release.yml

@@ -38,7 +38,7 @@ jobs:
       uses: cylc/release-actions/build-python-package@v1
 
     - name: Publish distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@v1.10.1
+      uses: pypa/gh-action-pypi-publish@v1.12.2
       with:
         user: __token__ # uses the API token feature of PyPI - least permissions possible
         password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/test_fast.yml

@@ -150,7 +150,7 @@ jobs:
         uses: actions/download-artifact@v4
 
       - name: Codecov upload
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           name: ${{ github.workflow }}
           flags: fast-tests

.github/workflows/test_functional.yml

@@ -335,7 +335,7 @@ jobs:
         uses: actions/download-artifact@v4
 
       - name: Codecov upload
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           name: ${{ github.workflow }}
           flags: functional-tests

CHANGES.md

@@ -101,8 +101,6 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 ### 🔧 Fixes
 
-[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
-
 [#6186](https://github.com/cylc/cylc-flow/pull/6186) - Fixed bug where using flow numbers with `cylc set` would not work correctly.
 
 [#6200](https://github.com/cylc/cylc-flow/pull/6200) - Fixed bug where a stalled paused workflow would be incorrectly reported as running, not paused
@@ -119,6 +117,8 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 [#6176](https://github.com/cylc/cylc-flow/pull/6176) - Fix bug where jobs which fail to submit are not shown in GUI/TUI if submission retries are set.
 
+[#6178](https://github.com/cylc/cylc-flow/pull/6178) - Fix an issue where Tui could hang when closing.
+
 ## __cylc-8.3.0 (Released 2024-06-18)__
 
 ### ⚠ Breaking Changes

CONTRIBUTING.md

@@ -29,7 +29,7 @@ We use [semver](https://semver.org/) to separate riskier changes (e.g. new featu
 (e.g. 8.1, 8.2, 8.3)
 
 **Bugfixes** and minor usability enhancements are made on bugfix branches and
-released as the next maintainance version (e.g. 8.0.1, 8.0.2, 8.0.3). E.G. if the issue is on the `8.0.x` milestone, branch off of `8.0.x` to
+released as the next maintenance version (e.g. 8.0.1, 8.0.2, 8.0.3). E.G. if the issue is on the `8.0.x` milestone, branch off of `8.0.x` to
 develop your bugfix, then raise the pull request against the `8.0.x` branch. We will later merge the `8.0.x` branch into `master`.
 
 Feel free to ask questions on the issue or
@@ -97,6 +97,8 @@ requests_).
  - Shixian Sheng
  - Utheri Wagura
  - Maxime Rio
+ - Paul Armstrong
+ - Paul Earnshaw
 <!-- end-shortlog -->
 
 (All contributors are identifiable with email addresses in the git version

changes.d/6137.feat.md

@@ -0,0 +1 @@
+New Cylc lint rule: S014: Don't use job runner specific execution time limit directives, use execution time limit.

changes.d/6168.feat.md

@@ -0,0 +1 @@
+Allow symlinking log/job separately from log

changes.d/6289.feat.md

@@ -0,0 +1 @@
+Made the errors resulting from Jinja2 `raise` and `assert` statements more straight forward.

changes.d/6440.feat.md

@@ -0,0 +1 @@
+The "cylc dump" command now prints task IDs. Use "--legacy" if you need the old format.

changes.d/6444.feat.md

@@ -0,0 +1 @@
+The scheduler now traps the SIGINT, SIGTERM and SIGHUP signals and will respond by shutting down in --now mode. If the workflow is already shutting down in --now mode, it will escalate the shutdown to --now --now mode.

changes.d/6476.break.md

@@ -0,0 +1 @@
+Remove support for the EmPy template engine.

conda-environment.yml

@@ -21,7 +21,6 @@ dependencies:
   - tomli >=2 # [py<3.11]
 
 # optional dependencies
-  #- empy >=3.3,<3.4
   #- pandas >=1.0,<2
   #- pympler
   #- matplotlib-base

cylc/flow/__init__.py

@@ -15,9 +15,8 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 """Set up the cylc environment."""
 
-import os
 import logging
-
+import os
 
 CYLC_LOG = 'cylc'
 
@@ -53,7 +52,7 @@ def environ_init():
 
 environ_init()
 
-__version__ = '8.3.7.dev'
+__version__ = '8.4.0.dev'
 
 
 def iter_entry_points(entry_point_name):

cylc/flow/broadcast_report.py

@@ -72,7 +72,7 @@ def get_broadcast_change_iter(modified_settings, is_cancel=False):
         value = setting
         keys_str = ""
         while isinstance(value, dict):
-            key, value = list(value.items())[0]
+            key, value = next(iter(value.items()))
             if isinstance(value, dict):
                 keys_str += "[" + key + "]"
             else:

cylc/flow/cfgspec/globalcfg.py

@@ -30,6 +30,7 @@
 from cylc.flow.exceptions import GlobalConfigError
 from cylc.flow.hostuserutil import get_user_home
 from cylc.flow.network.client_factory import CommsMeth
+from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
 from cylc.flow.parsec.config import (
     ConfigNode as Conf,
     ParsecConfig,
@@ -1148,55 +1149,21 @@ def default_for(
 
                     .. versionadded:: 8.0.0
                 """)
-                Conf('log', VDR.V_STRING, None, desc="""
-                    Alternative location for the log dir.
-
-                    If specified the workflow log directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/log`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/log``. If not specified
-                    the workflow log directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/log``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share', VDR.V_STRING, None, desc="""
-                    Alternative location for the share dir.
-
-                    If specified the workflow share directory will be
-                    created in ``<this-path>/cylc-run/<workflow-id>/share``
-                    and a symbolic link will be created from
-                    ``<$HOME/cylc-run/<workflow-id>/share``. If not specified
-                    the workflow share directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/share``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('share/cycle', VDR.V_STRING, None, desc="""
-                    Alternative directory for the share/cycle dir.
-
-                    If specified the workflow share/cycle directory
-                    will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/share/cycle``
-                    and a symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/share/cycle``. If not
-                    specified the workflow share/cycle directory will be
-                    created in ``$HOME/cylc-run/<workflow-id>/share/cycle``.
-
-                    .. versionadded:: 8.0.0
-                """)
-                Conf('work', VDR.V_STRING, None, desc="""
-                    Alternative directory for the work dir.
-
-                    If specified the workflow work directory will be created in
-                    ``<this-path>/cylc-run/<workflow-id>/work`` and a
-                    symbolic link will be created from
-                    ``$HOME/cylc-run/<workflow-id>/work``. If not specified
-                    the workflow work directory will be created in
-                    ``$HOME/cylc-run/<workflow-id>/work``.
-
-                    .. versionadded:: 8.0.0
-                """)
+                for folder, versionadded in SYMLINKABLE_LOCATIONS.items():
+                    Conf(folder, VDR.V_STRING, None, desc=f"""
+                        Alternative location for the {folder} dir.
+
+                        If specified the workflow {folder} directory will
+                        be created in
+                        ``<this-path>/cylc-run/<workflow-id>/{folder}``
+                        and a symbolic link will be created from
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+                        If not specified the workflow log directory will
+                        be created in
+                        ``$HOME/cylc-run/<workflow-id>/{folder}``.
+
+                        .. versionadded:: {versionadded}
+                    """)
     with Conf('platforms', desc='''
         Platforms allow you to define compute resources available at your
         site.
@@ -1311,7 +1278,7 @@ def default_for(
                 The means by which task progress messages are reported back to
                 the running workflow.
 
-                Options:
+                ..rubric:: Options:
 
                 zmq
                    Direct client-server TCP communication via network ports
@@ -1320,6 +1287,8 @@ def default_for(
                 ssh
                    Use non-interactive ssh for task communications
 
+                For more information, see :ref:`TaskComms`.
+
                 .. versionchanged:: 8.0.0
 
                    {REPLACES}``global.rc[hosts][<host>]task communication

cylc/flow/clean.py

@@ -126,7 +126,7 @@ def _clean_check(opts: 'Values', id_: str, run_dir: Path) -> None:
     except ContactFileExists as exc:
         raise ServiceFileError(
             f"Cannot clean running workflow {id_}.\n\n{exc}"
-        )
+        ) from None
 
 
 def init_clean(id_: str, opts: 'Values') -> None:
@@ -170,7 +170,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
             try:
                 platform_names = get_platforms_from_db(local_run_dir)
             except ServiceFileError as exc:
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from None
             except sqlite3.OperationalError as exc:
                 # something went wrong with the query
                 # e.g. the table/field we need isn't there
@@ -183,7 +183,7 @@ def init_clean(id_: str, opts: 'Values') -> None:
                     ' local files (you may need to remove files on other'
                     ' platforms manually).'
                 )
-                raise ServiceFileError(f"Cannot clean {id_} - {exc}")
+                raise ServiceFileError(f"Cannot clean {id_} - {exc}") from exc
 
         if platform_names and platform_names != {'localhost'}:
             remote_clean(
@@ -367,7 +367,8 @@ def remote_clean(
     except PlatformLookupError as exc:
         raise PlatformLookupError(
             f"Cannot clean {id_} on remote platforms as the workflow database "
-            f"is out of date/inconsistent with the global config - {exc}")
+            f"is out of date/inconsistent with the global config - {exc}"
+        ) from None
 
     queue: Deque[RemoteCleanQueueTuple] = deque()
     remote_clean_cmd = partial(

cylc/flow/command_validation.py

@@ -77,7 +77,7 @@ def flow_opts(flows: List[str], flow_wait: bool) -> None:
             try:
                 int(val)
             except ValueError:
-                raise InputError(ERR_OPT_FLOW_VAL)
+                raise InputError(ERR_OPT_FLOW_VAL) from None
 
     if flow_wait and flows[0] in {FLOW_NEW, FLOW_NONE}:
         raise InputError(ERR_OPT_FLOW_WAIT)

cylc/flow/commands.py

@@ -77,7 +77,6 @@
 from cylc.flow.network.schema import WorkflowStopMode
 from cylc.flow.parsec.exceptions import ParsecError
 from cylc.flow.task_id import TaskID
-from cylc.flow.task_state import TASK_STATUSES_ACTIVE, TASK_STATUS_FAILED
 from cylc.flow.workflow_status import RunMode, StopMode
 
 from metomi.isodatetime.parsers import TimePointParser
@@ -211,7 +210,7 @@ async def stop(
         try:
             mode = StopMode(mode)
         except ValueError:
-            raise CommandFailedError(f"Invalid stop mode: '{mode}'")
+            raise CommandFailedError(f"Invalid stop mode: '{mode}'") from None
         schd._set_stop(mode)
         if mode is StopMode.REQUEST_KILL:
             schd.time_next_kill = time()
@@ -256,19 +255,16 @@ async def poll_tasks(schd: 'Scheduler', tasks: Iterable[str]):
 
 @_command('kill_tasks')
 async def kill_tasks(schd: 'Scheduler', tasks: Iterable[str]):
-    """Kill all tasks or a task/family if options are provided."""
+    """Kill tasks.
+
+    Args:
+        tasks: Tasks/families/globs to kill.
+    """
     validate.is_tasks(tasks)
     yield
-    itasks, _, bad_items = schd.pool.filter_task_proxies(tasks)
-    if schd.get_run_mode() == RunMode.SIMULATION:
-        for itask in itasks:
-            if itask.state(*TASK_STATUSES_ACTIVE):
-                itask.state_reset(TASK_STATUS_FAILED)
-                schd.data_store_mgr.delta_task_state(itask)
-        yield len(bad_items)
-    else:
-        schd.task_job_mgr.kill_task_jobs(schd.workflow, itasks)
-        yield len(bad_items)
+    active, _, unmatched = schd.pool.filter_task_proxies(tasks)
+    num_unkillable = schd.kill_tasks(active)
+    yield len(unmatched) + num_unkillable
 
 
 @_command('hold')
@@ -308,7 +304,7 @@ async def set_verbosity(schd: 'Scheduler', level: Union[int, str]):
         lvl = int(level)
         LOG.setLevel(lvl)
     except (TypeError, ValueError) as exc:
-        raise CommandFailedError(exc)
+        raise CommandFailedError(exc) from None
     cylc.flow.flags.verbosity = log_level_to_verbosity(lvl)
     yield