Merge pull request #269 from wxtim/feat.reload_glbl_cfg_after_export_…

…CYLC_SYMLINKS

Repopulate cached GlobalConfig object if CYLC_SYMLINKS exported to env.

CHANGES.md

@@ -6,6 +6,14 @@ creating a new release entry be sure to copy & paste the span tag with the
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
 
+## __cylc-rose-1.4.0 (<span actions:bind='release-date'>Upcoming</span>)__
+
+### Features
+
+[#269](https://github.com/cylc/cylc-rose/pull/269) - Allow environment variables
+set in ``rose-suite.conf`` to be used when parsing ``global.cylc``.
+
+
 ## __cylc-rose-1.3.2 (<span actions:bind='release-date'>Released 2024-01-18</span>)__
 
 [#284](https://github.com/cylc/cylc-rose/pull/284) - Allow use of Metomi-Rose 2.2.*.

cylc/rose/__init__.py

@@ -70,11 +70,37 @@
    for ease of porting Cylc 7 workflows.
 
 
+The ``global.cylc`` file
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+The Cylc Rose Plugin forces the reloading of the ``global.cylc`` file
+to allow environment variables set by Rose to change the global configuration.
+
+For example you could use ``CYLC_SYMLINKS`` as a variable to control
+the behaviour of ``cylc install``:
+
+.. code-block:: cylc
+
+   #!jinja2
+   # part of a global.cylc file
+   [install]
+      [[symlink dirs]]
+         [[[hpc]]]
+   {% if environ["CYLC_SYMLINKS"] | default("x") == "A" %}
+            run = $LOCATION_A
+   {% elif environ["CYLC_SYMLINKS"] | default("x") == "B" %}
+            run = $LOCATION_B
+   {% else %}
+            run = $LOCATION_C
+   {% endif %}
+
+
+
 Special Variables
 -----------------
 
 The Cylc Rose plugin provides two environment/template variables
-to the Cylc scheduler:
+to the Cylc scheduler.
 
 ``ROSE_ORIG_HOST``
    Cylc commands (such as ``cylc install``, ``cylc validate`` and
@@ -111,6 +137,7 @@
       ``CYLC_VERSION`` will be removed from your configuration by the
       Cylc-Rose plugin, as it is now set by Cylc.
 
+
 Additional CLI options
 ----------------------
 You can use command line options to set or override

cylc/rose/utilities.py

@@ -27,6 +27,7 @@
 from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
 from cylc.flow.flags import cylc7_back_compat
+from cylc.flow.cfgspec.glbl_cfg import glbl_cfg
 from cylc.flow.hostuserutil import get_host
 from metomi.isodatetime.datetimeoper import DateTimeOperator
 from metomi.rose import __version__ as ROSE_VERSION
@@ -869,6 +870,13 @@ def export_environment(environment: Dict[str, str]) -> None:
     for key, val in environment.items():
         os.environ[key] = val
 
+    # If env vars have been set we want to force reload
+    # the global config so that the value of this vars
+    # can be used by Jinja2 in the global config.
+    # https://github.com/cylc/cylc-rose/issues/237
+    if environment:
+        glbl_cfg().load()
+
 
 def record_cylc_install_options(
     srcdir: Path,

tests/conftest.py

@@ -15,9 +15,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from types import SimpleNamespace
+from uuid import uuid4
 
 from cylc.flow import __version__ as CYLC_VERSION
 from cylc.flow.option_parsers import Options
+from cylc.flow.pathutil import get_workflow_run_dir
 from cylc.flow.scripts.install import get_option_parser as install_gop
 from cylc.flow.scripts.install import install_cli as cylc_install
 from cylc.flow.scripts.reinstall import get_option_parser as reinstall_gop
@@ -27,6 +29,16 @@
 import pytest
 
 
+@pytest.fixture()
+def workflow_name():
+    return 'cylc-rose-test-' + str(uuid4())[:8]
+
+
+@pytest.fixture(scope='module')
+def mod_workflow_name():
+    return 'cylc-rose-test-' + str(uuid4())[:8]
+
+
 @pytest.fixture(scope='module')
 def mod_capsys(request):
     from _pytest.capture import SysCapture
@@ -108,7 +120,7 @@ def _inner(srcpath, args=None):
     return _inner
 
 
-def _cylc_install_cli(capsys, caplog):
+def _cylc_install_cli(capsys, caplog, workflow_name):
     """Access the install CLI"""
     def _inner(srcpath, args=None):
         """Install a workflow.
@@ -119,16 +131,21 @@ def _inner(srcpath, args=None):
         """
         options = Options(install_gop(), args)()
         output = SimpleNamespace()
+        if not options.workflow_name:
+            options.workflow_name = workflow_name
+        if not args or not args.get('no_run_name', ''):
+            options.no_run_name = True
 
         try:
-            cylc_install(options, str(srcpath))
+            output.name, output.id = cylc_install(options, str(srcpath))
             output.ret = 0
             output.exc = ''
         except Exception as exc:
             output.ret = 1
             output.exc = exc
         output.logging = '\n'.join([i.message for i in caplog.records])
         output.out, output.err = capsys.readouterr()
+        output.run_dir = get_workflow_run_dir(output.id)
         return output
     return _inner
 
@@ -159,13 +176,14 @@ def _inner(workflow_id, opts=None):
 
 
 @pytest.fixture
-def cylc_install_cli(capsys, caplog):
-    return _cylc_install_cli(capsys, caplog)
+def cylc_install_cli(capsys, caplog, workflow_name):
+    return _cylc_install_cli(capsys, caplog, workflow_name)
 
 
 @pytest.fixture(scope='module')
-def mod_cylc_install_cli(mod_capsys, mod_caplog):
-    return _cylc_install_cli(mod_capsys, mod_caplog)
+def mod_cylc_install_cli(mod_capsys, mod_caplog, mod_workflow_name):
+    return _cylc_install_cli(
+        mod_capsys, mod_caplog, mod_workflow_name)
 
 
 @pytest.fixture

tests/functional/test_ROSE_ORIG_HOST.py

@@ -119,7 +119,7 @@ def fixture_install_flow(
     )
     install_conf_path = (
         fixture_provide_flow['flowpath'] /
-        'runN/opt/rose-suite-cylc-install.conf'
+        'opt/rose-suite-cylc-install.conf'
     )
     text = install_conf_path.read_text()
     text = re.sub('ROSE_ORIG_HOST=.*', 'ROSE_ORIG_HOST=foo', text)
@@ -142,7 +142,7 @@ def test_cylc_validate_srcdir(fixture_install_flow, mod_cylc_validate_cli):
 def test_cylc_validate_rundir(fixture_install_flow, mod_cylc_validate_cli):
     """Sanity check that workflow validates:
     """
-    flowpath = fixture_install_flow['flowpath'] / 'runN'
+    flowpath = fixture_install_flow['flowpath']
     result = mod_cylc_validate_cli(flowpath)
     assert 'ROSE_ORIG_HOST (env) is:' in result.logging
 

tests/functional/test_reinstall.py

@@ -118,14 +118,14 @@ def test_cylc_install_run(fixture_install_flow):
     'file_, expect',
     [
         (
-            'run1/rose-suite.conf', (
+            'rose-suite.conf', (
                 '# Config Options \'b c (cylc-install)\' from CLI appended to'
                 ' options already in `rose-suite.conf`.\n'
                 'opts=a b c (cylc-install)\n'
             )
         ),
         (
-            'run1/opt/rose-suite-cylc-install.conf', (
+            'opt/rose-suite-cylc-install.conf', (
                 '# This file records CLI Options.\n\n'
                 '!opts=b c\n'
                 f'\n[env]\n#{ROHIOS}\nROSE_ORIG_HOST={HOST}\n'
@@ -172,14 +172,14 @@ def test_cylc_reinstall_run(fixture_reinstall_flow):
     'file_, expect',
     [
         (
-            'run1/rose-suite.conf', (
+            'rose-suite.conf', (
                 '# Config Options \'b c d (cylc-install)\' from CLI appended '
                 'to options already in `rose-suite.conf`.\n'
                 'opts=a b c d (cylc-install)\n'
             )
         ),
         (
-            'run1/opt/rose-suite-cylc-install.conf', (
+            'opt/rose-suite-cylc-install.conf', (
                 '# This file records CLI Options.\n\n'
                 '!opts=b c d\n'
                 f'\n[env]\n#{ROHIOS}\nROSE_ORIG_HOST={HOST}\n'
@@ -230,14 +230,14 @@ def test_cylc_reinstall_run2(fixture_reinstall_flow2):
     'file_, expect',
     [
         (
-            'run1/rose-suite.conf', (
+            'rose-suite.conf', (
                 '# Config Options \'b c d (cylc-install)\' from CLI appended '
                 'to options already in `rose-suite.conf`.\n'
                 'opts=z b c d (cylc-install)\n'
             )
         ),
         (
-            'run1/opt/rose-suite-cylc-install.conf', (
+            'opt/rose-suite-cylc-install.conf', (
                 '# This file records CLI Options.\n\n'
                 '!opts=b c d\n'
                 f'\n[env]\n#{ROHIOS}\nROSE_ORIG_HOST={HOST}\n'

tests/functional/test_reinstall_clean.py

@@ -109,7 +109,7 @@ def test_cylc_install_run(fixture_install_flow):
     'file_, expect',
     [
         (
-            'run1/opt/rose-suite-cylc-install.conf', (
+            'opt/rose-suite-cylc-install.conf', (
                 '# This file records CLI Options.\n\n'
                 '!opts=bar\n\n'
                 '[env]\n'
@@ -163,7 +163,7 @@ def test_cylc_reinstall_run(fixture_reinstall_flow):
     'file_, expect',
     [
         (
-            'run1/opt/rose-suite-cylc-install.conf', (
+            'opt/rose-suite-cylc-install.conf', (
                 '# This file records CLI Options.\n\n'
                 '!opts=baz\n\n'
                 '[env]\n'

tests/functional/test_rose_fileinstall.py

@@ -63,7 +63,7 @@ def fixture_install_flow(fixture_provide_flow, request, cylc_install_cli):
 
     yield srcpath, datapath, flow_name, result, destpath
     if not request.session.testsfailed:
-        shutil.rmtree(destpath)
+        shutil.rmtree(destpath, ignore_errors=True)
 
 
 def test_rose_fileinstall_validate(fixture_provide_flow, cylc_validate_cli):
@@ -84,15 +84,15 @@ def test_rose_fileinstall_subfolders(fixture_install_flow):
     """File installed into a sub directory:
     """
     _, datapath, _, _, destpath = fixture_install_flow
-    assert ((destpath / 'runN/lib/python/lion.py').read_text() ==
+    assert ((destpath / 'lib/python/lion.py').read_text() ==
             (datapath / 'lion.py').read_text())
 
 
 def test_rose_fileinstall_concatenation(fixture_install_flow):
     """Multiple files concatenated on install(source contained wildcard):
     """
     _, datapath, _, _, destpath = fixture_install_flow
-    assert ((destpath / 'runN/data').read_text() ==
+    assert ((destpath / 'data').read_text() ==
             ((datapath / 'randoms1.data').read_text() +
             (datapath / 'randoms3.data').read_text()
              ))

tests/functional/test_utils.py

@@ -36,3 +36,94 @@ def test_basic(tmp_path):
     assert Path(tmp_path / 'src/rose-suite.conf').read_text() == (
         Path(tmp_path / 'dest/rose-suite.conf').read_text()
     )
+
+
+def test_global_config_environment_validate(
+    monkeypatch, tmp_path, cylc_validate_cli
+):
+    """It should reload the global config after exporting env variables.
+
+    See: https://github.com/cylc/cylc-rose/issues/237
+    """
+    # Setup global config:
+    global_conf = """#!jinja2
+        {% from "cylc.flow" import LOG %}
+        {% set cylc_symlinks = environ.get('CYLC_SYMLINKS', None) %}
+        {% do LOG.critical(cylc_symlinks) %}
+    """
+    conf_path = tmp_path / 'conf'
+    conf_path.mkdir()
+    monkeypatch.setenv('CYLC_CONF_PATH', conf_path)
+
+    # Setup workflow config:
+    (conf_path / 'global.cylc').write_text(global_conf)
+    (tmp_path / 'rose-suite.conf').write_text(
+        '[env]\nCYLC_SYMLINKS="Foo"\n')
+    (tmp_path / 'flow.cylc').write_text("""
+        [scheduling]
+            initial cycle point = now
+            [[graph]]
+                R1 = x
+        [runtime]
+            [[x]]
+    """)
+
+    # Validate the config:
+    output = cylc_validate_cli(tmp_path)
+    assert output.ret == 0
+
+    # CYLC_SYMLINKS == None the first time the global.cylc
+    # is loaded and "Foo" the second time.
+    assert output.logging.split('\n')[-1] == '"Foo"'
+
+
+def test_global_config_environment_validate2(
+    monkeypatch, tmp_path, cylc_install_cli
+):
+    """It should reload the global config after exporting env variables.
+
+    See: https://github.com/cylc/cylc-rose/issues/237
+    """
+    # Setup global config:
+    global_conf = (
+        '#!jinja2\n'
+        '[install]\n'
+        '    [[symlink dirs]]\n'
+        '        [[[localhost]]]\n'
+        '{% set cylc_symlinks = environ.get(\'CYLC_SYMLINKS\', None) %}\n'
+        '{% if cylc_symlinks == "foo" %}\n'
+        f'log = {str(tmp_path)}/foo\n'
+        '{% else %}\n'
+        f'log = {str(tmp_path)}/bar\n'
+        '{% endif %}\n'
+    )
+    glbl_conf_path = tmp_path / 'conf'
+    glbl_conf_path.mkdir()
+    (glbl_conf_path / 'global.cylc').write_text(global_conf)
+    monkeypatch.setenv('CYLC_CONF_PATH', glbl_conf_path)
+
+    # Setup workflow config:
+    (tmp_path / 'rose-suite.conf').write_text(
+        '[env]\nCYLC_SYMLINKS=foo\n')
+    (tmp_path / 'flow.cylc').write_text("""
+        [scheduling]
+            initial cycle point = now
+            [[graph]]
+                R1 = x
+        [runtime]
+            [[x]]
+    """)
+
+    # Install the config:
+    output = cylc_install_cli(tmp_path)
+    import sys
+    for i in output.logging.split('\n'):
+        print(i, file=sys.stderr)
+    assert output.ret == 0
+
+    # Assert symlink created back to test_path/foo:
+    expected_msg = (
+        f'Symlink created: {output.run_dir}/log -> '
+        f'{tmp_path}/foo/cylc-run/{output.id}/log'
+    )
+    assert expected_msg in output.logging.split('\n')[0]