🐛 fix(toml): split posargs string into separate args

A TOML command list entry holding only {posargs} was replaced with the
shell quoted join of the positional arguments, so `tox -- tests src`
handed the command a single `tests src` argument. Every INI user and most
TOML users read that entry as a stand-in for the arguments themselves.

The replacement already shell quotes what it produces, so splitting the
result back apart round-trips arguments containing spaces without a
second code path. The splitter that `to_command` used is now shared, so
both loaders agree on quoting and Windows path handling.

Author: gaborbernat

docs/changelog/4047.bugfix.rst

@@ -0,0 +1,2 @@
+A TOML command entry that is nothing but ``{posargs}`` now expands to one entry per positional argument, instead of
+collapsing them into a single shell quoted argument - by :user:`gaborbernat`.

docs/reference/config.rst

@@ -2424,15 +2424,18 @@ If the positional arguments are not set commands will become ``python a b``, oth
 The ``extend`` option instructs tox to unroll the positional arguments within the host structure. Without it the result
 would become ``["python", ["a", "b"]`` which would be invalid.
 
-Note that:
+The ``{posargs}`` string form gives the same result:
 
 .. code-block:: toml
 
     [env.A]
-    commands = [["python", "{posargs}" ]]
+    commands = [["python", "{posargs:a b}" ]]
 
-Differs in sense that the positional arguments will be set as a single argument, while in the original example they are
-passed through as separate.
+.. versionchanged:: 4.61
+
+    A list entry that is nothing but ``{posargs}`` expands into one entry per positional argument. Earlier releases
+    passed them on as a single, shell quoted argument. Embed the reference in a larger string (``"--opt={posargs}"``) to
+    keep the old behavior.
 
 Empty commands groups will be ignored:
 

src/tox/config/loader/str_convert.py

@@ -53,56 +53,14 @@ def to_dict(value: str, of_type: tuple[type[Any], type[Any]]) -> Iterator[tuple[
                     msg = f"dictionary lines must be of form key=value, found {row!r}"
                     raise TypeError(msg)
 
-    @staticmethod
-    def _win32_process_path_backslash(value: str, escape: str, special_chars: str) -> str:
-        """Escape backslash in value that is not followed by a special character.
-
-        This allows windows paths to be written without double backslash, while retaining the POSIX backslash escape
-        semantics for quotes and escapes.
-
-        """
-        result = []
-        for ix, char in enumerate(value):
-            result.append(char)
-            if char == escape:
-                last_char = value[ix - 1 : ix]
-                if last_char == escape:
-                    continue
-                next_char = value[ix + 1 : ix + 2]
-                if next_char not in {escape, *special_chars}:
-                    result.append(escape)  # escape escapes that are not themselves escaping a special character
-        return "".join(result)
-
     @staticmethod
     def to_command(value: str) -> Command:
         """At this point, ``value`` has already been substituted out, and all punctuation / escapes are final.
 
         Value will typically be stripped of whitespace when coming from an ini file.
 
         """
-        value = value.replace(r"\#", "#")
-        is_win = sys.platform == "win32"
-        if is_win:  # pragma: win32 cover
-            s = shlex.shlex(posix=True)
-            value = StrConvert._win32_process_path_backslash(
-                value,
-                escape=s.escape,
-                special_chars=s.quotes,
-            )
-        splitter = shlex.shlex(value, posix=True)
-        splitter.whitespace_split = True
-        splitter.commenters = ""  # comments handled earlier, and the shlex does not know escaped comment characters
-        args: list[str] = []
-        pos = 0
-        try:
-            for arg in splitter:
-                if is_win and len(arg) > 1 and arg[0] == arg[-1] and arg.startswith(("'", '"')):  # pragma: win32 cover
-                    # on Windows quoted arguments will remain quoted, strip it
-                    arg = arg[1:-1]  # ruff:ignore[redefined-loop-name]
-                args.append(arg)
-                pos = cast("StringIO", splitter.instream).tell()
-        except ValueError:
-            args.append(value[pos:])
+        args = split_args(value.replace(r"\#", "#"))
         if len(args) == 0:
             msg = f"attempting to parse {value!r} into a command failed"
             raise ValueError(msg)
@@ -134,4 +92,56 @@ def to_bool(value: str) -> bool:
         raise TypeError(msg)
 
 
-__all__ = ("StrConvert",)
+def split_args(value: str) -> list[str]:
+    """Split an already substituted shell-like string into its arguments.
+
+    :param value: the string to split, with all punctuation and escapes final
+
+    :returns: the arguments, with quoting resolved
+
+    """
+    is_win = sys.platform == "win32"
+    if is_win:  # pragma: win32 cover
+        s = shlex.shlex(posix=True)
+        value = _win32_process_path_backslash(value, escape=s.escape, special_chars=s.quotes)
+    splitter = shlex.shlex(value, posix=True)
+    splitter.whitespace_split = True
+    splitter.commenters = ""  # comments handled earlier, and the shlex does not know escaped comment characters
+    args: list[str] = []
+    pos = 0
+    try:
+        for arg in splitter:
+            if is_win and len(arg) > 1 and arg[0] == arg[-1] and arg.startswith(("'", '"')):  # pragma: win32 cover
+                # on Windows quoted arguments will remain quoted, strip it
+                arg = arg[1:-1]  # ruff:ignore[redefined-loop-name]
+            args.append(arg)
+            pos = cast("StringIO", splitter.instream).tell()
+    except ValueError:
+        args.append(value[pos:])
+    return args
+
+
+def _win32_process_path_backslash(value: str, escape: str, special_chars: str) -> str:
+    """Escape backslash in value that is not followed by a special character.
+
+    This allows windows paths to be written without double backslash, while retaining the POSIX backslash escape
+    semantics for quotes and escapes.
+
+    """
+    result = []
+    for ix, char in enumerate(value):
+        result.append(char)
+        if char == escape:
+            last_char = value[ix - 1 : ix]
+            if last_char == escape:
+                continue
+            next_char = value[ix + 1 : ix + 2]
+            if next_char not in {escape, *special_chars}:
+                result.append(escape)  # escape escapes that are not themselves escaping a special character
+    return "".join(result)
+
+
+__all__ = (
+    "StrConvert",
+    "split_args",
+)

src/tox/config/loader/toml/_replace.py

@@ -16,12 +16,15 @@
 from tox.config.loader.ini.factor import find_factor_groups
 from tox.config.loader.replacer import (
     MatchError,
+    MatchExpression,
     MatchRecursionError,
     ReplaceReference,
+    find_replace_expr,
     load_posargs,
     replace,
     replace_env,
 )
+from tox.config.loader.str_convert import split_args
 from tox.config.loader.stringify import stringify
 from tox.config.types import Command
 
@@ -76,6 +79,11 @@ def __call__(  # ruff:ignore[complex-structure, too-many-branches]
             # need to inspect every entry of the list to check for reference.
             res_list: list[TomlTypes] = []
             for val in value:  # apply replacement for every entry
+                if not skip_str and self.conf is not None and _is_posargs_expr(val):
+                    # a list entry that is nothing but a posargs reference stands for the arguments themselves, so
+                    # split the shell quoted replacement back apart instead of passing it on as one argument (#4047)
+                    res_list.extend(split_args(cast("str", self(val, depth))))
+                    continue
                 got = self(val, depth, skip_str=skip_str)
                 if isinstance(val, dict) and val.get("replace") and val.get("extend"):
                     # ``extend`` spreads an iterable result (list, set of extras, ...) into the
@@ -147,6 +155,13 @@ def _replace_ref(self, value: dict[str, TomlTypes], depth: int, *, skip_str: boo
         return value
 
 
+def _is_posargs_expr(value: TomlTypes) -> bool:
+    if not isinstance(value, str):
+        return False
+    parsed = find_replace_expr(value)
+    return len(parsed) == 1 and isinstance(parsed[0], MatchExpression) and parsed[0].expr[0] == ["posargs"]
+
+
 def _replace_glob_toml(conf: Config | None, value: dict[str, Any]) -> list[TomlTypes] | str:
     pattern = validate(value.get("pattern"), str)
     if not pattern:

tests/config/source/test_toml_tox.py

@@ -4,9 +4,11 @@
 import textwrap
 from typing import TYPE_CHECKING
 
-if TYPE_CHECKING:
-    import pytest
+import pytest
+
+from tox.execute.request import shell_cmd
 
+if TYPE_CHECKING:
     from tox.pytest import ToxProjectCreator
 
 
@@ -135,3 +137,38 @@ def test_config_in_toml_env_list_keyed_factor_description(tox_project: ToxProjec
     outcome = project.run("c", "-e", "sync-python-tt", "-k", "description")
     outcome.assert_success()
     outcome.assert_out_err("[testenv:sync-python-tt]\ndescription = Sync python to tt\n", "")
+
+
+@pytest.mark.parametrize(
+    ("posargs", "expected"),
+    [
+        pytest.param([], "python a b", id="default"),
+        pytest.param(["--"], "python", id="cleared"),
+        pytest.param(["--", "c", "d"], "python c d", id="set"),
+        pytest.param(["--", "c d", "e"], f"python {shell_cmd(['c d'])} e", id="set-with-space"),
+    ],
+)
+def test_config_in_toml_posargs_string_is_split(
+    tox_project: ToxProjectCreator, posargs: list[str], expected: str
+) -> None:
+    project = tox_project({
+        "tox.toml": """
+        [env.A]
+        commands = [["python", "{posargs:a b}"]]
+        """
+    })
+    outcome = project.run("c", "-e", "A", "-k", "commands", *posargs)
+    outcome.assert_success()
+    outcome.assert_out_err(f"[testenv:A]\ncommands = {expected}\n", "")
+
+
+def test_config_in_toml_posargs_string_within_argument_is_joined(tox_project: ToxProjectCreator) -> None:
+    project = tox_project({
+        "tox.toml": """
+        [env.A]
+        commands = [["python", "--opt={posargs}"]]
+        """
+    })
+    outcome = project.run("c", "-e", "A", "-k", "commands", "--", "c", "d")
+    outcome.assert_success()
+    outcome.assert_out_err(f"[testenv:A]\ncommands = python {shell_cmd(['--opt=c d'])}\n", "")