From c55a9b2f708938cf91ad8fce524689d545afce51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Tue, 4 Aug 2026 17:58:25 -0700 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(config):=20catalog=20environme?= =?UTF-8?q?nts=20per=20PEP=20832?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Editors have no way to find the environments tox builds under .tox, so using one means copying a path out of tox devenv and pasting it into a settings dialog, again after every recreate. PEP 832 fixes that with a .python-envs file at the project root that any tool can write and any editor can read, with the last line naming the default environment. tox now writes that file at the end of a run. The ordering puts the environment a developer edits code against last: one named dev, then a develop install, then the earlier entries of env_list, so trimming lines from the bottom falls back to the next best choice. Lines outside the work dir belong to another tool and survive untouched, including their claim on the last line. A recreated environment leaves the catalog while it rebuilds so nothing points at a half-built interpreter. Writing into the project root is on by default, since the convention only pays off when tools follow it without being asked; python_envs turns it off. --- .gitignore | 1 + docs/changelog/4013.feature.rst | 6 ++ docs/explanation.rst | 25 +++++ docs/how-to/usage.rst | 38 +++++++ docs/reference/config.rst | 15 +++ docs/tutorial/getting-started.rst | 17 +++ src/tox/config/sets.py | 6 ++ src/tox/pytest.py | 2 +- src/tox/session/cmd/run/common.py | 20 ++++ src/tox/tox.schema.json | 4 + src/tox/tox_env/api.py | 3 + src/tox/util/python_envs.py | 71 +++++++++++++ tests/session/cmd/run/test_python_envs.py | 77 ++++++++++++++ tests/util/test_python_envs.py | 120 ++++++++++++++++++++++ 14 files changed, 404 insertions(+), 1 deletion(-) create mode 100644 docs/changelog/4013.feature.rst create mode 100644 src/tox/util/python_envs.py create mode 100644 tests/session/cmd/run/test_python_envs.py create mode 100644 tests/util/test_python_envs.py diff --git a/.gitignore b/.gitignore index 8654da708..660a249e7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ __pycache__ /tests/demo_pkg_setuptools/build/lib/demo_pkg_setuptools/__init__.py /tests/demo_pkg_inline.lock /tests/demo_pkg_inline/.tox/ +.python-envs diff --git a/docs/changelog/4013.feature.rst b/docs/changelog/4013.feature.rst new file mode 100644 index 000000000..f8ff5f7f1 --- /dev/null +++ b/docs/changelog/4013.feature.rst @@ -0,0 +1,6 @@ +Catalog the created tox environments in a :PEP:`832` ``.python-envs`` file next to the configuration file, so that +editors such as VS Code or PyCharm offer them as interpreters instead of asking you for a path under ``.tox``. The +default environment comes last, and tox prefers one named ``dev`` for that spot, then one that installs the project in +development mode, then the first of :ref:`env_list`. tox leaves the lines other tools wrote alone, and takes an +environment out of the catalog while it recreates it. Set :ref:`python_envs` to ``false`` to skip the file - by +:user:`gaborbernat`. (:issue:`4013`) diff --git a/docs/explanation.rst b/docs/explanation.rst index 3f53e9b1e..9578f43f1 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -721,6 +721,31 @@ upgrade that changes the derived value) triggers automatic recreation. This design mirrors tox's own auto-provisioning mechanism (``requires`` / ``min_version``), where tox bootstraps itself into a separate environment when the running installation doesn't meet the declared requirements. +*********************** + Environment discovery +*********************** + +Editors need an interpreter path before they can offer completion, navigation or a debugger. tox keeps its environments +under ``.tox``, a directory editors have no reason to search, so for years the answer was to copy a path out of ``tox +devenv`` output and paste it into a settings dialog, then repeat it after every recreate. + +:PEP:`832` standardizes where tools publish that answer. A project may hold a virtual environment at ``.venv``, and a +``.python-envs`` file at the project root lists any further environments, one directory per line. The last line is the +default, so a reader that supports only one environment still knows which to take. VS Code reads the file today and +PyCharm honors the ``.venv`` half of the convention. + +tox writes ``.python-envs`` at the end of a run, listing every environment that exists on disk. Ordering follows how +useful an environment is to an editor rather than how tox happens to schedule it: an environment named ``dev`` wins, +then one installing the project in development mode, then the earlier entries of :ref:`env_list`. Since the catalog +lists the preferred environment last, removing lines from the bottom degrades to the next best choice. + +Two rules keep the file honest. tox owns the lines under :ref:`work_dir` and rewrites them wholesale, while lines +pointing elsewhere came from another tool and survive untouched, including the last line and the default it claims. And +because a recreated environment is unusable between the moment tox empties the directory and the moment the new +interpreter lands, tox takes the environment out of the catalog first and puts it back once the run ends. + +Set :ref:`python_envs` to ``false`` if you would rather tox left the project root alone. + ******************* Known limitations ******************* diff --git a/docs/how-to/usage.rst b/docs/how-to/usage.rst index 948c9ceae..5d1f420c7 100644 --- a/docs/how-to/usage.rst +++ b/docs/how-to/usage.rst @@ -162,6 +162,44 @@ The ``tox exec`` subcommand runs an arbitrary command inside a tox environment w The command must be in the environment's ``PATH`` or listed in :ref:`allowlist_externals`. ``tox exec`` is useful for debugging, running one-off scripts, or interactively exploring an environment without modifying your configuration. +.. _howto_editor_env: + +*************************************** + Open a tox environment in your editor +*************************************** + +Editors pick up interpreters through the :PEP:`832` ``.python-envs`` file, which tox writes next to your configuration +file after each run. Name the environment you develop against ``dev`` and tox puts it on the last line, the one readers +treat as the default: + +.. code-block:: toml + + [env.dev] + description = "development environment" + package = "editable" + dependency_groups = [ "dev" ] + +Create it, then reload your editor window: + +.. code-block:: bash + + tox run -e dev --notest + +The file lists one directory per line, so you can check what an editor will see: + +.. code-block:: bash + + $ cat .python-envs + .tox/3.13 + .tox/dev + +Commit the file when your team keeps environments in the same place, otherwise add it to your ``.gitignore``. To keep +tox from writing it at all, set :ref:`python_envs` in the core section: + +.. code-block:: toml + + python_envs = false + .. ------------------------------------------------------------------------------------------ .. Configuration (frequently needed) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 4487ef0fe..2374ca42f 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -539,6 +539,21 @@ the top level of ``tox.toml``. Placing these options in an environment section ( this directory for the project package. This ensures tox works correctly when having parallel runs (as each session will have its own copy of the project package - e.g. the source distribution). +.. conf:: + :keys: python_envs + :default: true + :version_added: 4.59.0 + + Catalog the created tox environments in a :pep:`832` ``.python-envs`` file next to the configuration file, so that + editors such as VS Code or PyCharm offer them as interpreters. The file holds one environment directory per line, + with the default one last. tox prefers an environment named ``dev`` for that spot, then one that installs the + project in development mode, then the first entry of :ref:`env_list`. It lists only environments that exist, and + takes one out of the catalog while it recreates it. + + Lines pointing outside the :ref:`work_dir` come from another tool, so tox keeps them, including their claim on the + last line. Commit the file when everyone working on the project has their environments in the same place, otherwise + ignore it. + .. conf:: :keys: no_package, skipsdist :default: false diff --git a/docs/tutorial/getting-started.rst b/docs/tutorial/getting-started.rst index ecb47768b..885a06940 100644 --- a/docs/tutorial/getting-started.rst +++ b/docs/tutorial/getting-started.rst @@ -287,6 +287,23 @@ has changed), use ``--skip-env-install``: tox run -e 3.13 --skip-env-install +****************************** + Using an environment at hand +****************************** + +After a run, tox records the environments it created in a ``.python-envs`` file next to your configuration, following +:PEP:`832`. Editors such as VS Code read it and offer those interpreters, so you can pick one instead of hunting for a +path under ``.tox``: + +.. code-block:: bash + + $ tox run -e 3.13 + $ cat .python-envs + .tox/3.13 + +The last line is the default. Name an environment ``dev`` and tox puts it there, which is what you want for the +environment you edit code against - see :ref:`howto_editor_env`. + ******************************** Listing available environments ******************************** diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index f26edcb06..2c1dd5304 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -315,6 +315,12 @@ def register_config(self) -> None: default=self._default_temp_dir, desc="a folder for temporary files (is not cleaned at start)", ) + self.add_config( + keys=["python_envs"], + of_type=bool, + default=True, + desc="catalog the created tox environments in a PEP-832 .python-envs file, so editors can discover them", + ) self.add_constant("host_python", "the host python executable path", sys.executable) def _on_duplicate_conf(self, key: str, definition: ConfigDefinition[V]) -> None: diff --git a/src/tox/pytest.py b/src/tox/pytest.py index 12befc389..bf449b026 100644 --- a/src/tox/pytest.py +++ b/src/tox/pytest.py @@ -147,7 +147,7 @@ def __init__( # ruff:ignore[too-many-arguments] @staticmethod def _setup_files(dest: Path, base: Path | None, content: dict[str, Any]) -> None: if base is not None: - shutil.copytree(str(base), str(dest), ignore=shutil.ignore_patterns(".tox")) + shutil.copytree(str(base), str(dest), ignore=shutil.ignore_patterns(".tox", ".python-envs")) dest.mkdir(exist_ok=True) for key, value in content.items(): if not isinstance(key, str): diff --git a/src/tox/session/cmd/run/common.py b/src/tox/session/cmd/run/common.py index 3909c9960..b543bf188 100644 --- a/src/tox/session/cmd/run/common.py +++ b/src/tox/session/cmd/run/common.py @@ -9,6 +9,7 @@ from concurrent.futures import FIRST_COMPLETED, CancelledError, Future, ThreadPoolExecutor from concurrent.futures import wait as wait_futures from fnmatch import fnmatchcase +from operator import itemgetter from pathlib import Path from signal import SIGINT, Handlers, signal from threading import Event, Thread @@ -25,6 +26,7 @@ from tox.session.cmd.run.single import ToxEnvRunResult, run_one from tox.tox_env.errors import Fail from tox.util.graph import stable_topological_sort +from tox.util.python_envs import record_python_envs from tox.util.spinner import MISS_DURATION, Spinner if TYPE_CHECKING: @@ -282,6 +284,8 @@ def _run_thread() -> tuple[Callable[[int, FrameType | None], Any] | int | Handle ordered_results = _order_results(state, results, to_run_list) # write the journal write_journal(state.conf.options.result_json, state._journal) # ruff:ignore[private-member-access] + # let editors discover the environments + _record_python_envs(state) # warn about unused config keys _warn_unused_config(state) # report the outcome @@ -313,6 +317,22 @@ def _order_results(state: State, results: list[ToxEnvRunResult], to_run_list: li return ordered +def _record_python_envs(state: State) -> None: + core = state.conf.core + if not core["python_envs"]: + return + ranked: list[tuple[tuple[bool, bool, int], Path]] = [] + for at, name in enumerate(state.envs.iter(only_active=False)): + env = state.envs[name] + if not (env.env_dir / "pyvenv.cfg").exists(): # not a Python environment a reader could use + continue + develop = "package" in env.conf and env.conf["package"] in {"editable", "editable-legacy"} + # the default environment goes last: prefer one named dev, then a develop install, then the env list order + ranked.append(((name == "dev", develop, -at), env.env_dir)) + envs = [env_dir for _, env_dir in sorted(ranked, key=itemgetter(0))] + record_python_envs(cast("Path", core["tox_root"]), cast("Path", core["work_dir"]), envs) + + class ToxSpinner(Spinner): def __init__(self, enabled: bool, state: State, total: int) -> None: # ruff:ignore[boolean-type-hint-positional-argument] stream = state._options.log_handler.stdout # ruff:ignore[private-member-access] diff --git a/src/tox/tox.schema.json b/src/tox/tox.schema.json index 85bbf6ea5..7a65e49ce 100644 --- a/src/tox/tox.schema.json +++ b/src/tox/tox.schema.json @@ -32,6 +32,10 @@ "type": "string", "description": "a folder for temporary files (is not cleaned at start)" }, + "python_envs": { + "type": "boolean", + "description": "catalog the created tox environments in a PEP-832 .python-envs file, so editors can discover them" + }, "env_list": { "type": "array", "items": { diff --git a/src/tox/tox_env/api.py b/src/tox/tox_env/api.py index 25eb5fcdb..cc93fcfce 100644 --- a/src/tox/tox_env/api.py +++ b/src/tox/tox_env/api.py @@ -17,6 +17,7 @@ from tox.tox_env.errors import Fail, Recreate, Skip from tox.tox_env.info import Info from tox.util.path import ensure_cachedir_tag, ensure_empty_dir, ensure_gitignore +from tox.util.python_envs import forget_python_env from tox.util.redact import redact_value if TYPE_CHECKING: @@ -347,6 +348,8 @@ def _clean(self, transitive: bool = False) -> None: # ruff:ignore[unused-method if env_dir.exists(): LOGGER.warning("remove tox env folder %s", env_dir) ensure_empty_dir(env_dir, except_filename="file.lock") + if self.core["python_envs"]: # drop it so nothing points at the environment while it rebuilds + forget_python_env(cast("Path", self.core["tox_root"]), cast("Path", self.core["work_dir"]), env_dir) self._log_id = 0 # we deleted logs, so start over counter self.cache.reset() self._run_state.update({"setup": False, "clean": True}) diff --git a/src/tox/util/python_envs.py b/src/tox/util/python_envs.py new file mode 100644 index 000000000..975955c43 --- /dev/null +++ b/src/tox/util/python_envs.py @@ -0,0 +1,71 @@ +"""Catalog the project environments in a :PEP:`832` ``.python-envs`` file.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from filelock import FileLock + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + from pathlib import Path + +_FILE_NAME: Final[str] = ".python-envs" +_LOCK_NAME: Final[str] = ".python-envs.lock" +_VENV_NAME: Final[str] = ".venv" + + +def record_python_envs(root: Path, work_dir: Path, envs: Sequence[Path]) -> None: + """Write the :PEP:`832` ``.python-envs`` catalog of the tox environments. + + Writes the entries in the given order, so the caller decides the default environment by putting it last. Lines + pointing outside *work_dir* come from another tool, so they stay as they are, and one of them sitting last keeps + that spot. Skips a ``.venv`` under *root*, :PEP:`832` already treats it as the implicit final entry. + + :param root: the directory holding the file, the project root + :param work_dir: the directory tox owns, *envs* replaces the lines under it + :param envs: the environment directories to catalog, least preferred first + + """ + keep = [e for e in envs if e != root / _VENV_NAME] + ours = set(keep) + _rewrite(root, work_dir, lambda path: path in ours or path.is_relative_to(work_dir), keep) + + +def forget_python_env(root: Path, work_dir: Path, env: Path) -> None: + """Drop an environment from the :PEP:`832` ``.python-envs`` catalog. + + Call this when the environment stops being usable, such as while tox recreates it, so that nothing points a reader + at a half-built environment. + + :param root: the directory holding the file, the project root + :param work_dir: the directory tox owns, hosts the lock guarding the file + :param env: the environment directory to drop + + """ + _rewrite(root, work_dir, lambda path: path == env, []) + + +def _rewrite(root: Path, work_dir: Path, is_ours: Callable[[Path], bool], envs: Sequence[Path]) -> None: + file = root / _FILE_NAME + if not (envs or file.exists()): + return + work_dir.mkdir(parents=True, exist_ok=True) + with FileLock(work_dir / _LOCK_NAME): + current = file.read_text(encoding="utf-8") if file.exists() else None + lines = [line for raw in (current or "").split("\n") if (line := raw.rstrip("\r"))] + kept = [line for line in lines if not is_ours(root / line)] + tail = [kept.pop()] if kept and lines[-1] == kept[-1] else [] + content = "".join(f"{line}\n" for line in [*kept, *(_as_line(e, root) for e in envs), *tail]) + if content != current: # rewriting identical content would churn the file for no gain + file.write_text(content, encoding="utf-8") + + +def _as_line(path: Path, root: Path) -> str: + return str(path.relative_to(root) if path.is_relative_to(root) else path) + + +__all__ = [ + "forget_python_env", + "record_python_envs", +] diff --git a/tests/session/cmd/run/test_python_envs.py b/tests/session/cmd/run/test_python_envs.py new file mode 100644 index 000000000..46c370a75 --- /dev/null +++ b/tests/session/cmd/run/test_python_envs.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + from tox.pytest import ToxProject, ToxProjectCreator + + +def _catalog(project: ToxProject) -> list[str]: + return (project.path / ".python-envs").read_text(encoding="utf-8").splitlines() + + +def test_python_envs_default_is_first_of_env_list(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a", "b" ]\nno_package = true\n'}) + + project.run("r", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}b", f".tox{os.sep}a"] + + +def test_python_envs_default_is_dev(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a", "dev" ]\nno_package = true\n'}) + + project.run("r", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}a", f".tox{os.sep}dev"] + + +def test_python_envs_default_is_editable(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + toml = 'env_list = [ "a", "b" ]\n[env.a]\npackage = "skip"\n[env.b]\npackage = "editable"\n' + project = tox_project({"tox.toml": toml}, base=demo_pkg_inline) + project.patch_execute(lambda request: 0 if "install" in request.run_id else None) + + project.run("r", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}a", f".tox{os.sep}b"] + + +def test_python_envs_skips_environment_not_created(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a", "b" ]\nno_package = true\n'}) + + project.run("r", "-e", "a", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}a"] + + +def test_python_envs_off(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a" ]\nno_package = true\npython_envs = false\n'}) + + project.run("r", "--notest").assert_success() + project.run("r", "-r", "--notest").assert_success() + + assert not (project.path / ".python-envs").exists() + + +@pytest.mark.parametrize("recreate", [pytest.param(True, id="recreate"), pytest.param(False, id="reuse")]) +def test_python_envs_forgotten_while_recreated(tox_project: ToxProjectCreator, recreate: bool) -> None: + project = tox_project({ + "tox.toml": 'env_list = [ "a" ]\nno_package = true\n[env_run_base]\ncommands = [ [ "python", "show.py" ] ]\n', + "show.py": """ + import pathlib + + catalog = pathlib.Path(".python-envs") + print("catalog:", *(catalog.read_text().split() if catalog.exists() else ())) + """, + }) + project.run("r").assert_success() + + outcome = project.run("r", *(["-r"] if recreate else [])) + + outcome.assert_success() + assert (f"catalog: .tox{os.sep}a" in outcome.out) is not recreate diff --git a/tests/util/test_python_envs.py b/tests/util/test_python_envs.py new file mode 100644 index 000000000..4845a03b8 --- /dev/null +++ b/tests/util/test_python_envs.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from tox.util.python_envs import forget_python_env, record_python_envs + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + (tmp_path / ".tox").mkdir() + return tmp_path + + +def _catalog(root: Path) -> str: + return (root / ".python-envs").read_text(encoding="utf-8") + + +def test_record_creates_catalog(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13", root / ".tox" / "dev"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n.tox{os.sep}dev\n" + + +def test_record_keeps_env_outside_root_absolute(root: Path, tmp_path_factory: pytest.TempPathFactory) -> None: + outside = tmp_path_factory.mktemp("elsewhere") / "dev" + + record_python_envs(root, root / ".tox", [outside]) + + assert _catalog(root) == f"{outside}\n" + + +def test_record_skips_dot_venv(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13", root / ".venv"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_record_without_envs_creates_nothing(root: Path) -> None: + record_python_envs(root, root / ".tox", []) + + assert not (root / ".python-envs").exists() + + +@pytest.mark.parametrize( + "existing", + [ + pytest.param("../shared\n", id="trailing_newline"), + pytest.param("../shared", id="no_trailing_newline"), + pytest.param("../shared\r\n", id="crlf"), + pytest.param("../shared\n\n", id="blank_line"), + ], +) +def test_record_keeps_foreign_line(root: Path, existing: str) -> None: + (root / ".python-envs").write_text(existing, encoding="utf-8") + + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n../shared\n" + + +def test_record_leaves_foreign_default_last(root: Path) -> None: + (root / ".python-envs").write_text("../first\n../default\n", encoding="utf-8") + + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + + assert _catalog(root) == f"../first\n.tox{os.sep}dev\n../default\n" + + +def test_record_prunes_envs_no_longer_known(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.12", root / ".tox" / "3.13"]) + + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_record_does_not_duplicate_hand_written_env(root: Path) -> None: + (root / ".python-envs").write_text(f".tox{os.sep}dev\n../shared\n", encoding="utf-8") + + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + + assert _catalog(root) == f".tox{os.sep}dev\n../shared\n" + + +def test_record_unchanged_does_not_write(root: Path, mocker: MockerFixture) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + write_text = mocker.spy(Path, "write_text") + + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + + assert write_text.call_count == 0 + + +def test_forget_drops_env(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13", root / ".tox" / "dev"]) + + forget_python_env(root, root / ".tox", root / ".tox" / "dev") + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_forget_leaves_other_envs(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13"]) + + forget_python_env(root, root / ".tox", root / ".tox" / "dev") + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_forget_without_catalog(root: Path) -> None: + forget_python_env(root, root / ".tox", root / ".tox" / "dev") + + assert not (root / ".python-envs").exists()