diff --git a/.gitignore b/.gitignore index 8654da708..660a249e7 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ __pycache__ /tests/demo_pkg_setuptools/build/lib/demo_pkg_setuptools/__init__.py /tests/demo_pkg_inline.lock /tests/demo_pkg_inline/.tox/ +.python-envs diff --git a/docs/changelog/4013.feature.rst b/docs/changelog/4013.feature.rst new file mode 100644 index 000000000..f8ff5f7f1 --- /dev/null +++ b/docs/changelog/4013.feature.rst @@ -0,0 +1,6 @@ +Catalog the created tox environments in a :PEP:`832` ``.python-envs`` file next to the configuration file, so that +editors such as VS Code or PyCharm offer them as interpreters instead of asking you for a path under ``.tox``. The +default environment comes last, and tox prefers one named ``dev`` for that spot, then one that installs the project in +development mode, then the first of :ref:`env_list`. tox leaves the lines other tools wrote alone, and takes an +environment out of the catalog while it recreates it. Set :ref:`python_envs` to ``false`` to skip the file - by +:user:`gaborbernat`. (:issue:`4013`) diff --git a/docs/explanation.rst b/docs/explanation.rst index 3f53e9b1e..9578f43f1 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -721,6 +721,31 @@ upgrade that changes the derived value) triggers automatic recreation. This design mirrors tox's own auto-provisioning mechanism (``requires`` / ``min_version``), where tox bootstraps itself into a separate environment when the running installation doesn't meet the declared requirements. +*********************** + Environment discovery +*********************** + +Editors need an interpreter path before they can offer completion, navigation or a debugger. tox keeps its environments +under ``.tox``, a directory editors have no reason to search, so for years the answer was to copy a path out of ``tox +devenv`` output and paste it into a settings dialog, then repeat it after every recreate. + +:PEP:`832` standardizes where tools publish that answer. A project may hold a virtual environment at ``.venv``, and a +``.python-envs`` file at the project root lists any further environments, one directory per line. The last line is the +default, so a reader that supports only one environment still knows which to take. VS Code reads the file today and +PyCharm honors the ``.venv`` half of the convention. + +tox writes ``.python-envs`` at the end of a run, listing every environment that exists on disk. Ordering follows how +useful an environment is to an editor rather than how tox happens to schedule it: an environment named ``dev`` wins, +then one installing the project in development mode, then the earlier entries of :ref:`env_list`. Since the catalog +lists the preferred environment last, removing lines from the bottom degrades to the next best choice. + +Two rules keep the file honest. tox owns the lines under :ref:`work_dir` and rewrites them wholesale, while lines +pointing elsewhere came from another tool and survive untouched, including the last line and the default it claims. And +because a recreated environment is unusable between the moment tox empties the directory and the moment the new +interpreter lands, tox takes the environment out of the catalog first and puts it back once the run ends. + +Set :ref:`python_envs` to ``false`` if you would rather tox left the project root alone. + ******************* Known limitations ******************* diff --git a/docs/how-to/usage.rst b/docs/how-to/usage.rst index 948c9ceae..5d1f420c7 100644 --- a/docs/how-to/usage.rst +++ b/docs/how-to/usage.rst @@ -162,6 +162,44 @@ The ``tox exec`` subcommand runs an arbitrary command inside a tox environment w The command must be in the environment's ``PATH`` or listed in :ref:`allowlist_externals`. ``tox exec`` is useful for debugging, running one-off scripts, or interactively exploring an environment without modifying your configuration. +.. _howto_editor_env: + +*************************************** + Open a tox environment in your editor +*************************************** + +Editors pick up interpreters through the :PEP:`832` ``.python-envs`` file, which tox writes next to your configuration +file after each run. Name the environment you develop against ``dev`` and tox puts it on the last line, the one readers +treat as the default: + +.. code-block:: toml + + [env.dev] + description = "development environment" + package = "editable" + dependency_groups = [ "dev" ] + +Create it, then reload your editor window: + +.. code-block:: bash + + tox run -e dev --notest + +The file lists one directory per line, so you can check what an editor will see: + +.. code-block:: bash + + $ cat .python-envs + .tox/3.13 + .tox/dev + +Commit the file when your team keeps environments in the same place, otherwise add it to your ``.gitignore``. To keep +tox from writing it at all, set :ref:`python_envs` in the core section: + +.. code-block:: toml + + python_envs = false + .. ------------------------------------------------------------------------------------------ .. Configuration (frequently needed) diff --git a/docs/reference/config.rst b/docs/reference/config.rst index 4487ef0fe..2374ca42f 100644 --- a/docs/reference/config.rst +++ b/docs/reference/config.rst @@ -539,6 +539,21 @@ the top level of ``tox.toml``. Placing these options in an environment section ( this directory for the project package. This ensures tox works correctly when having parallel runs (as each session will have its own copy of the project package - e.g. the source distribution). +.. conf:: + :keys: python_envs + :default: true + :version_added: 4.59.0 + + Catalog the created tox environments in a :pep:`832` ``.python-envs`` file next to the configuration file, so that + editors such as VS Code or PyCharm offer them as interpreters. The file holds one environment directory per line, + with the default one last. tox prefers an environment named ``dev`` for that spot, then one that installs the + project in development mode, then the first entry of :ref:`env_list`. It lists only environments that exist, and + takes one out of the catalog while it recreates it. + + Lines pointing outside the :ref:`work_dir` come from another tool, so tox keeps them, including their claim on the + last line. Commit the file when everyone working on the project has their environments in the same place, otherwise + ignore it. + .. conf:: :keys: no_package, skipsdist :default: false diff --git a/docs/tutorial/getting-started.rst b/docs/tutorial/getting-started.rst index ecb47768b..885a06940 100644 --- a/docs/tutorial/getting-started.rst +++ b/docs/tutorial/getting-started.rst @@ -287,6 +287,23 @@ has changed), use ``--skip-env-install``: tox run -e 3.13 --skip-env-install +****************************** + Using an environment at hand +****************************** + +After a run, tox records the environments it created in a ``.python-envs`` file next to your configuration, following +:PEP:`832`. Editors such as VS Code read it and offer those interpreters, so you can pick one instead of hunting for a +path under ``.tox``: + +.. code-block:: bash + + $ tox run -e 3.13 + $ cat .python-envs + .tox/3.13 + +The last line is the default. Name an environment ``dev`` and tox puts it there, which is what you want for the +environment you edit code against - see :ref:`howto_editor_env`. + ******************************** Listing available environments ******************************** diff --git a/src/tox/config/sets.py b/src/tox/config/sets.py index f26edcb06..2c1dd5304 100644 --- a/src/tox/config/sets.py +++ b/src/tox/config/sets.py @@ -315,6 +315,12 @@ def register_config(self) -> None: default=self._default_temp_dir, desc="a folder for temporary files (is not cleaned at start)", ) + self.add_config( + keys=["python_envs"], + of_type=bool, + default=True, + desc="catalog the created tox environments in a PEP-832 .python-envs file, so editors can discover them", + ) self.add_constant("host_python", "the host python executable path", sys.executable) def _on_duplicate_conf(self, key: str, definition: ConfigDefinition[V]) -> None: diff --git a/src/tox/pytest.py b/src/tox/pytest.py index 12befc389..bf449b026 100644 --- a/src/tox/pytest.py +++ b/src/tox/pytest.py @@ -147,7 +147,7 @@ def __init__( # ruff:ignore[too-many-arguments] @staticmethod def _setup_files(dest: Path, base: Path | None, content: dict[str, Any]) -> None: if base is not None: - shutil.copytree(str(base), str(dest), ignore=shutil.ignore_patterns(".tox")) + shutil.copytree(str(base), str(dest), ignore=shutil.ignore_patterns(".tox", ".python-envs")) dest.mkdir(exist_ok=True) for key, value in content.items(): if not isinstance(key, str): diff --git a/src/tox/session/cmd/run/common.py b/src/tox/session/cmd/run/common.py index 3909c9960..b543bf188 100644 --- a/src/tox/session/cmd/run/common.py +++ b/src/tox/session/cmd/run/common.py @@ -9,6 +9,7 @@ from concurrent.futures import FIRST_COMPLETED, CancelledError, Future, ThreadPoolExecutor from concurrent.futures import wait as wait_futures from fnmatch import fnmatchcase +from operator import itemgetter from pathlib import Path from signal import SIGINT, Handlers, signal from threading import Event, Thread @@ -25,6 +26,7 @@ from tox.session.cmd.run.single import ToxEnvRunResult, run_one from tox.tox_env.errors import Fail from tox.util.graph import stable_topological_sort +from tox.util.python_envs import record_python_envs from tox.util.spinner import MISS_DURATION, Spinner if TYPE_CHECKING: @@ -282,6 +284,8 @@ def _run_thread() -> tuple[Callable[[int, FrameType | None], Any] | int | Handle ordered_results = _order_results(state, results, to_run_list) # write the journal write_journal(state.conf.options.result_json, state._journal) # ruff:ignore[private-member-access] + # let editors discover the environments + _record_python_envs(state) # warn about unused config keys _warn_unused_config(state) # report the outcome @@ -313,6 +317,22 @@ def _order_results(state: State, results: list[ToxEnvRunResult], to_run_list: li return ordered +def _record_python_envs(state: State) -> None: + core = state.conf.core + if not core["python_envs"]: + return + ranked: list[tuple[tuple[bool, bool, int], Path]] = [] + for at, name in enumerate(state.envs.iter(only_active=False)): + env = state.envs[name] + if not (env.env_dir / "pyvenv.cfg").exists(): # not a Python environment a reader could use + continue + develop = "package" in env.conf and env.conf["package"] in {"editable", "editable-legacy"} + # the default environment goes last: prefer one named dev, then a develop install, then the env list order + ranked.append(((name == "dev", develop, -at), env.env_dir)) + envs = [env_dir for _, env_dir in sorted(ranked, key=itemgetter(0))] + record_python_envs(cast("Path", core["tox_root"]), cast("Path", core["work_dir"]), envs) + + class ToxSpinner(Spinner): def __init__(self, enabled: bool, state: State, total: int) -> None: # ruff:ignore[boolean-type-hint-positional-argument] stream = state._options.log_handler.stdout # ruff:ignore[private-member-access] diff --git a/src/tox/tox.schema.json b/src/tox/tox.schema.json index 85bbf6ea5..7a65e49ce 100644 --- a/src/tox/tox.schema.json +++ b/src/tox/tox.schema.json @@ -32,6 +32,10 @@ "type": "string", "description": "a folder for temporary files (is not cleaned at start)" }, + "python_envs": { + "type": "boolean", + "description": "catalog the created tox environments in a PEP-832 .python-envs file, so editors can discover them" + }, "env_list": { "type": "array", "items": { diff --git a/src/tox/tox_env/api.py b/src/tox/tox_env/api.py index 25eb5fcdb..cc93fcfce 100644 --- a/src/tox/tox_env/api.py +++ b/src/tox/tox_env/api.py @@ -17,6 +17,7 @@ from tox.tox_env.errors import Fail, Recreate, Skip from tox.tox_env.info import Info from tox.util.path import ensure_cachedir_tag, ensure_empty_dir, ensure_gitignore +from tox.util.python_envs import forget_python_env from tox.util.redact import redact_value if TYPE_CHECKING: @@ -347,6 +348,8 @@ def _clean(self, transitive: bool = False) -> None: # ruff:ignore[unused-method if env_dir.exists(): LOGGER.warning("remove tox env folder %s", env_dir) ensure_empty_dir(env_dir, except_filename="file.lock") + if self.core["python_envs"]: # drop it so nothing points at the environment while it rebuilds + forget_python_env(cast("Path", self.core["tox_root"]), cast("Path", self.core["work_dir"]), env_dir) self._log_id = 0 # we deleted logs, so start over counter self.cache.reset() self._run_state.update({"setup": False, "clean": True}) diff --git a/src/tox/util/python_envs.py b/src/tox/util/python_envs.py new file mode 100644 index 000000000..975955c43 --- /dev/null +++ b/src/tox/util/python_envs.py @@ -0,0 +1,71 @@ +"""Catalog the project environments in a :PEP:`832` ``.python-envs`` file.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Final + +from filelock import FileLock + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + from pathlib import Path + +_FILE_NAME: Final[str] = ".python-envs" +_LOCK_NAME: Final[str] = ".python-envs.lock" +_VENV_NAME: Final[str] = ".venv" + + +def record_python_envs(root: Path, work_dir: Path, envs: Sequence[Path]) -> None: + """Write the :PEP:`832` ``.python-envs`` catalog of the tox environments. + + Writes the entries in the given order, so the caller decides the default environment by putting it last. Lines + pointing outside *work_dir* come from another tool, so they stay as they are, and one of them sitting last keeps + that spot. Skips a ``.venv`` under *root*, :PEP:`832` already treats it as the implicit final entry. + + :param root: the directory holding the file, the project root + :param work_dir: the directory tox owns, *envs* replaces the lines under it + :param envs: the environment directories to catalog, least preferred first + + """ + keep = [e for e in envs if e != root / _VENV_NAME] + ours = set(keep) + _rewrite(root, work_dir, lambda path: path in ours or path.is_relative_to(work_dir), keep) + + +def forget_python_env(root: Path, work_dir: Path, env: Path) -> None: + """Drop an environment from the :PEP:`832` ``.python-envs`` catalog. + + Call this when the environment stops being usable, such as while tox recreates it, so that nothing points a reader + at a half-built environment. + + :param root: the directory holding the file, the project root + :param work_dir: the directory tox owns, hosts the lock guarding the file + :param env: the environment directory to drop + + """ + _rewrite(root, work_dir, lambda path: path == env, []) + + +def _rewrite(root: Path, work_dir: Path, is_ours: Callable[[Path], bool], envs: Sequence[Path]) -> None: + file = root / _FILE_NAME + if not (envs or file.exists()): + return + work_dir.mkdir(parents=True, exist_ok=True) + with FileLock(work_dir / _LOCK_NAME): + current = file.read_text(encoding="utf-8") if file.exists() else None + lines = [line for raw in (current or "").split("\n") if (line := raw.rstrip("\r"))] + kept = [line for line in lines if not is_ours(root / line)] + tail = [kept.pop()] if kept and lines[-1] == kept[-1] else [] + content = "".join(f"{line}\n" for line in [*kept, *(_as_line(e, root) for e in envs), *tail]) + if content != current: # rewriting identical content would churn the file for no gain + file.write_text(content, encoding="utf-8") + + +def _as_line(path: Path, root: Path) -> str: + return str(path.relative_to(root) if path.is_relative_to(root) else path) + + +__all__ = [ + "forget_python_env", + "record_python_envs", +] diff --git a/tests/session/cmd/run/test_python_envs.py b/tests/session/cmd/run/test_python_envs.py new file mode 100644 index 000000000..46c370a75 --- /dev/null +++ b/tests/session/cmd/run/test_python_envs.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import os +from typing import TYPE_CHECKING + +import pytest + +if TYPE_CHECKING: + from pathlib import Path + + from tox.pytest import ToxProject, ToxProjectCreator + + +def _catalog(project: ToxProject) -> list[str]: + return (project.path / ".python-envs").read_text(encoding="utf-8").splitlines() + + +def test_python_envs_default_is_first_of_env_list(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a", "b" ]\nno_package = true\n'}) + + project.run("r", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}b", f".tox{os.sep}a"] + + +def test_python_envs_default_is_dev(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a", "dev" ]\nno_package = true\n'}) + + project.run("r", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}a", f".tox{os.sep}dev"] + + +def test_python_envs_default_is_editable(tox_project: ToxProjectCreator, demo_pkg_inline: Path) -> None: + toml = 'env_list = [ "a", "b" ]\n[env.a]\npackage = "skip"\n[env.b]\npackage = "editable"\n' + project = tox_project({"tox.toml": toml}, base=demo_pkg_inline) + project.patch_execute(lambda request: 0 if "install" in request.run_id else None) + + project.run("r", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}a", f".tox{os.sep}b"] + + +def test_python_envs_skips_environment_not_created(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a", "b" ]\nno_package = true\n'}) + + project.run("r", "-e", "a", "--notest").assert_success() + + assert _catalog(project) == [f".tox{os.sep}a"] + + +def test_python_envs_off(tox_project: ToxProjectCreator) -> None: + project = tox_project({"tox.toml": 'env_list = [ "a" ]\nno_package = true\npython_envs = false\n'}) + + project.run("r", "--notest").assert_success() + project.run("r", "-r", "--notest").assert_success() + + assert not (project.path / ".python-envs").exists() + + +@pytest.mark.parametrize("recreate", [pytest.param(True, id="recreate"), pytest.param(False, id="reuse")]) +def test_python_envs_forgotten_while_recreated(tox_project: ToxProjectCreator, recreate: bool) -> None: + project = tox_project({ + "tox.toml": 'env_list = [ "a" ]\nno_package = true\n[env_run_base]\ncommands = [ [ "python", "show.py" ] ]\n', + "show.py": """ + import pathlib + + catalog = pathlib.Path(".python-envs") + print("catalog:", *(catalog.read_text().split() if catalog.exists() else ())) + """, + }) + project.run("r").assert_success() + + outcome = project.run("r", *(["-r"] if recreate else [])) + + outcome.assert_success() + assert (f"catalog: .tox{os.sep}a" in outcome.out) is not recreate diff --git a/tests/util/test_python_envs.py b/tests/util/test_python_envs.py new file mode 100644 index 000000000..4845a03b8 --- /dev/null +++ b/tests/util/test_python_envs.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + +from tox.util.python_envs import forget_python_env, record_python_envs + +if TYPE_CHECKING: + from pytest_mock import MockerFixture + + +@pytest.fixture +def root(tmp_path: Path) -> Path: + (tmp_path / ".tox").mkdir() + return tmp_path + + +def _catalog(root: Path) -> str: + return (root / ".python-envs").read_text(encoding="utf-8") + + +def test_record_creates_catalog(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13", root / ".tox" / "dev"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n.tox{os.sep}dev\n" + + +def test_record_keeps_env_outside_root_absolute(root: Path, tmp_path_factory: pytest.TempPathFactory) -> None: + outside = tmp_path_factory.mktemp("elsewhere") / "dev" + + record_python_envs(root, root / ".tox", [outside]) + + assert _catalog(root) == f"{outside}\n" + + +def test_record_skips_dot_venv(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13", root / ".venv"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_record_without_envs_creates_nothing(root: Path) -> None: + record_python_envs(root, root / ".tox", []) + + assert not (root / ".python-envs").exists() + + +@pytest.mark.parametrize( + "existing", + [ + pytest.param("../shared\n", id="trailing_newline"), + pytest.param("../shared", id="no_trailing_newline"), + pytest.param("../shared\r\n", id="crlf"), + pytest.param("../shared\n\n", id="blank_line"), + ], +) +def test_record_keeps_foreign_line(root: Path, existing: str) -> None: + (root / ".python-envs").write_text(existing, encoding="utf-8") + + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n../shared\n" + + +def test_record_leaves_foreign_default_last(root: Path) -> None: + (root / ".python-envs").write_text("../first\n../default\n", encoding="utf-8") + + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + + assert _catalog(root) == f"../first\n.tox{os.sep}dev\n../default\n" + + +def test_record_prunes_envs_no_longer_known(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.12", root / ".tox" / "3.13"]) + + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13"]) + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_record_does_not_duplicate_hand_written_env(root: Path) -> None: + (root / ".python-envs").write_text(f".tox{os.sep}dev\n../shared\n", encoding="utf-8") + + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + + assert _catalog(root) == f".tox{os.sep}dev\n../shared\n" + + +def test_record_unchanged_does_not_write(root: Path, mocker: MockerFixture) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + write_text = mocker.spy(Path, "write_text") + + record_python_envs(root, root / ".tox", [root / ".tox" / "dev"]) + + assert write_text.call_count == 0 + + +def test_forget_drops_env(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13", root / ".tox" / "dev"]) + + forget_python_env(root, root / ".tox", root / ".tox" / "dev") + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_forget_leaves_other_envs(root: Path) -> None: + record_python_envs(root, root / ".tox", [root / ".tox" / "3.13"]) + + forget_python_env(root, root / ".tox", root / ".tox" / "dev") + + assert _catalog(root) == f".tox{os.sep}3.13\n" + + +def test_forget_without_catalog(root: Path) -> None: + forget_python_env(root, root / ".tox", root / ".tox" / "dev") + + assert not (root / ".python-envs").exists()