From c823579ebf6a4f65472efe287b3f547f22ef05fc Mon Sep 17 00:00:00 2001 From: Bas Nijholt Date: Thu, 28 Aug 2025 12:04:38 -0700 Subject: [PATCH 1/3] test: add test for duplicate local dependencies bug This test demonstrates a bug where local dependencies that are referenced by multiple projects appear multiple times in the pip install command. The test currently fails, showing that the same dependency is added 3 times instead of once. Related to CI issue: https://github.com/ionq/system_performance/actions/runs/17304244901/job/49123013838?pr=925 --- tests/test_cli.py | 131 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index dd299553..e72b2f82 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -691,3 +691,134 @@ def fake_env_name_to_prefix( # Optionally, verify that our fake function printed the expected message. output = capsys.readouterr().out assert "Fake create called with" in output + + +def test_no_duplicate_local_dependencies_in_install_command(tmp_path: Path) -> None: + """Test that local dependencies are not duplicated in the pip install command. + + This test reproduces the issue where the same local dependency appears + multiple times in the pip install command when it's referenced by multiple + projects. + """ + # Create a shared local dependency that will be referenced multiple times + shared_dep = tmp_path / "shared_dependency" + shared_dep.mkdir() + (shared_dep / "setup.py").write_text( + textwrap.dedent( + """\ + from setuptools import setup + setup(name="shared_dep", version="0.1.0") + """ + ) + ) + + # Create multiple projects that all reference the same shared dependency + projects = [] + for i in range(3): + project = tmp_path / f"project_{i}" + project.mkdir() + + # Create a requirements.yaml that references the shared dependency + (project / "requirements.yaml").write_text( + textwrap.dedent( + f"""\ + name: project_{i} + dependencies: + - numpy + local_dependencies: + - ../shared_dependency + """ + ) + ) + + # Create a minimal setup.py to make it pip installable + (project / "setup.py").write_text( + textwrap.dedent( + f"""\ + from setuptools import setup + setup(name="project_{i}", version="0.1.0") + """ + ) + ) + + projects.append(project / "requirements.yaml") + + # Mock subprocess.run to capture the pip install commands + pip_install_commands = [] + + def mock_run(cmd, *args, **kwargs): + # Capture pip install commands with -e flags + if isinstance(cmd, list) and "pip" in str(cmd) and "install" in cmd: + # Look for -e flags in the command + editable_packages = [] + i = 0 + while i < len(cmd): + if cmd[i] == "-e" and i + 1 < len(cmd): + editable_packages.append(cmd[i + 1]) + i += 2 + else: + i += 1 + if editable_packages: + pip_install_commands.append(editable_packages) + + # Don't actually run the command in tests + from unittest.mock import MagicMock + result = MagicMock() + result.returncode = 0 + return result + + import warnings + + with patch("subprocess.run", side_effect=mock_run): + # Run the install command with all projects + # Expect a warning about unmanaged local dependency + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + _install_command( + *projects, + conda_executable=None, + conda_env_name=None, + conda_env_prefix=None, + conda_lock_file=None, + dry_run=False, + editable=True, + skip_local=False, + skip_pip=True, # Skip regular pip deps to focus on local deps + skip_conda=True, # Skip conda deps + no_dependencies=False, + ignore_pins=None, + overwrite_pins=None, + skip_dependencies=None, + no_uv=True, + verbose=False, + ) + + # Check that the shared dependency appears only once in pip install commands + all_editable_packages = [] + for packages in pip_install_commands: + all_editable_packages.extend(packages) + + # Count how many times the shared_dependency appears + shared_dep_str = str(shared_dep.resolve()) + shared_dep_count = sum( + 1 for pkg in all_editable_packages + if str(Path(pkg).resolve()) == shared_dep_str + ) + + # The shared dependency should appear exactly once, not multiple times + assert shared_dep_count == 1, ( + f"Expected shared_dependency to appear once in pip install command, " + f"but it appeared {shared_dep_count} times. " + f"All editable packages: {all_editable_packages}" + ) + + # Also check that each project appears exactly once + for i in range(3): + project_path = str((tmp_path / f"project_{i}").resolve()) + project_count = sum( + 1 for pkg in all_editable_packages + if str(Path(pkg).resolve()) == project_path + ) + assert project_count == 1, ( + f"Expected project_{i} to appear once, but it appeared {project_count} times" + ) From 35051d680a3b25f9f10240ca30d8a7faa2c8b200 Mon Sep 17 00:00:00 2001 From: Bas Nijholt Date: Thu, 28 Aug 2025 12:05:27 -0700 Subject: [PATCH 2/3] fix: prevent duplicate local dependencies in pip install command When multiple projects reference the same local dependency, the deduplication logic was not properly preventing duplicates. The installable_set was created once but not updated as new dependencies were added, causing the same dependency to appear multiple times. This fix updates the installable_set each time a new dependency is added, ensuring proper deduplication. Fixes the issue where packages like third_party/cypress were being repeated 9 times in the pip install command in CI. --- unidep/_cli.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/unidep/_cli.py b/unidep/_cli.py index e820d6e1..32fef3e9 100755 --- a/unidep/_cli.py +++ b/unidep/_cli.py @@ -1065,12 +1065,12 @@ def _install_command( # noqa: PLR0912, PLR0915 names = {k.name: [dep.name for dep in v] for k, v in local_dependencies.items()} print(f"📝 Found local dependencies: {names}\n") installable_set = {p.resolve() for p in installable} - installable += [ - dep - for deps in local_dependencies.values() - for dep in deps - if dep.resolve() not in installable_set - ] + for deps in local_dependencies.values(): + for dep in deps: + dep_resolved = dep.resolve() + if dep_resolved not in installable_set: + installable.append(dep) + installable_set.add(dep_resolved) if installable: pip_flags = ["--no-deps"] # we just ran pip/conda install, so skip if verbose: From 09dfdc841146631d715aa75d7591b24647d6be53 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 28 Aug 2025 19:06:35 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_cli.py | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index e72b2f82..e7c1af14 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -695,7 +695,7 @@ def fake_env_name_to_prefix( def test_no_duplicate_local_dependencies_in_install_command(tmp_path: Path) -> None: """Test that local dependencies are not duplicated in the pip install command. - + This test reproduces the issue where the same local dependency appears multiple times in the pip install command when it's referenced by multiple projects. @@ -708,16 +708,16 @@ def test_no_duplicate_local_dependencies_in_install_command(tmp_path: Path) -> N """\ from setuptools import setup setup(name="shared_dep", version="0.1.0") - """ - ) + """, + ), ) - + # Create multiple projects that all reference the same shared dependency projects = [] for i in range(3): project = tmp_path / f"project_{i}" project.mkdir() - + # Create a requirements.yaml that references the shared dependency (project / "requirements.yaml").write_text( textwrap.dedent( @@ -727,25 +727,25 @@ def test_no_duplicate_local_dependencies_in_install_command(tmp_path: Path) -> N - numpy local_dependencies: - ../shared_dependency - """ - ) + """, + ), ) - + # Create a minimal setup.py to make it pip installable (project / "setup.py").write_text( textwrap.dedent( f"""\ from setuptools import setup setup(name="project_{i}", version="0.1.0") - """ - ) + """, + ), ) - + projects.append(project / "requirements.yaml") - + # Mock subprocess.run to capture the pip install commands pip_install_commands = [] - + def mock_run(cmd, *args, **kwargs): # Capture pip install commands with -e flags if isinstance(cmd, list) and "pip" in str(cmd) and "install" in cmd: @@ -760,15 +760,16 @@ def mock_run(cmd, *args, **kwargs): i += 1 if editable_packages: pip_install_commands.append(editable_packages) - + # Don't actually run the command in tests from unittest.mock import MagicMock + result = MagicMock() result.returncode = 0 return result - + import warnings - + with patch("subprocess.run", side_effect=mock_run): # Run the install command with all projects # Expect a warning about unmanaged local dependency @@ -792,31 +793,31 @@ def mock_run(cmd, *args, **kwargs): no_uv=True, verbose=False, ) - + # Check that the shared dependency appears only once in pip install commands all_editable_packages = [] for packages in pip_install_commands: all_editable_packages.extend(packages) - + # Count how many times the shared_dependency appears shared_dep_str = str(shared_dep.resolve()) shared_dep_count = sum( - 1 for pkg in all_editable_packages - if str(Path(pkg).resolve()) == shared_dep_str + 1 for pkg in all_editable_packages if str(Path(pkg).resolve()) == shared_dep_str ) - + # The shared dependency should appear exactly once, not multiple times assert shared_dep_count == 1, ( f"Expected shared_dependency to appear once in pip install command, " f"but it appeared {shared_dep_count} times. " f"All editable packages: {all_editable_packages}" ) - + # Also check that each project appears exactly once for i in range(3): project_path = str((tmp_path / f"project_{i}").resolve()) project_count = sum( - 1 for pkg in all_editable_packages + 1 + for pkg in all_editable_packages if str(Path(pkg).resolve()) == project_path ) assert project_count == 1, (