Refactor eco testing

Author: ssbarnea

.github/workflows/tox.yml

@@ -41,7 +41,6 @@ jobs:
             hook
             docs
             schemas
-            eco
             pre
             py311-devel
             py310-lower
@@ -110,6 +109,7 @@ jobs:
         with:
           path: |
             .cache/eco
+            test/fixtures/eco/*/.tox
             examples/playbooks/collections/ansible_collections
             ~/.cache/ansible-compat
             ~/.ansible/collections

.gitignore

@@ -82,3 +82,4 @@ node_modules
 .direnv
 uv.lock
 .ansible
+test/fixtures/eco

.gitmodules

@@ -0,0 +1,21 @@
+[submodule "test/fixtures/eco/bootstrap"]
+	path = test/fixtures/eco/bootstrap
+	url = https://github.com/robertdebock/ansible-role-bootstrap
+[submodule "test/fixtures/eco/cisco.nxos"]
+	path = test/fixtures/eco/cisco.nxos
+	url = https://github.com/ansible-collections/cisco.nxos
+[submodule "test/fixtures/eco/ansible_collection_system"]
+	path = test/fixtures/eco/ansible_collection_system
+	url = https://github.com/devroles/ansible_collection_system
+[submodule "test/fixtures/eco/ansible-docker-rootless"]
+	path = test/fixtures/eco/ansible-docker-rootless
+	url = https://github.com/konstruktoid/ansible-docker-rootless
+[submodule "test/fixtures/eco/ansible-role-hardening"]
+	path = test/fixtures/eco/ansible-role-hardening
+	url = https://github.com/konstruktoid/ansible-role-hardening
+[submodule "test/fixtures/eco/ansible-role-mysql"]
+	path = test/fixtures/eco/ansible-role-mysql
+	url = https://github.com/geerlingguy/ansible-role-mysql.git
+[submodule "test/fixtures/eco/debops"]
+	path = test/fixtures/eco/debops
+	url = https://github.com/debops/debops

.pre-commit-config.yaml

@@ -22,6 +22,7 @@ exclude: >
     examples/playbooks/vars/empty.transformed.yml|
     examples/playbooks/vars/empty.yml|
     src/ansiblelint/schemas/rulebook.json|
+    test/fixtures/eco/.*|
     test/schemas/data/licenses.json|
     test/schemas/negative_test|
     test/schemas/package-lock.json

codecov.yml

@@ -2,7 +2,6 @@
 comment: false
 coverage:
   status:
-    patch: true
+    patch: false # avoid false positives
     project:
-      default:
-        threshold: 0.5%
+      default: true

cspell.config.yaml

@@ -19,6 +19,7 @@ ignorePaths:
   - docs/requirements.in
   # Test fixtures generated from outside
   - test/**/*.result
+  - test/fixtures/eco/**
   - src/ansiblelint/schemas/*.json
   # Other
   - "*.svg"

pyproject.toml

@@ -51,7 +51,7 @@ repository = "https://github.com/ansible/ansible-lint"
 [tool.codespell]
 # indention is a typo in ruamel.yaml's API
 ignore-words-list = "indention"
-skip = ".tox,.mypy_cache,build,.git,.eggs,pip-wheel-metadata"
+skip = ".tox,.mypy_cache,build,.git,.eggs,pip-wheel-metadata,test/fixtures/eco/*.json"
 
 # Keep this default because xml/report do not know to use load it from config file:
 # data_file = ".coverage"
@@ -86,7 +86,7 @@ disallow_untyped_calls = true
 disallow_untyped_defs = true
 error_summary = true
 # site-packages is here to help vscode mypy integration getting confused
-exclude = "(.cache|.config|.eggs|.tox|_readthedocs|build|dist|test/local-content|site|site-packages|~/.pyenv|examples|plugins/modules).*"
+exclude = "(.cache|.config|.eggs|.tox|_readthedocs|build|dist|test/local-content|site|site-packages|~/.pyenv|examples|plugins/modules|test/fixtures/eco).*"
 # https://github.com/python/mypy/issues/12664
 follow_untyped_imports = true
 incremental = false
@@ -392,7 +392,7 @@ reportPrivateImportUsage = false
 [tool.pytest.ini_options]
 # do not add options here as this will likely break either console runs or IDE
 # integration like vscode or pycharm
-addopts = "-p no:pytest_cov --durations=10 --failed-first"
+addopts = "-p no:pytest_cov --durations=10 --failed-first -m 'not eco'"
 doctest_optionflags = ["ALLOW_UNICODE", "ELLIPSIS"]
 filterwarnings = [
   "error",
@@ -409,6 +409,9 @@ filterwarnings = [
   "ignore:Attribute s is deprecated and will be removed in Python 3.14; use value instead:DeprecationWarning"
 ]
 junit_family = "legacy"
+markers = [
+  "eco: test impact on ecosystem repositories (slow)"
+]
 minversion = "4.6.6"
 # https://code.visualstudio.com/docs/python/testing
 # coverage is re-enabled in `tox.ini`. That approach is safer than
@@ -435,6 +438,7 @@ norecursedirs = [
   "plugins",
   "site",
   "src/ansible_lint.egg-info",
+  "test/fixtures/eco",
   "test/schemas"
 ]
 python_files = [
@@ -574,6 +578,7 @@ exclude = [
   "collections",
   "examples",
   "plugins",
+  "test/fixtures/eco",
   "test/local-content",
   "venv",
   "src/ansiblelint/_version.py"