Enable absolute % coverage enforcement (Azure#39696)

* enable absolute coverage enforcement. honoring absolute_cov and absolute_cov_percent keys respectively. defaults to 95% coverage required
* ensure that we apply the coverage rc file when generating the coverage xml report
* enable absolute_cov enforcement for azure-ai-ml

Author: scbedd

doc/eng_sys_checks.md

@@ -138,6 +138,18 @@ sphinx = false
 
 If a package does not yet have a `pyproject.toml`, creating one with just the section `[tool.azure-sdk-build]` will do no harm to the release of the package in question.
 
+### Coverage Enforcement
+
+This repository supports enforcement of an absolute coverage % per package. Set:
+
+```
+[tool.azure-sdk-build]
+absolute_cov = true
+absolute_cov_percent = 75.00
+```
+
+After it is implemented, the `relative_cov` key will enable the prevention of **negative** code coverage contributions.
+
 ## Environment variables important to CI
 
 There are a few differences from a standard local invocation of `tox <env>`. Primarily, these differences adjust the checks to be friendly to parallel invocation. These adjustments are necessary to prevent random CI crashes.

eng/tox/run_coverage.py

@@ -7,23 +7,13 @@
 
 from typing import Optional
 
-from ci_tools.parsing import ParsedSetup
+from ci_tools.parsing import ParsedSetup, get_config_setting
 from ci_tools.variables import in_ci
 from ci_tools.environment_exclusions import is_check_enabled
-from coverage.exceptions import NoDataError
+from ci_tools.functions import get_total_coverage
 
-def get_total_coverage(coverage_file: str) -> Optional[float]:
-    cov = coverage.Coverage(data_file=coverage_file)
-    cov.load()
-    try:
-        report = cov.report()
-        return report
-    except NoDataError as e:
-        logging.warning(f"This package did not generate any coverage output: {e}")
-        return None
-    except Exception as e:
-        logging.error(f"An error occurred while generating the coverage report: {e}")
-        return None
+logging.basicConfig(level=logging.INFO)
+coveragerc_file = os.path.join(os.path.dirname(__file__), "tox.ini")
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
@@ -38,16 +28,24 @@ def get_total_coverage(coverage_file: str) -> Optional[float]:
         required=True,
     )
 
+    parser.add_argument(
+        "-r",
+        "--root",
+        dest="repo_root",
+        help="The root of the directory. Source paths are relative to this.",
+        required=True,
+    )
+
     args = parser.parse_args()
     pkg_details = ParsedSetup.from_path(args.target_package)
 
     possible_coverage_file = os.path.join(args.target_package, ".coverage")
 
     if os.path.exists(possible_coverage_file):
-        total_coverage = get_total_coverage(possible_coverage_file)
-        if total_coverage is not None:
-            logging.info(f"Total coverage for {pkg_details.name} is {total_coverage:.2f}%")
+        total_coverage = get_total_coverage(possible_coverage_file, coveragerc_file, pkg_details.name, args.repo_root)
 
+        if total_coverage is not None:
+            # log the metric for reporting before doing anything else
             if in_ci():
                 metric_obj = {}
                 metric_obj["value"] = total_coverage / 100
@@ -59,6 +57,21 @@ def get_total_coverage(coverage_file: str) -> Optional[float]:
                 # before the 'logmetric' start string.
                 print(f'logmetric: {json.dumps(metric_obj)}')
 
-    if is_check_enabled(args.target_package, "cov_enforcement", False):
-        logging.info("Coverage enforcement is enabled for this package.")
+            # todo: add relative coverage comparison after we generate eng/coverages.json file in main
+            # when we add that, we will call the config setting relative_cov to check if this is enabled
+            # there, we will only ever compare the baseline % against the generated %
+
+            # as a fallback, we can always check the absolute coverage % against the config setting
+            # if the config setting is not set, we will not enforce any coverage
+            if is_check_enabled(args.target_package, "absolute_cov", False):
+                logging.info("Coverage enforcement is enabled for this package.")
+
+                # if this threshold is not set in config setting, the default will be very high
+                cov_threshold = get_config_setting(args.target_package, "absolute_cov_percent", 95.0)
+                if total_coverage < float(cov_threshold):
+                    logging.error(
+                        f"Coverage for {pkg_details.name} is below the threshold of {cov_threshold:.2f}% (actual: {total_coverage:.2f}%)"
+                    )
+                    exit(1)
+
 

eng/tox/tox.ini

@@ -73,7 +73,7 @@ commands =
     python {repository_root}/eng/tox/create_package_and_install.py -d {envtmpdir} -p {tox_root} -w {envtmpdir}
     python -m pip freeze
     pytest {[pytest]default_args} {posargs} {tox_root}
-    python {repository_root}/eng/tox/run_coverage.py -t {tox_root}
+    python {repository_root}/eng/tox/run_coverage.py -t {tox_root} -r {repository_root}
 
 
 [testenv:pylint]

scripts/devops_tasks/create_coverage.py

@@ -19,6 +19,7 @@
 
 root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
 coverage_dir = os.path.join(root_dir, "_coverage/")
+toxrc = os.path.join(root_dir, "eng","tox", "tox.ini")
 
 
 def collect_tox_coverage_files():
@@ -52,7 +53,7 @@ def collect_tox_coverage_files():
 def generate_coverage_xml():
     if os.path.exists(coverage_dir):
         logging.info("Generating coverage XML")
-        commands = ["coverage", "xml", "-i"]
+        commands = ["coverage", "xml", "-i", "--rcfile", toxrc]
         run_check_call(commands, root_dir, always_exit=False)
     else:
         logging.error("Coverage file is not available in {} to generate coverage XML".format(coverage_dir))

scripts/devops_tasks/tox_harness.py

@@ -35,27 +35,6 @@
 dependency_tools_path = os.path.join(root_dir, "eng", "dependency_tools.txt")
 
 
-def combine_coverage_files(targeted_packages):
-    # find tox.ini file. tox.ini is used to combine coverage paths to generate formatted report
-    tox_ini_file = os.path.join(root_dir, "eng", "tox", "tox.ini")
-    config_file_flag = "--rcfile={}".format(tox_ini_file)
-
-    if os.path.isfile(tox_ini_file):
-        # for every individual coverage file, run coverage combine to combine path
-        for package_dir in [package for package in targeted_packages]:
-            coverage_file = os.path.join(package_dir, ".coverage")
-            if os.path.isfile(coverage_file):
-                cov_cmd_array = [sys.executable, "-m", "coverage", "combine"]
-                # tox.ini file has coverage paths to combine
-                # Pas tox.ini as coverage config file
-                cov_cmd_array.extend([config_file_flag, coverage_file])
-                run_check_call(cov_cmd_array, package_dir)
-    else:
-        # not a hard error at this point
-        # this combine step is required only for modules if report has package name starts with .tox
-        logging.error("tox.ini is not found in path {}".format(root_dir))
-
-
 def collect_tox_coverage_files(targeted_packages):
     root_coverage_dir = os.path.join(root_dir, "_coverage/")
 

sdk/ml/azure-ai-ml/pyproject.toml

@@ -6,6 +6,8 @@ pylint = true
 mindependency = false
 latestdependency = false
 black = true
+absolute_cov = true
+absolute_cov_percent = 21.34
 
 [tool.isort]
 profile = "black"

tools/azure-sdk-tools/ci_tools/functions.py

@@ -8,6 +8,7 @@
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version, parse, InvalidVersion
 from pkg_resources import Requirement
+import io
 
 from ci_tools.variables import discover_repo_root, DEV_BUILD_IDENTIFIER, str_to_bool
 from ci_tools.parsing import ParsedSetup, get_config_setting, get_pyproject
@@ -695,6 +696,40 @@ def is_package_compatible(
     return True
 
 
+def get_total_coverage(coverage_file: str, coverage_config_file: str, package_name: str, repo_root: Optional[str] = None) -> Optional[float]:
+    try:
+        import coverage
+        from coverage.exceptions import NoDataError
+    except ImportError:
+        logging.error("Coverage is not installed.")
+        return None
+
+    cov = coverage.Coverage(data_file=coverage_file, config_file=coverage_config_file)
+    cov.load()
+    original = os.getcwd()
+    output = io.StringIO()
+
+    old_stdout = sys.stdout
+    sys.stdout = output
+
+    report = 0.0
+    try:
+        if repo_root:
+            os.chdir(repo_root)
+        logging.info(f"Running coverage report against \"{coverage_file}\" with \"{coverage_config_file}\" from \"{os.getcwd()}\".")
+        report = cov.report()
+    except NoDataError as e:
+        logging.info(f"Package {package_name} did not generate any coverage output: {e}")
+    except Exception as e:
+        logging.error(f"An error occurred while generating the coverage report for {package_name}: {e}")
+    finally:
+        if repo_root:
+            os.chdir(original)
+        sys.stdout = old_stdout
+        logging.info(f"Total coverage {report} for package {package_name}")
+        return report
+
+
 def resolve_compatible_package(package_name: str, immutable_requirements: List[Requirement]) -> Optional[str]:
     """
     This function attempts to resolve a compatible package version for whatever set of immutable_requirements that