fix(ci_visibility): fix corner case in Python 3.9 code coverage instrumentation [backport 4.1]

ddtrace/internal/coverage/instrumentation_py3_9.py

@@ -145,7 +145,7 @@ def update_location_data(
                 new_data.append(line_delta)
 
                 # Also add the current trap size to the accumulated offset
-                accumulated_new_offset = trap_map[current_orig_offset] << 1
+                accumulated_new_offset = trap_map.get(current_orig_offset, 0) << 1
                 current_new_offset += accumulated_new_offset
 
         return bytes(new_data)

pyproject.toml

@@ -145,6 +145,7 @@ exclude = [
     "tests/contrib/grpc/hello_pb2.py",
     "tests/contrib/django_celery/app/*",
     "tests/contrib/protobuf/schemas/**/*.py",
+    "tests/coverage/included_path/py39_line_numbers.py",
     "tests/appsec/iast/fixtures/ast/str/non_utf8_content.py",
     "tests/appsec/iast/fixtures/aspects/str/non_utf8_content.py",
     "tests/lib-injection/dd-lib-python-init-test-protobuf-old/addressbook_pb2.py"

releasenotes/notes/ci_visibility-fix-coverage-py39-line-numbers-f93fe457098505cb.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    CI Visibility: This fix resolves an issue where code coverage instrumentation in Python 3.9 would raise an exception
+    while handling line numbers in some corner cases.

tests/coverage/included_path/py39_line_numbers.py

@@ -0,0 +1,19 @@
+x = {}
+
+def g(*args, **kwargs):
+    pass
+
+
+def precise_line_numbers():
+    g(None,
+      **x)
+
+
+def imprecise_line_numbers():
+    g(
+        None, **x)
+
+
+def call_all_functions():
+    precise_line_numbers()
+    imprecise_line_numbers()

tests/coverage/test_coverage_py39_line_numebers.py

@@ -0,0 +1,52 @@
+import pytest
+
+
+@pytest.mark.subprocess
+def test_coverage_py39_line_numbers():
+    """
+    In Python 3.9, `dis.findlinestarts()` (used to determine where to insert instrumentation calls) can return
+    imprecise line number values in some corner cases. This test ensures that we do not explode in such cases (even
+    though the line number information may not be 100% precise).
+    """
+    import os
+    from pathlib import Path
+
+    from ddtrace.internal.coverage.code import ModuleCodeCollector
+    from ddtrace.internal.coverage.installer import install
+    from tests.coverage.utils import _get_relpath_dict
+
+    cwd_path = os.getcwd()
+    include_path = Path(cwd_path + "/tests/coverage/included_path/")
+
+    install(include_paths=[include_path], collect_import_time_coverage=True)
+
+    from tests.coverage.included_path.py39_line_numbers import call_all_functions
+
+    ModuleCodeCollector.start_coverage()
+    call_all_functions()
+    ModuleCodeCollector.stop_coverage()
+
+    executable = _get_relpath_dict(cwd_path, ModuleCodeCollector._instance.lines)
+    covered = _get_relpath_dict(cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=False))
+    covered_with_imports = _get_relpath_dict(
+        cwd_path, ModuleCodeCollector._instance._get_covered_lines(include_imported=True)
+    )
+
+    expected_executable = {
+        "tests/coverage/included_path/py39_line_numbers.py": {1, 3, 4, 7, 8, 9, 12, 13, 14, 17, 18, 19},
+    }
+    expected_covered = {
+        "tests/coverage/included_path/py39_line_numbers.py": {4, 8, 9, 13, 14, 18, 19},
+    }
+    expected_covered_with_imports = {
+        "tests/coverage/included_path/py39_line_numbers.py": {1, 3, 4, 7, 8, 9, 12, 13, 14, 17, 18, 19},
+    }
+
+    assert executable == expected_executable, (
+        f"Executable lines mismatch: expected={expected_executable} vs actual={executable}"
+    )
+    assert covered == expected_covered, f"Covered lines mismatch: expected={expected_covered} vs actual={covered}"
+    assert covered_with_imports == expected_covered_with_imports, (
+        f"Covered lines with imports mismatch: expected={expected_covered_with_imports} "
+        f"vs actual={covered_with_imports}"
+    )