Add reusable Python guide bases and family snippets

.github/scripts/guide_ci_tasks.py

@@ -18,6 +18,7 @@ class GuideMetadata:
     slug: str
     publish: bool
     base: str | None
+    python_app: bool = False
 
 
 def kebab_case_to_gradle_name(name: str) -> str:
@@ -41,15 +42,21 @@ def guide_slugs_from_paths(paths: Iterable[str]) -> set[str]:
 
 
 def load_guides(guides_dir: Path) -> dict[str, GuideMetadata]:
-    guides = {}
+    metadata_by_slug = {}
     for metadata_path in sorted(guides_dir.glob("*/metadata.json")):
         with metadata_path.open(encoding="utf-8") as metadata_file:
-            metadata = json.load(metadata_file)
-        slug = metadata_path.parent.name
+            metadata_by_slug[metadata_path.parent.name] = json.load(metadata_file)
+    has_apps = {slug: bool(metadata.get("apps")) for slug, metadata in metadata_by_slug.items()}
+    # Match GuideUtils' sorted, in-place merge of base applications.
+    for slug, metadata in metadata_by_slug.items():
+        has_apps[slug] = has_apps[slug] or has_apps.get(metadata.get("base"), False)
+    guides = {}
+    for slug, metadata in metadata_by_slug.items():
         guides[slug] = GuideMetadata(
             slug=slug,
             publish=metadata.get("publish", True) is not False,
             base=metadata.get("base"),
+            python_app="PYTHON" in (metadata.get("languages") or []) and has_apps[slug],
         )
     return guides
 
@@ -75,12 +82,18 @@ def resolve_impacted_guide_slugs(
     )
 
 
-def tasks_for_changed_files(changed_files: Iterable[str], guides_dir: Path) -> list[str]:
+def tasks_for_changed_files(
+    changed_files: Iterable[str], guides_dir: Path, *, jvm_only: bool = False,
+) -> list[str]:
     guides = load_guides(guides_dir)
-    return [
-        f"{kebab_case_to_gradle_name(slug)}Build"
-        for slug in resolve_impacted_guide_slugs(changed_files, guides)
-    ]
+    tasks = []
+    for slug in resolve_impacted_guide_slugs(changed_files, guides):
+        task_slug = kebab_case_to_gradle_name(slug)
+        task = f"{task_slug}Build"
+        if jvm_only and guides[slug].python_app:
+            task += f" -x {task_slug}RunPythonTestScript"
+        tasks.append(task)
+    return tasks
 
 
 def matrix_for_tasks(tasks: Iterable[str]) -> dict[str, list[str]]:
@@ -124,6 +137,10 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--guides-dir", help="Guides directory. Defaults to <repo>/guides.")
     parser.add_argument("--base", help="Base commit for git diff.")
     parser.add_argument("--head", default="HEAD", help="Head commit for git diff.")
+    parser.add_argument(
+        "--jvm-only", action="store_true",
+        help="Exclude only Python runtime test tasks; retain JVM tests and guide generation.",
+    )
     parser.add_argument(
         "--diff-mode",
         choices=("range", "merge-base"),
@@ -153,7 +170,7 @@ def main() -> int:
         if args.changed_files is not None
         else git_changed_files(repo, args.base, args.head, args.diff_mode)
     )
-    tasks = tasks_for_changed_files(changed_files, guides_dir)
+    tasks = tasks_for_changed_files(changed_files, guides_dir, jvm_only=args.jvm_only)
     matrix = matrix_for_tasks(tasks)
 
     print(f"Changed files: {json.dumps(changed_files)}", file=sys.stderr)

.github/scripts/test_distribution_timing.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+
+
+BASE = Path(__file__).resolve().parents[2] / "guides/distribution-base"
+LANGUAGES = ("python", "java", "groovy", "kotlin")
+JVM_BRANCHES = (("java", "java"), ("java", "crac"), ("groovy", "java"), ("kotlin", "java"))
+HARNESS = r'''
+if [[ "$INHERIT_ERREXIT" == "yes" ]]; then
+  set -o posix
+fi
+curl() {
+  if [[ "$TIMING_RESULT" == "timeout" ]]; then
+    SECONDS=$((SECONDS + 21))
+    return 1
+  fi
+  return 0
+}
+sleep() { :; }
+docker() {
+  printf 'docker %s\n' "$*" >> "$COMMAND_LOG"
+  if [[ "$1" == "run" ]]; then
+    printf 'fake-container\n'
+  fi
+}
+native_fake() { printf 'native start\n' >> "$COMMAND_LOG"; }
+java() { printf 'java start %s\n' "$*" >> "$COMMAND_LOG"; }
+kill() { printf 'native cleanup %s\n' "$*" >> "$COMMAND_LOG"; }
+source "$1" "${@:2}"
+'''
+
+
+class DistributionTimingTest(unittest.TestCase):
+    def run_timing(self, language: str, branch: str, result: str, inherit_errexit: bool = False):
+        with tempfile.TemporaryDirectory() as directory:
+            command_log = Path(directory) / "commands.log"
+            environment = os.environ | {
+                "COMMAND_LOG": str(command_log),
+                "TIMING_RESULT": result,
+                "INHERIT_ERREXIT": "yes" if inherit_errexit else "no",
+            }
+            arguments = {
+                "docker": ["-d", "fake-image"], "native": ["-n", "native_fake"],
+                "java": ["-j", "fake.jar"], "crac": ["-c", "fake-checkpoint"],
+            }[branch]
+            completed = subprocess.run(
+                ["bash", "-c", HARNESS, "timing-test", str(BASE / language / "ttfr.sh"), *arguments],
+                env=environment, text=True, capture_output=True, timeout=5,
+            )
+            commands = command_log.read_text() if command_log.exists() else ""
+        cleanup = "docker container kill fake-container" if branch == "docker" else "native cleanup"
+        return completed, commands, cleanup
+
+    def check_timeout(self, language: str, branch: str, inherit_errexit: bool = False):
+        completed, commands, cleanup = self.run_timing(language, branch, "timeout", inherit_errexit)
+        self.assertIn("No response from the app in 20 seconds", completed.stderr)
+        self.assertNotEqual(0, completed.returncode, completed.stdout)
+        self.assertIn(cleanup, commands)
+        self.assertNotIn(" seconds", completed.stdout)
+
+    def test_docker_timeout(self):
+        for language in LANGUAGES:
+            with self.subTest(language=language):
+                self.check_timeout(language, "docker")
+
+    def test_native_timeout(self):
+        for language in LANGUAGES:
+            with self.subTest(language=language):
+                self.check_timeout(language, "native")
+
+    def test_docker_timeout_with_inherited_errexit(self):
+        for language in LANGUAGES:
+            with self.subTest(language=language):
+                self.check_timeout(language, "docker", True)
+
+    def test_native_timeout_with_inherited_errexit(self):
+        for language in LANGUAGES:
+            with self.subTest(language=language):
+                self.check_timeout(language, "native", True)
+
+    def test_jvm_timeout(self):
+        for language, branch in JVM_BRANCHES:
+            with self.subTest(language=language, branch=branch):
+                self.check_timeout(language, branch)
+
+    def test_jvm_timeout_with_inherited_errexit(self):
+        for language, branch in JVM_BRANCHES:
+            with self.subTest(language=language, branch=branch):
+                self.check_timeout(language, branch, True)
+
+    def check_success(self, language: str, branch: str):
+        completed, commands, cleanup = self.run_timing(language, branch, "success")
+        self.assertEqual(0, completed.returncode, completed.stderr)
+        self.assertEqual(1, commands.count(cleanup))
+        self.assertIn(" seconds", completed.stdout)
+
+    def test_docker_success(self):
+        for language in LANGUAGES:
+            with self.subTest(language=language):
+                self.check_success(language, "docker")
+
+    def test_native_success(self):
+        for language in LANGUAGES:
+            with self.subTest(language=language):
+                self.check_success(language, "native")
+
+    def test_jvm_success(self):
+        for language, branch in JVM_BRANCHES:
+            with self.subTest(language=language, branch=branch):
+                self.check_success(language, branch)
+
+
+if __name__ == "__main__":
+    unittest.main()