Add observability tests

.github/workflows/ci.yaml

@@ -7,6 +7,7 @@ concurrency:
   cancel-in-progress: true
 
 on:
+  workflow_dispatch:
   pull_request:
     paths-ignore:
       - '.gitignore'

tests/integration/conftest.py

@@ -2,11 +2,50 @@
 # Copyright 2022 Canonical Ltd.
 # See LICENSE file for licensing details.
 import pytest
+import pytest_asyncio
+from _pytest.config import Config
 from pytest_operator.plugin import OpsTest
 
 
+OpsTestK8s = OpsTest
+
+
+def pytest_configure(config: Config):
+    config.addinivalue_line("markers", "abort_on_fail_k8s")
+
+
 @pytest.fixture(scope="module")
 async def charm(ops_test: OpsTest):
     """Build the charm-under-test."""
     # Build charm from local source folder.
     yield await ops_test.build_charm(".")
+
+
+@pytest.fixture(autouse=True)
+def abort_on_fail_k8s(request):
+    if OpsTestK8s._instance is None:
+        # If we don't have an ops_test already in play, this should be a no-op.
+        yield
+        return
+    ops_test = OpsTestK8s._instance
+    if ops_test.aborted:
+        pytest.xfail("aborted")
+
+    yield
+    abort_on_fail = request.node.get_closest_marker("abort_on_fail_k8s")
+    failed = getattr(request.node, "failed", False)
+    if abort_on_fail and abort_on_fail.kwargs.get("abort_on_xfail", False):
+        failed = failed or request.node.xfailed
+    if failed and abort_on_fail:
+        ops_test.aborted = True
+
+
+@pytest_asyncio.fixture(scope="module")
+async def ops_test_k8s(request, tmp_path_factory):
+    request.config.option.controller = "microk8s-localhost"
+    ops_test = OpsTestK8s(request, tmp_path_factory)
+    await ops_test._setup_model()
+    OpsTestK8s._instance = ops_test
+    yield ops_test
+    OpsTestK8s._instance = None
+    await ops_test._cleanup_models()

tests/integration/test_backups.py

@@ -18,6 +18,7 @@
     get_password,
     get_primary,
     get_unit_address,
+    run_command_on_unit,
     scale_application,
     switchover,
     wait_for_idle_on_blocked,
@@ -426,3 +427,13 @@ async def test_invalid_config_and_recovery_after_fixing_it(
     await ops_test.model.wait_for_idle(
         apps=[database_app_name, S3_INTEGRATOR_APP_NAME], status="active"
     )
+
+
+@pytest.mark.group(1)
+async def test_pgbackrest_logs_presence(ops_test: OpsTest):
+    database_app_name = f"new-{DATABASE_APP_NAME}"
+    unit_name = f"{database_app_name}/0"
+    logs_path = "/var/snap/charmed-postgresql/common/var/log"
+    check_pgbackrest_logs_cmd = f"sudo cat {logs_path}/pgbackrest/* 2> /dev/null"
+    stdout = await run_command_on_unit(ops_test, unit_name, check_pgbackrest_logs_cmd)
+    assert stdout.strip(), "pgbackrest logs not present"

tests/integration/test_charm.py

@@ -12,6 +12,8 @@
 from pytest_operator.plugin import OpsTest
 from tenacity import Retrying, stop_after_attempt, wait_exponential, wait_fixed
 
+from integration.helpers import run_command_on_unit
+
 from .helpers import (
     CHARM_SERIES,
     DATABASE_APP_NAME,
@@ -56,6 +58,24 @@ async def test_deploy(ops_test: OpsTest, charm: str):
     assert ops_test.model.applications[DATABASE_APP_NAME].units[0].workload_status == "active"
 
 
+@pytest.mark.group(1)
+@pytest.mark.abort_on_fail
+@pytest.mark.parametrize("unit_id", UNIT_IDS)
+async def test_logs_presence(ops_test: OpsTest, unit_id: int):
+    """Test if patroni and postgresql logs are present."""
+    logs_path = "/var/snap/charmed-postgresql/common/var/log"
+    check_patroni_logs_cmd = f"sudo cat {logs_path}/patroni/* 2> /dev/null"
+    check_postgresql_logs_cmd = f"sudo cat {logs_path}/postgresql/* 2> /dev/null"
+
+    unit = ops_test.model.applications[DATABASE_APP_NAME].units[unit_id]
+
+    stdout = await run_command_on_unit(ops_test, unit.name, check_patroni_logs_cmd)
+    assert stdout.strip(), "patroni logs not present"
+
+    stdout = await run_command_on_unit(ops_test, unit.name, check_postgresql_logs_cmd)
+    assert stdout.strip(), "postgresql logs not present"
+
+
 @pytest.mark.group(1)
 @pytest.mark.abort_on_fail
 @pytest.mark.parametrize("unit_id", UNIT_IDS)

tests/integration/test_observability.py

@@ -0,0 +1,59 @@
+import asyncio
+
+import pytest_asyncio
+from _pytest.config import Config
+from pytest_operator.plugin import OpsTest
+import pytest
+from integration.helpers import CHARM_SERIES, APPLICATION_NAME
+from integration.conftest import OpsTestK8s
+
+
+COS_BUNDLE_NAME = "cos-lite"
+GRAFANA_AGENT_APPLICATION_NAME = "grafana-agent"
+
+
+@pytest.mark.group(1)
+@pytest.mark.abort_on_fail
+@pytest.mark.abort_on_fail_k8s
+async def test_deploy(ops_test: OpsTest, ops_test_k8s: OpsTestK8s, charm: str):
+    await asyncio.gather(
+        ops_test.model.deploy(
+            charm,
+            application_name=APPLICATION_NAME,
+            num_units=2,
+            series=CHARM_SERIES,
+            config={"profile": "testing"},
+        ),
+        ops_test.model.deploy(GRAFANA_AGENT_APPLICATION_NAME),
+        ops_test_k8s.model.deploy(
+            COS_BUNDLE_NAME,
+            trust=True
+        ),
+    )
+
+    await asyncio.gather(
+        ops_test.model.wait_for_idle(apps=[APPLICATION_NAME], status="active", timeout=1000),
+        ops_test_k8s.model.wait_for_idle(status="active", timeout=1000),
+    )
+
+    await ops_test.model.integrate(APPLICATION_NAME, GRAFANA_AGENT_APPLICATION_NAME)
+
+    await ops_test.model.wait_for_idle()
+
+    # Setup monitoring integrations (with cos-lite).
+    await ops_test_k8s.model.create_offer("grafana:grafana-dashboard", "grafana-dashboards")
+    await ops_test_k8s.model.create_offer("loki:logging", "loki-logging")
+    await ops_test_k8s.model.create_offer("prometheus:receive-remote-write", "prometheus-receive-remote-write")
+    await ops_test.model.consume("admin/cos.grafana-dashboards", controller_name="microk8s-localhost")
+    await ops_test.model.consume("admin/cos.loki-logging", controller_name="microk8s-localhost")
+    await ops_test.model.consume("admin/cos.prometheus-receive-remote-write", controller_name="microk8s-localhost")
+    await ops_test.model.integrate(GRAFANA_AGENT_APPLICATION_NAME, "grafana-dashboards")
+    await ops_test.model.integrate(GRAFANA_AGENT_APPLICATION_NAME, "loki-logging")
+    await ops_test.model.integrate(GRAFANA_AGENT_APPLICATION_NAME, "prometheus-receive-remote-write")
+
+    await asyncio.gather(
+        ops_test.model.wait_for_idle(status="active"),
+        ops_test_k8s.model.wait_for_idle(status="active"),
+    )
+
+