diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e2d33988..edd9a934 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -125,6 +125,21 @@ updates: schedule: interval: "daily" + - directory: "/application/grafana" + package-ecosystem: "pip" + schedule: + interval: "daily" + + - directory: "/application/grafana" + package-ecosystem: "docker" + schedule: + interval: "daily" + + - directory: "/application/grafana" + package-ecosystem: "docker-compose" + schedule: + interval: "daily" + - directory: "/application/open-webui" package-ecosystem: "docker-compose" schedule: diff --git a/.github/workflows/application-grafana.yml b/.github/workflows/application-grafana.yml new file mode 100644 index 00000000..4ad420d4 --- /dev/null +++ b/.github/workflows/application-grafana.yml @@ -0,0 +1,64 @@ +name: "Grafana" + +on: + pull_request: + paths: + - '.github/workflows/application-grafana.yml' + - 'application/grafana/**' + push: + branches: [ main ] + paths: + - '.github/workflows/application-grafana.yml' + - 'application/grafana/**' + + # Allow job to be triggered manually. + workflow_dispatch: + + # Run job each night after CrateDB nightly has been published. + schedule: + - cron: '0 4 * * *' + +# Cancel in-progress jobs when pushing to the same branch. +concurrency: + cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.ref }} + +jobs: + + test: + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: + - "ubuntu-latest" + cratedb-version: + - "nightly" + grafana-version: + - "9.5.21" + - "10.4.19" + - "11.6" + - "12.3" + - "12.4" + - "nightly" + + env: + OS_TYPE: ${{ matrix.os }} + CRATEDB_VERSION: ${{ matrix.cratedb-version }} + GRAFANA_VERSION: ${{ matrix.grafana-version }} + + name: " + Grafana ${{ matrix.grafana-version }}, + CrateDB ${{ matrix.cratedb-version }} + " + steps: + + - name: Acquire sources + uses: actions/checkout@v6 + + - name: Validate application/grafana + run: | + # TODO: Generalize invocation into `ngr` test runner. + cd application/grafana + bash test.sh diff --git a/application/grafana/compose.yml b/application/grafana/compose.yml new file mode 100644 index 00000000..c7423b83 --- /dev/null +++ b/application/grafana/compose.yml @@ -0,0 +1,54 @@ +services: + + cratedb: + image: docker.io/crate/crate:${CRATEDB_VERSION:-latest} + command: > + crate \ + '-Cdiscovery.type=single-node' \ + '-Cstats.enabled=true' + ports: + - 4200:4200 + - 5432:5432 + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:4200"] + start_period: 3s + interval: 1.5s + retries: 30 + timeout: 30s + + grafana: + image: docker.io/grafana/grafana:${GRAFANA_VERSION:-latest} + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + ports: + - "3000:3000" + depends_on: + - cratedb + healthcheck: + test: ["CMD", "curl", "--fail", "http://localhost:3000"] + start_period: 3s + interval: 1.5s + retries: 30 + timeout: 30s + + example-weather: + build: + context: . + dockerfile_inline: | + FROM docker.io/python:3.14-slim-trixie + RUN apt-get update && apt-get install --yes git + ADD requirements.txt / + ADD example-weather.py / + COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + ENV UV_SYSTEM_PYTHON=true + RUN uv pip install --requirement requirements.txt + command: python example-weather.py + depends_on: + cratedb: + condition: service_healthy + grafana: + condition: service_healthy + profiles: + - tasks diff --git a/application/grafana/example-weather.py b/application/grafana/example-weather.py new file mode 100644 index 00000000..58597219 --- /dev/null +++ b/application/grafana/example-weather.py @@ -0,0 +1,184 @@ +""" +Example program demonstrating how to work with CrateDB and Grafana using +[grafana-client] and [grafanalib]. + +[grafana-client]: https://github.com/grafana-toolbox/grafana-client +[grafanalib]: https://github.com/weaveworks/grafanalib +""" +import dataclasses +import json +import logging + +from cratedb_toolkit.datasets import load_dataset +from grafana_client import GrafanaApi +from grafana_client.client import GrafanaClientError +from grafana_client.model import DatasourceIdentifier +from grafana_client.util import setup_logging +from grafanalib._gen import DashboardEncoder +from grafanalib.core import (SHORT_FORMAT, Dashboard, Graph, GridPos, + SqlTarget, Time, YAxes, YAxis) +from yarl import URL + +logger = logging.getLogger(__name__) + + +DATASOURCE_UID = "cratedb-v2KYBt37k" +DASHBOARD_UID = "cratedb-weather-demo" +CRATEDB_SQLALCHEMY_URL = "crate://crate:crate@cratedb:4200/" +CRATEDB_GRAFANA_URL = "cratedb:5432" +GRAFANA_URL = "http://grafana:3000" + + +@dataclasses.dataclass +class PanelInfo: + """ + Minimal information defining a minimal graph panel. + """ + title: str + field: str + unit: str + + +def provision(grafana: GrafanaApi): + """ + Provision CrateDB and Grafana. + + - Load example weather data into CrateDB. + - Provision Grafana with data source and dashboard. + """ + + logger.info("Loading data into CrateDB") + + # Load example data into CrateDB. + dataset = load_dataset("tutorial/weather-basic") + dataset.dbtable(dburi=CRATEDB_SQLALCHEMY_URL, table="example.weather_data").load() + + logger.info("Provisioning Grafana data source and dashboard") + + # Create Grafana data source. + try: + grafana.datasource.get_datasource_by_uid(DATASOURCE_UID) + grafana.datasource.delete_datasource_by_uid(DATASOURCE_UID) + except GrafanaClientError as ex: + if ex.status_code != 404: + raise + grafana.datasource.create_datasource( + { + "uid": DATASOURCE_UID, + "name": "CrateDB", + "type": "postgres", + "access": "proxy", + "url": CRATEDB_GRAFANA_URL, + "jsonData": { + "database": "doc", + "postgresVersion": 1200, + "sslmode": "disable", + }, + "user": "crate", + "secureJsonData": { + "password": "crate", + }, + } + ) + + # Create Grafana dashboard. + dashboard = Dashboard( + uid=DASHBOARD_UID, + title="CrateDB example weather dashboard", + time=Time('2023-01-01T00:00:00Z', '2023-09-01T00:00:00Z'), + refresh=None, + ) + panel_infos = [ + PanelInfo(title="Weather » Temperature", field="temperature", unit="celsius"), + PanelInfo(title="Weather » Humidity", field="humidity", unit="humidity"), + PanelInfo(title="Weather » Wind speed", field="wind_speed", unit="velocitykmh"), + ] + for panel_info in panel_infos: + column_name = panel_info.field + unit = panel_info.unit + dashboard.panels.append( + Graph( + title=f"{panel_info.title}", + dataSource=DATASOURCE_UID, + targets=[ + SqlTarget( + rawSql=f""" + SELECT + $__timeGroupAlias("timestamp", $__interval), + "location", + MEAN("{column_name}") AS "{column_name}" + FROM "example"."weather_data" + WHERE $__timeFilter("timestamp") + GROUP BY "time", "location" + ORDER BY "time" + """, + refId="A", + ), + ], + yAxes=YAxes( + YAxis(format=unit), + YAxis(format=SHORT_FORMAT), + ), + gridPos=GridPos(h=8, w=24, x=0, y=9), + ) + ) + # Encode grafanalib `Dashboard` entity to dictionary. + dashboard_payload = { + "dashboard": json.loads(json.dumps(dashboard, sort_keys=True, cls=DashboardEncoder)), + "overwrite": True, + "message": "Updated by grafanalib", + } + response = grafana.dashboard.update_dashboard(dashboard_payload) + + # Display dashboard URL. + dashboard_url = URL(f"{grafana.url}{response['url']}").with_user(None).with_password(None) + logger.info(f"Dashboard URL: {dashboard_url}") + + +def validate_datasource(grafana: GrafanaApi): + """ + Validate Grafana data source. + """ + logger.info("Validating data source") + health = grafana.datasource.health_inquiry(DATASOURCE_UID) + logger.info("Health status: %s", health.status) + logger.info("Health message: %s", health.message) + assert health.success is True, "Grafana data source is not healthy" + + +def validate_dashboard(grafana: GrafanaApi): + """ + Validate Grafana dashboard by enumerating and executing all panel targets' `rawSql` expressions. + """ + logger.info("Validating dashboard") + dashboard = grafana.dashboard.get_dashboard(DASHBOARD_UID) + for panel in dashboard["dashboard"].get("panels", []): + for target in panel.get("targets", []): + logger.info("Validating SQL target:\n%s", target["rawSql"]) + + response = grafana.datasource.smartquery(DatasourceIdentifier(uid=DATASOURCE_UID), target["rawSql"]) + status = response["results"]["test"]["status"] + queries = [frame["schema"]["meta"]["executedQueryString"] for frame in response["results"]["test"]["frames"]] + logger.info("Status: %s", status) + logger.info("Executed queries:\n%s", "\n".join(queries)) + + assert status == 200, "Dashboard query status is not 200" + + +if __name__ == "__main__": + """ + Boilerplate bootloader. Create a `GrafanaApi` instance and run example. + """ + + # Setup logging. + setup_logging(level=logging.INFO) + + # Create a `GrafanaApi` instance. + grafana_client = GrafanaApi.from_url(GRAFANA_URL) + + # Invoke example conversation. + provision(grafana_client) + + # Validate Grafana data source and dashboard. + validate_datasource(grafana_client) + validate_dashboard(grafana_client) diff --git a/application/grafana/requirements.txt b/application/grafana/requirements.txt new file mode 100644 index 00000000..bb725f13 --- /dev/null +++ b/application/grafana/requirements.txt @@ -0,0 +1,3 @@ +cratedb-toolkit +grafana-client<6 +grafanalib<0.8 diff --git a/application/grafana/test.sh b/application/grafana/test.sh new file mode 100644 index 00000000..3f82556d --- /dev/null +++ b/application/grafana/test.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# Use Grafana with CrateDB. + +# The miniature stack defines {Docker,Podman} services and tasks to spin +# up CrateDB and Grafana, provision data into CrateDB, and a corresponding +# data source and dashboard into Grafana. + +# https://github.com/grafana/grafana +# https://github.com/crate/crate + +# Start services. +docker compose up --detach --wait + +# Run weather data example. +docker compose run --rm example-weather