Adapt Sql Coordinates

.github/workflows/ci.yml

@@ -88,6 +88,12 @@ jobs:
       #    run test suite
       #----------------------------------------------
       - name: Run tests
+        if: matrix.os != 'windows-latest'
         run: |
           source $VENV
           pytest tests/
+      - name: Run tests (Windows - skip Docker tests)
+        if: matrix.os == 'windows-latest'
+        run: |
+          source $VENV
+          pytest tests/ -m "not docker_required"

CHANGELOG.md

@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 ### Changed
+- Adapt Coordinate type to use WKBElement Geography [#287](https://github.com/ie3-institute/pypsdm/issues/287)
 
 ### Removed
 

nbs/create_grid.ipynb

@@ -66,7 +66,11 @@
    "source": [
     "from pypsdm.models.input.create.participants import create_energy_management_systems\n",
     "\n",
-    "em_uuids = [\"74d45b31-c35b-403b-96ac-5210754eb4de\",\"093e03ee-2d38-4e91-9f44-c71d92c07d89\",\"22af972d-19a8-415d-8c0a-50fa1238a7d9\"]\n",
+    "em_uuids = [\n",
+    "    \"74d45b31-c35b-403b-96ac-5210754eb4de\",\n",
+    "    \"093e03ee-2d38-4e91-9f44-c71d92c07d89\",\n",
+    "    \"22af972d-19a8-415d-8c0a-50fa1238a7d9\",\n",
+    "]\n",
     "\n",
     "data_dict = {\n",
     "    \"uuid\": em_uuids,\n",

pyproject.toml

@@ -20,6 +20,8 @@ sqlmodel = "^0.0.24"
 pyhocon = "^0.3.60"
 numba = "^0.61.2"
 scipy = "^1.15.2" # required by numba
+geoalchemy2 = "^0.17.1"
+psycopg2 = "^2.9.10"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.5"

pypsdm/db/weather/models.py

@@ -1,21 +1,32 @@
-import binascii
 from datetime import datetime
-from typing import Optional
+from typing import Any, ClassVar, Dict
 
+from geoalchemy2 import Geography, WKBElement
 from shapely import Point
 from shapely.wkb import loads
+from sqlalchemy import Column
 from sqlmodel import Field, SQLModel
 
 
+def ensure_bytes(wkb_element):
+    # Get the data from the WKBElement
+    data = wkb_element.data
+    # Check if it's a memoryview and convert it to bytes if necessary
+    if isinstance(data, memoryview):
+        return data.tobytes()
+    elif isinstance(data, bytes):
+        return data
+    else:
+        raise TypeError("Unexpected data type for WKBElement data")
+
+
 class WeatherValue(SQLModel, table=True):
     """
     Represents the ICON weather model.
     """
 
-    time: Optional[datetime] = Field(primary_key=True)
-    coordinate_id: Optional[int] = Field(
-        primary_key=True, default=None, foreign_key="coordinate.id"
-    )
+    time: datetime = Field(primary_key=True)
+    coordinate_id: int = Field(primary_key=True, foreign_key="coordinate.id")
     aswdifd_s: float = Field()
     aswdir_s: float = Field()
     t2m: float = Field()
@@ -57,8 +68,21 @@ def name_mapping():
 
 
 class Coordinate(SQLModel, table=True):
-    id: int = Field(primary_key=True)
-    coordinate: str = Field()
+    """Represents a geographical coordinate."""
+
+    model_config: ClassVar[Dict[str, Any]] = {"arbitrary_types_allowed": True}
+
+    id: int = Field(default=None, primary_key=True)
+
+    coordinate: Geography = Field(
+        sa_column=Column(
+            Geography(geometry_type="POINT", srid=4326, spatial_index=False)
+        )
+    )
+
+    def __init__(self, id: int, coordinate: Geography):
+        self.id = id
+        self.coordinate = coordinate
 
     def __eq__(self, other):
         return self.id == other.id
@@ -68,9 +92,8 @@ def __hash__(self):
 
     @property
     def point(self) -> Point:
-        wkb_str = self.coordinate
-        wkb_bytes = binascii.unhexlify(wkb_str)
-        return loads(wkb_bytes)
+        point_bytes = ensure_bytes(self.coordinate)
+        return loads(point_bytes)
 
     @property
     def latitude(self) -> float:
@@ -89,6 +112,7 @@ def x(self) -> float:
         return self.point.x
 
     @staticmethod
-    def from_xy(id, x, y):
-        wkb = Point(x, y).wkb_hex
-        return Coordinate(id=id, coordinate=wkb)
+    def from_xy(id: int, x: float, y: float) -> "Coordinate":
+        point = Point(x, y)
+        wkb = WKBElement(point.wkb, srid=4326)
+        return Coordinate(id, wkb)

tests/db/weather/docker/conftest.py

@@ -0,0 +1,109 @@
+import subprocess
+import time
+
+import pytest
+import sqlalchemy
+from sqlalchemy import text
+from sqlmodel import Session, SQLModel, create_engine
+
+
+@pytest.fixture(scope="session")
+@pytest.mark.docker_required
+def docker_postgres():
+    """Start a PostgreSQL container for testing."""
+    container_name = "test_weather"
+
+    # Check if the container already exists
+    check_cmd = [
+        "docker",
+        "ps",
+        "-a",
+        "--filter",
+        f"name={container_name}",
+        "--format",
+        "{{.Names}}",
+    ]
+    result = subprocess.run(check_cmd, capture_output=True, text=True)
+
+    if container_name in result.stdout:
+        # Remove existing container
+        subprocess.run(["docker", "rm", "-f", container_name], check=True)
+
+    # Start a new container
+    run_cmd = [
+        "docker",
+        "run",
+        "--name",
+        container_name,
+        "-e",
+        "POSTGRES_USER=postgres",
+        "-e",
+        "POSTGRES_PASSWORD=postgres",
+        "-e",
+        "POSTGRES_DB=test_weather",
+        "-p",
+        "5433:5432",
+        "-d",
+        "postgis/postgis:17-3.4",
+    ]
+    subprocess.run(run_cmd, check=True)
+
+    # Wait for the database to be ready
+    time.sleep(5)
+
+    yield "postgresql://postgres:postgres@localhost:5433/test_weather"
+
+    # Cleanup
+    subprocess.run(["docker", "rm", "-f", container_name], check=True)
+
+
+@pytest.fixture(scope="session")
+@pytest.mark.docker_required
+def db_engine(docker_postgres):
+    """Create a database engine connected to the Docker PostgreSQL instance."""
+    engine = create_engine(docker_postgres)
+
+    # Try connecting multiple times with increasing delays
+    max_retries = 5
+    retry_delay = 1  # seconds
+
+    for attempt in range(max_retries):
+        try:
+            # Enable PostGIS
+            with engine.connect() as conn:
+                conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis;"))
+                conn.commit()
+
+            SQLModel.metadata.create_all(engine)
+            break
+
+        except sqlalchemy.exc.OperationalError as e:
+            if attempt < max_retries - 1:
+                print(
+                    f"Connection attempt {attempt + 1} failed. Retrying in {retry_delay}s..."
+                )
+                time.sleep(retry_delay)
+                retry_delay *= 2  # Exponential backoff
+            else:
+                raise Exception(
+                    f"Failed to connect after {max_retries} attempts"
+                ) from e
+
+    return engine
+
+
+@pytest.fixture
+@pytest.mark.docker_required
+def db_session(db_engine):
+    """Create a new database session for a test."""
+    with Session(db_engine) as session:
+        yield session
+        session.rollback()  # Rollback after each test
+
+
+@pytest.fixture(scope="function", autouse=True)
+@pytest.mark.docker_required
+def reset_db(db_engine):
+    """Drop and recreate all tables before each test."""
+    SQLModel.metadata.drop_all(db_engine)
+    SQLModel.metadata.create_all(db_engine)

tests/db/weather/docker/docker_compose.yml

@@ -0,0 +1,22 @@
+version: '3'
+
+services:
+  postgis:
+    image: postgis/postgis:17-3.4
+    container_name: test_weather
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=test_weather
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    ports:
+      - "5433:5432"
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  pgdata:

tests/db/weather/docker/test_weather_models.py

@@ -0,0 +1,62 @@
+from datetime import datetime
+
+import pytest
+
+from pypsdm.db.weather.models import Coordinate, WeatherValue
+
+
+@pytest.mark.docker_required
+def test_create_coordinate(db_session):
+    """Test creating a coordinate."""
+    coordinates = []
+    coord_1 = Coordinate.from_xy(1, 8.645, 50.123)
+    coord_2 = Coordinate.from_xy(2, 7.46, 51.5)
+    coordinates: list[Coordinate] = coordinates + [coord_1, coord_2]
+
+    for coord in coordinates:
+        db_session.add(coord)
+    db_session.commit()
+
+    first_coordinate: Coordinate = db_session.get(Coordinate, 1)
+    second_coordinate = db_session.get(Coordinate, 2)
+    missing_coordinate = db_session.get(Coordinate, 3)
+
+    assert first_coordinate is not None
+    assert first_coordinate.longitude == 8.645
+    assert first_coordinate.latitude == 50.123
+
+    assert second_coordinate is not None
+    assert second_coordinate.x == 7.46
+    assert second_coordinate.y == 51.5
+
+    assert missing_coordinate is None
+
+
+@pytest.mark.docker_required
+def test_create_weather_value(db_session):
+    """Test creating a weather value."""
+    berlin = Coordinate.from_xy(id=4, x=13.405, y=52.52)
+
+    db_session.add(berlin)
+    db_session.commit()
+
+    now = datetime.now()
+    weather = WeatherValue(
+        time=now,
+        coordinate_id=berlin.id,
+        aswdifd_s=210.5,
+        aswdir_s=620.3,
+        t2m=293.15,
+        u131m=5.5,
+        v131m=2.2,
+    )
+    db_session.add(weather)
+    db_session.commit()
+
+    weather_value = db_session.get(WeatherValue, (weather.time, weather.coordinate_id))
+
+    assert weather_value is not None
+    assert weather_value.coordinate_id == berlin.id
+    assert abs(weather_value.temperature - 20.0) < 0.1
+    assert weather_value.diffuse_irradiance == 210.5
+    assert weather_value.direct_irradiance == 620.3

tests/db/weather/test_proxy.py

@@ -7,59 +7,35 @@
 def test_weighted_interpolation_coordinates():
     nearest_coordinates = [
         (
-            Coordinate(
-                id=13890,
-                coordinate="0101000020E610000000000000008026400000000000E04A40",
-            ),
+            Coordinate.from_xy(13890, 11.25, 53.75),
             15336.967985536437,
         ),
         (
-            Coordinate(
-                id=13891,
-                coordinate="0101000020E610000000000000000027400000000000E04A40",
-            ),
+            Coordinate.from_xy(13891, 11.5, 53.75),
             15741.508288878984,
         ),
         (
-            Coordinate(
-                id=14079,
-                coordinate="0101000020E610000000000000008026400000000000C04A40",
-            ),
+            Coordinate.from_xy(14079, 11.25, 53.5),
             16606.405396805483,
         ),
         (
-            Coordinate(
-                id=14080,
-                coordinate="0101000020E610000000000000000027400000000000C04A40",
-            ),
+            Coordinate.from_xy(14080, 11.5, 53.5),
             16982.92952636213,
         ),
         (
-            Coordinate(
-                id=13889,
-                coordinate="0101000020E610000000000000000026400000000000E04A40",
-            ),
+            Coordinate.from_xy(13889, 11.0, 53.75),
             27650.80452994915,
         ),
         (
-            Coordinate(
-                id=13892,
-                coordinate="0101000020E610000000000000008027400000000000E04A40",
-            ),
+            Coordinate.from_xy(13892, 11.75, 53.75),
             28324.62399357631,
         ),
         (
-            Coordinate(
-                id=14078,
-                coordinate="0101000020E610000000000000000026400000000000C04A40",
-            ),
+            Coordinate.from_xy(14078, 11.0, 53.5),
             28429.952194089583,
         ),
         (
-            Coordinate(
-                id=14081,
-                coordinate="0101000020E610000000000000008027400000000000C04A40",
-            ),
+            Coordinate.from_xy(14081, 11.75, 53.5),
             29089.579082292785,
         ),
     ]