Merge branch 'dev' into df/#1131-psdm-update-for-tap-water-demand

Author: danielfeismann

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased/Snapshot]
 
 ### Added
+- Enhance `TimeSeriesSource` with method to retrieve all time keys after a given key [#543](https://github.com/ie3-institute/PowerSystemDataModel/issues/543)
+- Enhance `WeatherSource` with method to retrieve all time keys after a given key [#572](https://github.com/ie3-institute/PowerSystemDataModel/issues/572)
 
 ### Fixed
 

build.gradle

@@ -73,7 +73,7 @@ dependencies {
   testImplementation 'org.junit.jupiter:junit-jupiter:5.11.0'
   testImplementation "org.spockframework:spock-core:2.3-groovy-$groovyVersion"
   testImplementation 'org.objenesis:objenesis:3.4' // Mock creation with constructor parameters
-  testImplementation 'net.bytebuddy:byte-buddy:1.14.18' // Mocks of classes
+  testImplementation 'net.bytebuddy:byte-buddy:1.15.0' // Mocks of classes
 
   // testcontainers (docker framework for testing)
   testImplementation "org.testcontainers:testcontainers:$testcontainersVersion"
@@ -91,11 +91,11 @@ dependencies {
   // Databases
   implementation 'org.influxdb:influxdb-java:2.24'
   implementation 'com.couchbase.client:java-client:3.7.2'
-  runtimeOnly 'org.postgresql:postgresql:42.7.3' // postgresql jdbc driver required during runtime
+  runtimeOnly 'org.postgresql:postgresql:42.7.4' // postgresql jdbc driver required during runtime
 
   implementation 'commons-io:commons-io:2.16.1' // I/O functionalities
   implementation 'commons-codec:commons-codec:1.17.1' // needed by commons-compress
-  implementation 'org.apache.commons:commons-compress:1.27.0' // I/O functionalities
+  implementation 'org.apache.commons:commons-compress:1.27.1' // I/O functionalities
 }
 
 tasks.withType(JavaCompile) {

src/main/java/edu/ie3/datamodel/io/factory/timeseries/CosmoTimeBasedWeatherValueFactory.java

@@ -45,11 +45,6 @@ public CosmoTimeBasedWeatherValueFactory() {
     super();
   }
 
-  @Override
-  public String getTimeFieldString() {
-    return TIME;
-  }
-
   @Override
   protected List<Set<String>> getFields(Class<?> entityClass) {
     Set<String> minConstructorParams =

src/main/java/edu/ie3/datamodel/io/factory/timeseries/IconTimeBasedWeatherValueFactory.java

@@ -42,11 +42,6 @@ public IconTimeBasedWeatherValueFactory(DateTimeFormatter dateTimeFormatter) {
     super(dateTimeFormatter);
   }
 
-  @Override
-  public String getTimeFieldString() {
-    return TIME;
-  }
-
   @Override
   protected List<Set<String>> getFields(Class<?> entityClass) {
     Set<String> minParameters =

src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedSimpleValueFactory.java

@@ -11,7 +11,6 @@
 import edu.ie3.datamodel.models.StandardUnits;
 import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue;
 import edu.ie3.datamodel.models.value.*;
-import edu.ie3.util.TimeUtil;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
@@ -26,26 +25,13 @@ public class TimeBasedSimpleValueFactory<V extends Value>
   private static final String REACTIVE_POWER = "q";
   private static final String HEAT_DEMAND = "heatDemand";
 
-  private final TimeUtil timeUtil;
-
   public TimeBasedSimpleValueFactory(Class<? extends V> valueClasses) {
     super(valueClasses);
-    this.timeUtil = TimeUtil.withDefaults;
   }
 
   public TimeBasedSimpleValueFactory(
       Class<? extends V> valueClasses, DateTimeFormatter dateTimeFormatter) {
-    super(valueClasses);
-    this.timeUtil = new TimeUtil(dateTimeFormatter);
-  }
-
-  /**
-   * Return the field name for the date time
-   *
-   * @return the field name for the date time
-   */
-  public String getTimeFieldString() {
-    return TIME;
+    super(valueClasses, dateTimeFormatter);
   }
 
   @Override

src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedValueFactory.java

@@ -8,6 +8,10 @@
 import edu.ie3.datamodel.io.factory.Factory;
 import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue;
 import edu.ie3.datamodel.models.value.Value;
+import edu.ie3.util.TimeUtil;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
 
 /**
  * Abstract class that is able to build {@link TimeBasedValue}s from "flat" information
@@ -18,7 +22,52 @@
 public abstract class TimeBasedValueFactory<D extends TimeBasedValueData<V>, V extends Value>
     extends Factory<V, D, TimeBasedValue<V>> {
 
+  protected static final String TIME = "time";
+
+  protected final TimeUtil timeUtil;
+
   protected TimeBasedValueFactory(Class<? extends V>... valueClasses) {
     super(valueClasses);
+    this.timeUtil = TimeUtil.withDefaults;
+  }
+
+  protected TimeBasedValueFactory(
+      Class<? extends V> valueClasses, DateTimeFormatter dateTimeFormatter) {
+    super(valueClasses);
+    this.timeUtil = new TimeUtil(dateTimeFormatter);
+  }
+
+  protected TimeBasedValueFactory(Class<? extends V> valueClasses, TimeUtil timeUtil) {
+    super(valueClasses);
+    this.timeUtil = timeUtil;
+  }
+
+  /**
+   * Return the field name for the date time
+   *
+   * @return the field name for the date time
+   */
+  public String getTimeFieldString() {
+    return TIME;
+  }
+
+  /**
+   * Method to extract a time string from a given map and convert into a {@link ZonedDateTime}.
+   *
+   * @param fieldsToAttributes map with time field
+   * @return a {@link ZonedDateTime}
+   */
+  public ZonedDateTime extractTime(Map<String, String> fieldsToAttributes) {
+    return toZonedDateTime(fieldsToAttributes.get(getTimeFieldString()));
+  }
+
+  /**
+   * Method to convert a given string into a {@link ZonedDateTime}.
+   *
+   * @param time string to convert
+   * @return a {@link ZonedDateTime}
+   */
+  public ZonedDateTime toZonedDateTime(String time) {
+    return timeUtil.toZonedDateTime(time);
   }
 }

src/main/java/edu/ie3/datamodel/io/factory/timeseries/TimeBasedWeatherValueFactory.java

@@ -15,24 +15,18 @@
  */
 public abstract class TimeBasedWeatherValueFactory
     extends TimeBasedValueFactory<TimeBasedWeatherValueData, WeatherValue> {
-  protected static final String TIME = "time";
   protected static final String COORDINATE_ID = "coordinateId";
 
-  protected final TimeUtil timeUtil;
-
   protected TimeBasedWeatherValueFactory() {
     super(WeatherValue.class);
-    this.timeUtil = TimeUtil.withDefaults;
   }
 
   protected TimeBasedWeatherValueFactory(DateTimeFormatter dateTimeFormatter) {
-    super(WeatherValue.class);
-    this.timeUtil = new TimeUtil(dateTimeFormatter);
+    super(WeatherValue.class, dateTimeFormatter);
   }
 
   protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) {
-    super(WeatherValue.class);
-    this.timeUtil = timeUtil;
+    super(WeatherValue.class, timeUtil);
   }
 
   /**
@@ -43,11 +37,4 @@ protected TimeBasedWeatherValueFactory(TimeUtil timeUtil) {
   public String getCoordinateIdFieldString() {
     return COORDINATE_ID;
   }
-
-  /**
-   * Return the field name for the date time
-   *
-   * @return the field name for the date time
-   */
-  public abstract String getTimeFieldString();
 }

src/main/java/edu/ie3/datamodel/io/source/TimeSeriesSource.java

@@ -51,4 +51,12 @@ public abstract IndividualTimeSeries<V> getTimeSeries(ClosedInterval<ZonedDateTi
       throws SourceException;
 
   public abstract Optional<V> getValue(ZonedDateTime time) throws SourceException;
+
+  /**
+   * Method to return all time keys after a given timestamp.
+   *
+   * @param time given time
+   * @return a list of time keys
+   */
+  public abstract List<ZonedDateTime> getTimeKeysAfter(ZonedDateTime time);
 }

src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java

@@ -18,6 +18,7 @@
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.commons.lang3.tuple.Pair;
 import org.locationtech.jts.geom.Point;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,8 +30,6 @@ public abstract class WeatherSource extends EntitySource {
 
   protected TimeBasedWeatherValueFactory weatherFactory;
 
-  protected Map<Point, IndividualTimeSeries<WeatherValue>> coordinateToTimeSeries;
-
   protected IdCoordinateSource idCoordinateSource;
 
   protected static final String COORDINATE_ID = "coordinateid";
@@ -65,6 +64,14 @@ public abstract Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(
   public abstract Optional<TimeBasedValue<WeatherValue>> getWeather(
       ZonedDateTime date, Point coordinate) throws SourceException;
 
+  public abstract Map<Point, List<ZonedDateTime>> getTimeKeysAfter(ZonedDateTime time)
+      throws SourceException;
+
+  public List<ZonedDateTime> getTimeKeysAfter(ZonedDateTime time, Point coordinate)
+      throws SourceException {
+    return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList());
+  }
+
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
   /**
@@ -111,6 +118,37 @@ protected Map<Point, IndividualTimeSeries<WeatherValue>> mapWeatherValuesToPoint
     return coordinateToTimeSeriesMap;
   }
 
+  protected Map<Point, List<ZonedDateTime>> toTimeKeys(
+      Stream<Map<String, String>> fieldMaps, TimeBasedWeatherValueFactory factory) {
+    return groupTime(
+        fieldMaps.map(
+            fieldMap -> {
+              String coordinateValue = fieldMap.get(COORDINATE_ID);
+              int coordinateId = Integer.parseInt(coordinateValue);
+              Optional<Point> coordinate = idCoordinateSource.getCoordinate(coordinateId);
+              ZonedDateTime time = factory.extractTime(fieldMap);
+
+              if (coordinate.isEmpty()) {
+                log.warn("Unable to match coordinate ID {} to a point", coordinateId);
+              }
+              return Pair.of(coordinate, time);
+            }));
+  }
+
+  protected Map<Point, List<ZonedDateTime>> groupTime(
+      Stream<Pair<Optional<Point>, ZonedDateTime>> values) {
+    return values
+        .filter(pair -> pair.getKey().isPresent())
+        .map(pair -> Pair.of(pair.getKey().get(), pair.getValue()))
+        .collect(Collectors.groupingBy(Pair::getKey, Collectors.toSet()))
+        .entrySet()
+        .stream()
+        .collect(
+            Collectors.toMap(
+                Map.Entry::getKey,
+                e -> e.getValue().stream().map(Pair::getValue).sorted().toList()));
+  }
+
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
   /**

src/main/java/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSource.java

@@ -25,6 +25,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.stream.Collectors;
+import org.apache.commons.lang3.tuple.Pair;
 import org.locationtech.jts.geom.Point;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -165,6 +166,37 @@ public Optional<TimeBasedValue<WeatherValue>> getWeather(ZonedDateTime date, Poi
     }
   }
 
+  @Override
+  public Map<Point, List<ZonedDateTime>> getTimeKeysAfter(ZonedDateTime time) {
+    String query = createQueryStringForFollowingTimeKeys(time);
+    CompletableFuture<QueryResult> futureResult = connector.query(query);
+    QueryResult queryResult = futureResult.join();
+    List<JsonObject> jsonWeatherInputs = Collections.emptyList();
+    try {
+      jsonWeatherInputs = queryResult.rowsAsObject();
+    } catch (DecodingFailureException ex) {
+      logger.error("Querying weather inputs failed!", ex);
+    }
+    if (jsonWeatherInputs != null && !jsonWeatherInputs.isEmpty()) {
+      return groupTime(
+          jsonWeatherInputs.stream()
+              .map(
+                  json -> {
+                    int coordinateId = json.getInt(COORDINATE_ID);
+                    Optional<Point> coordinate = idCoordinateSource.getCoordinate(coordinateId);
+                    ZonedDateTime timestamp =
+                        weatherFactory.toZonedDateTime(
+                            json.getString(weatherFactory.getTimeFieldString()));
+                    if (coordinate.isEmpty()) {
+                      log.warn("Unable to match coordinate ID {} to a point", coordinateId);
+                    }
+                    return Pair.of(coordinate, timestamp);
+                  })
+              .filter(value -> value.getValue().isAfter(time)));
+    }
+    return Collections.emptyMap();
+  }
+
   /**
    * Generates a key for weather documents with the pattern: {@code
    * weather::<coordinate_id>::<time>}
@@ -199,6 +231,18 @@ private String createQueryStringForIntervalAndCoordinate(
     return basicQuery + whereClause;
   }
 
+  /**
+   * Create a query string to search for all time keys that comes after the given time.
+   *
+   * @param time given timestamp
+   * @return the query string
+   */
+  public String createQueryStringForFollowingTimeKeys(ZonedDateTime time) {
+    String basicQuery = "SELECT a.coordinateid, a.time FROM " + connector.getBucketName() + " AS a";
+    String whereClause = " WHERE META().id > '" + generateWeatherKey(time, 0) + "'";
+    return basicQuery + whereClause;
+  }
+
   /**
    * Converts a JsonObject into TimeBasedWeatherValueData by extracting all fields into a field to
    * value map and then removing the coordinate from it to supply as a parameter

src/main/java/edu/ie3/datamodel/io/source/csv/CsvTimeSeriesSource.java

@@ -128,6 +128,11 @@ public Optional<V> getValue(ZonedDateTime time) {
     return timeSeries.getValue(time);
   }
 
+  @Override
+  public List<ZonedDateTime> getTimeKeysAfter(ZonedDateTime time) {
+    return timeSeries.getTimeKeysAfter(time);
+  }
+
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
   /**

src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java

@@ -41,6 +41,7 @@
 
 /** Implements a WeatherSource for CSV files by using the CsvTimeSeriesSource as a base */
 public class CsvWeatherSource extends WeatherSource {
+  private final Map<Point, IndividualTimeSeries<WeatherValue>> coordinateToTimeSeries;
 
   private final CsvDataSource dataSource;
 
@@ -64,8 +65,7 @@ public CsvWeatherSource(
       throws SourceException {
     super(idCoordinateSource, weatherFactory);
     this.dataSource = new CsvDataSource(csvSep, folderPath, fileNamingStrategy);
-
-    coordinateToTimeSeries = getWeatherTimeSeries();
+    this.coordinateToTimeSeries = getWeatherTimeSeries();
   }
 
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
@@ -111,6 +111,12 @@ public Optional<TimeBasedValue<WeatherValue>> getWeather(ZonedDateTime date, Poi
     return timeSeries.getTimeBasedValue(date);
   }
 
+  @Override
+  public Map<Point, List<ZonedDateTime>> getTimeKeysAfter(ZonedDateTime time) {
+    return coordinateToTimeSeries.entrySet().stream()
+        .collect(Collectors.toMap(Map.Entry::getKey, t -> t.getValue().getTimeKeysAfter(time)));
+  }
+
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
   /**