diff --git a/CHANGELOG.md b/CHANGELOG.md index af56acbbe..7f47becff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -57,6 +57,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Changed `operationTime` to `operationFrom` and `operationUntil` in read the docs [1507](https://github.com/ie3-institute/PowerSystemDataModel/issues/1507) - Refactored validation of value classes [#1561](https://github.com/ie3-institute/PowerSystemDataModel/issues/1561) - Handled SonarQube Issues [#1588](https://github.com/ie3-institute/PowerSystemDataModel/issues/1588) +- Updated `Couchbase` version within the tests [#1586](https://github.com/ie3-institute/PowerSystemDataModel/issues/1586) ## [8.1.0] - 2025-07-25 diff --git a/build.gradle b/build.gradle index c574455fe..9ca395e47 100644 --- a/build.gradle +++ b/build.gradle @@ -73,6 +73,7 @@ dependencies { // testing testImplementation "org.apache.groovy:groovy:$groovyBinaryVersion" + testImplementation "org.apache.groovy:groovy-json:$groovyBinaryVersion" testImplementation "org.junit.platform:junit-platform-launcher:$junitVersion" testImplementation "org.junit.jupiter:junit-jupiter:$junitVersion" diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy new file mode 100644 index 000000000..8bbfef307 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy @@ -0,0 +1,115 @@ +/* + * © 2026. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.couchbase + +import edu.ie3.datamodel.io.connectors.CouchbaseConnector +import edu.ie3.test.helper.TestContainerHelper +import edu.ie3.test.helper.WeatherSourceTestHelper +import groovy.json.JsonSlurper +import org.testcontainers.couchbase.BucketDefinition +import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.spock.Testcontainers +import spock.lang.Shared +import spock.lang.Specification + +import java.time.Duration + +@Testcontainers +abstract class AbstractCouchbaseWeatherSourceIT extends Specification implements TestContainerHelper, WeatherSourceTestHelper { + + @Shared + BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") + + @Shared + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:8.0.0") + .withBucket(bucketDefinition) + .withExposedPorts(8091, 8092, 8093, 8094, 11210) + .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early + + @Shared + CouchbaseWeatherSource source + + static String coordinateIdColumnName = "coordinateid" + + abstract String getJsonResourcePath() + abstract Object getWeatherFactory() + abstract Object getCoordinateSource() + String getDtfPattern() { + return "yyyy-MM-dd'T'HH:mm:ssxxx" + } + + def setupSpec() { + // create an index for the document keys + couchbaseContainer.execInContainer("cbq", + "-e", "http://localhost:8093", + "-u", couchbaseContainer.username, + "-p", couchbaseContainer.password, + "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") + + // Create connector to import the data from json document + def connector = new CouchbaseConnector( + couchbaseContainer.connectionString, + bucketDefinition.name, + couchbaseContainer.username, + couchbaseContainer.password, + Duration.ofSeconds(30)) + + // Wait for bucket to be ready + println "Waiting for Couchbase bucket to be ready..." + int maxTries = 10 + boolean ready = false + for (int i = 0; i < maxTries; i++) { + try { + connector.getSourceFields() + ready = true + println "Couchbase bucket ready" + break + } catch (Exception ignored) { + if (i % 10 == 0) { + println "Waiting for bucket... (try ${i+1})" + } + sleep(1000) + } + } + + if (!ready) { + println "Couchbase bucket did not become ready in time!" + System.out.flush() + throw new RuntimeException("Couchbase bucket did not become ready in time!") + } + + // Insert test data from JSON file + println "Inserting test data from JSON file..." + def jsonFile = new File(getJsonResourcePath()) + def jsonSlurper = new JsonSlurper() + def weatherDocs = jsonSlurper.parse(jsonFile) as List + def insertCount = 0 + weatherDocs.each { doc -> + try { + def coordinateId = doc[coordinateIdColumnName] + def time = doc["time"] + def key = "weather::${coordinateId}::${time}" + connector.persist(key, doc).join() + insertCount++ + } catch (Exception ex) { + println "WARNING: Failed to insert document for coordinateid ${doc[coordinateIdColumnName]} and time ${doc["time"]}: ${ex.message}" + } + } + println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" + + source = new CouchbaseWeatherSource(connector, getCoordinateSource(), coordinateIdColumnName, getWeatherFactory(), getDtfPattern()) + println "setupSpec completed" + System.out.flush() + } + + def "The test container can establish a valid connection"() { + when: + def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + + then: + connector.connectionValid + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index a032cb5bc..1f62b15ba 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -5,81 +5,30 @@ */ package edu.ie3.datamodel.io.source.couchbase -import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.test.common.CosmoWeatherTestData -import edu.ie3.test.helper.TestContainerHelper -import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.interval.ClosedInterval import org.locationtech.jts.geom.Point -import org.testcontainers.couchbase.BucketDefinition -import org.testcontainers.couchbase.CouchbaseContainer import org.testcontainers.spock.Testcontainers -import org.testcontainers.utility.MountableFile -import spock.lang.Shared -import spock.lang.Specification - -import java.time.Duration @Testcontainers -class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContainerHelper, WeatherSourceTestHelper { - - @Shared - BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") - - @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.6.0") - .withBucket(bucketDefinition) - .withExposedPorts(8091, 8092, 8093, 8094, 11210) - .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early - - @Shared - CouchbaseWeatherSource source - - static String coordinateIdColumnName = "coordinateid" - - def setupSpec() { - // Copy import file with json array of documents into docker - MountableFile couchbaseWeatherJsonsFile = getMountableFile("_weather/cosmo/weather.json") - couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather_cosmo.json") - - // create an index for the document keys - couchbaseContainer.execInContainer("cbq", - "-e", "http://localhost:8093", - "-u", couchbaseContainer.username, - "-p", couchbaseContainer.password, - "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") - - //import the json documents from the copied file - couchbaseContainer.execInContainer("cbimport", "json", - "-cluster", "http://localhost:8091", - "--bucket", "ie3_in", - "--username", couchbaseContainer.username, - "--password", couchbaseContainer.password, - "--format", "list", - "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%", - "--dataset", "file:///home/weather_cosmo.json") +class CouchbaseWeatherSourceCosmoIT extends AbstractCouchbaseWeatherSourceIT { + @Override + String getJsonResourcePath() { + return "src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/cosmo/weather.json" + } - // increased timeout to deal with CI under high load - def connector = new CouchbaseConnector( - couchbaseContainer.connectionString, - bucketDefinition.name, - couchbaseContainer.username, - couchbaseContainer.password, - Duration.ofSeconds(20)) - def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) + @Override + Object getWeatherFactory() { + return new CosmoTimeBasedWeatherValueFactory() } - def "The test container can establish a valid connection"() { - when: - def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) - then: - connector.connectionValid + @Override + Object getCoordinateSource() { + return CosmoWeatherTestData.coordinateSource } def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { @@ -119,8 +68,6 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } - - def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { given: def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index ec7663ab8..bc1e5ed2d 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -5,79 +5,28 @@ */ package edu.ie3.datamodel.io.source.couchbase -import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.test.common.IconWeatherTestData -import edu.ie3.test.helper.TestContainerHelper -import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.interval.ClosedInterval -import org.testcontainers.couchbase.BucketDefinition -import org.testcontainers.couchbase.CouchbaseContainer import org.testcontainers.spock.Testcontainers -import spock.lang.Shared -import spock.lang.Specification - -import java.time.Duration @Testcontainers -class CouchbaseWeatherSourceIconIT extends Specification implements TestContainerHelper, WeatherSourceTestHelper { - - @Shared - BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") - - @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.6.0") - .withBucket(bucketDefinition) - .withExposedPorts(8091, 8092, 8093, 8094, 11210) - .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early - - @Shared - CouchbaseWeatherSource source - - static String coordinateIdColumnName = "coordinateid" - - def setupSpec() { - // Copy import file with json array of documents into docker - def couchbaseWeatherJsonsFile = getMountableFile("_weather/icon/weather.json") - couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather_icon.json") - - // create an index for the document keys - couchbaseContainer.execInContainer("cbq", - "-e", "http://localhost:8093", - "-u", couchbaseContainer.username, - "-p", couchbaseContainer.password, - "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") - - //import the json documents from the copied file - couchbaseContainer.execInContainer("cbimport", "json", - "-cluster", "http://localhost:8091", - "--bucket", "ie3_in", - "--username", couchbaseContainer.username, - "--password", couchbaseContainer.password, - "--format", "list", - "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%", - "--dataset", "file:///home/weather_icon.json") - - // increased timeout to deal with CI under high load - def connector = new CouchbaseConnector( - couchbaseContainer.connectionString, - bucketDefinition.name, - couchbaseContainer.username, - couchbaseContainer.password, - Duration.ofSeconds(20)) - def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new IconTimeBasedWeatherValueFactory() - source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) +class CouchbaseWeatherSourceIconIT extends AbstractCouchbaseWeatherSourceIT { + @Override + String getJsonResourcePath() { + return "src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/icon/weather.json" } - def "The test container can establish a valid connection"() { - when: - def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + @Override + Object getWeatherFactory() { + return new IconTimeBasedWeatherValueFactory() + } - then: - connector.connectionValid + @Override + Object getCoordinateSource() { + return IconWeatherTestData.coordinateSource } def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() {