From fd8b1cc4dd24e98df53730aeaa866d4393f94908 Mon Sep 17 00:00:00 2001 From: danielfeismann Date: Wed, 25 Mar 2026 20:11:04 +0100 Subject: [PATCH 1/5] Updated `Couchbase` version within the tests --- CHANGELOG.md | 1 + build.gradle | 1 + .../CouchbaseWeatherSourceCosmoIT.groovy | 72 ++++++++++++++----- .../CouchbaseWeatherSourceIconIT.groovy | 70 ++++++++++++++---- 4 files changed, 114 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17b4083bb7..38a8938800 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Refactored validation of entity classes [#1561](https://github.com/ie3-institute/PowerSystemDataModel/issues/1561) - Changed `operationTime` to `operationFrom` and `operationUntil` in read the docs [1507](https://github.com/ie3-institute/PowerSystemDataModel/issues/1507) - Refactored validation of value classes [#1561](https://github.com/ie3-institute/PowerSystemDataModel/issues/1561) +- Updated `Couchbase` version within the tests [#1586](https://github.com/ie3-institute/PowerSystemDataModel/issues/1586) ## [8.1.0] - 2025-07-25 diff --git a/build.gradle b/build.gradle index a28783d9d8..051811e320 100644 --- a/build.gradle +++ b/build.gradle @@ -73,6 +73,7 @@ dependencies { // testing testImplementation "org.apache.groovy:groovy:$groovyBinaryVersion" + testImplementation 'org.apache.groovy:groovy-json:5.0.4' testImplementation "org.junit.platform:junit-platform-launcher:$junitVersion" testImplementation "org.junit.jupiter:junit-jupiter:$junitVersion" diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index a032cb5bce..3d1edbbfc1 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -31,7 +31,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.6.0") + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:8.0.0") .withBucket(bucketDefinition) .withExposedPorts(8091, 8092, 8093, 8094, 11210) .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early @@ -42,10 +42,6 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain static String coordinateIdColumnName = "coordinateid" def setupSpec() { - // Copy import file with json array of documents into docker - MountableFile couchbaseWeatherJsonsFile = getMountableFile("_weather/cosmo/weather.json") - couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather_cosmo.json") - // create an index for the document keys couchbaseContainer.execInContainer("cbq", "-e", "http://localhost:8093", @@ -53,15 +49,58 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain "-p", couchbaseContainer.password, "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") - //import the json documents from the copied file - couchbaseContainer.execInContainer("cbimport", "json", - "-cluster", "http://localhost:8091", - "--bucket", "ie3_in", - "--username", couchbaseContainer.username, - "--password", couchbaseContainer.password, - "--format", "list", - "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%", - "--dataset", "file:///home/weather_cosmo.json") + // Create connector to import the data from json document + def importConnector = new CouchbaseConnector( + couchbaseContainer.connectionString, + bucketDefinition.name, + couchbaseContainer.username, + couchbaseContainer.password, + Duration.ofSeconds(30)) + + // Wait for bucket to be ready + println "Waiting for Couchbase bucket to be ready..." + int maxTries = 10 + boolean ready = false + for (int i = 0; i < maxTries; i++) { + try { + importConnector.getSourceFields() + ready = true + println "Couchbase bucket ready" + break + } catch (Exception ex) { + if (i % 10 == 0) { + println "Waiting for bucket... (try ${i+1})" + } + sleep(1000) + } + } + + if (!ready) { + println "Couchbase bucket did not become ready in time!" + System.out.flush() + throw new RuntimeException("Couchbase bucket did not become ready in time!") + } + + // Insert test data from JSON file + println "Inserting test data from JSON file..." + def jsonFile = new File("src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/cosmo/weather.json") + def jsonSlurper = new groovy.json.JsonSlurper() + def weatherDocs = jsonSlurper.parse(jsonFile) as List + def insertCount = 0 + weatherDocs.each { doc -> + try { + def coordinateId = doc["coordinateid"] + def time = doc["time"] + def key = "weather::${coordinateId}::${time}" + importConnector.persist(key, doc).join() + insertCount++ + } catch (Exception ex) { + println "WARNING: Failed to insert document for coordinateid ${doc["coordinateid"]} and time ${doc["time"]}: ${ex.message}" + } + } + println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" + importConnector.shutdown() + System.out.flush() // increased timeout to deal with CI under high load def connector = new CouchbaseConnector( @@ -73,11 +112,14 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" def weatherFactory = new CosmoTimeBasedWeatherValueFactory() source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) + println "setupSpec completed" + System.out.flush() } def "The test container can establish a valid connection"() { when: def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + then: connector.connectionValid } @@ -119,8 +161,6 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain equalsIgnoreUUID(coordinateToTimeSeries.get(CosmoWeatherTestData.COORDINATE_193187), timeSeries193187) } - - def "A CouchbaseWeatherSource can read all weather data in a given time interval"() { given: def timeInterval = new ClosedInterval(CosmoWeatherTestData.TIME_15H, CosmoWeatherTestData.TIME_17H) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index ec7663ab83..12895c731e 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -13,6 +13,7 @@ import edu.ie3.test.common.IconWeatherTestData import edu.ie3.test.helper.TestContainerHelper import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.interval.ClosedInterval +import groovy.json.JsonSlurper import org.testcontainers.couchbase.BucketDefinition import org.testcontainers.couchbase.CouchbaseContainer import org.testcontainers.spock.Testcontainers @@ -28,7 +29,7 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.6.0") + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:8.0.0") .withBucket(bucketDefinition) .withExposedPorts(8091, 8092, 8093, 8094, 11210) .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early @@ -39,10 +40,6 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine static String coordinateIdColumnName = "coordinateid" def setupSpec() { - // Copy import file with json array of documents into docker - def couchbaseWeatherJsonsFile = getMountableFile("_weather/icon/weather.json") - couchbaseContainer.copyFileToContainer(couchbaseWeatherJsonsFile, "/home/weather_icon.json") - // create an index for the document keys couchbaseContainer.execInContainer("cbq", "-e", "http://localhost:8093", @@ -50,15 +47,58 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine "-p", couchbaseContainer.password, "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") - //import the json documents from the copied file - couchbaseContainer.execInContainer("cbimport", "json", - "-cluster", "http://localhost:8091", - "--bucket", "ie3_in", - "--username", couchbaseContainer.username, - "--password", couchbaseContainer.password, - "--format", "list", - "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%", - "--dataset", "file:///home/weather_icon.json") + // Create connector to import the data from json document + def importConnector = new CouchbaseConnector( + couchbaseContainer.connectionString, + bucketDefinition.name, + couchbaseContainer.username, + couchbaseContainer.password, + Duration.ofSeconds(30)) + + // Wait for bucket to be ready + println "Waiting for Couchbase bucket to be ready..." + int maxTries = 10 + boolean ready = false + for (int i = 0; i < maxTries; i++) { + try { + importConnector.getSourceFields() + ready = true + println "Couchbase bucket ready" + break + } catch (Exception ex) { + if (i % 10 == 0) { + println "Waiting for bucket... (try ${i+1})" + } + sleep(1000) + } + } + + if (!ready) { + println "Couchbase bucket did not become ready in time!" + System.out.flush() + throw new RuntimeException("Couchbase bucket did not become ready in time!") + } + + // Insert test data from JSON file + println "Inserting test data from JSON file..." + def jsonFile = new File("src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/icon/weather.json") + def jsonSlurper = new groovy.json.JsonSlurper() + def weatherDocs = jsonSlurper.parse(jsonFile) as List + def insertCount = 0 + weatherDocs.each { doc -> + try { + def coordinateId = doc["coordinateid"] + def time = doc["time"] + def key = "weather::${coordinateId}::${time}" + importConnector.persist(key, doc).join() + insertCount++ + } catch (Exception ex) { + println "WARNING: Failed to insert document for coordinateid ${doc["coordinateid"]} and time ${doc["time"]}: ${ex.message}" + } + } + println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" + importConnector.shutdown() + System.out.flush() // increased timeout to deal with CI under high load def connector = new CouchbaseConnector( @@ -70,6 +110,8 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" def weatherFactory = new IconTimeBasedWeatherValueFactory() source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) + println "setupSpec completed" + System.out.flush() } def "The test container can establish a valid connection"() { From d6975e7a2ac2eff3bdeedd6542c1b5e0ecc4503a Mon Sep 17 00:00:00 2001 From: danielfeismann Date: Fri, 27 Mar 2026 14:19:27 +0100 Subject: [PATCH 2/5] include feedback from review --- .../CouchbaseWeatherSourceCosmoIT.groovy | 15 +++------------ .../couchbase/CouchbaseWeatherSourceIconIT.groovy | 15 +++------------ 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 3d1edbbfc1..2e8803efb4 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -50,7 +50,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") // Create connector to import the data from json document - def importConnector = new CouchbaseConnector( + def connector = new CouchbaseConnector( couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, @@ -63,7 +63,7 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain boolean ready = false for (int i = 0; i < maxTries; i++) { try { - importConnector.getSourceFields() + connector.getSourceFields() ready = true println "Couchbase bucket ready" break @@ -92,23 +92,14 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain def coordinateId = doc["coordinateid"] def time = doc["time"] def key = "weather::${coordinateId}::${time}" - importConnector.persist(key, doc).join() + connector.persist(key, doc).join() insertCount++ } catch (Exception ex) { println "WARNING: Failed to insert document for coordinateid ${doc["coordinateid"]} and time ${doc["time"]}: ${ex.message}" } } println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" - importConnector.shutdown() - System.out.flush() - // increased timeout to deal with CI under high load - def connector = new CouchbaseConnector( - couchbaseContainer.connectionString, - bucketDefinition.name, - couchbaseContainer.username, - couchbaseContainer.password, - Duration.ofSeconds(20)) def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" def weatherFactory = new CosmoTimeBasedWeatherValueFactory() source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index 12895c731e..6c2244cfdb 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -48,7 +48,7 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") // Create connector to import the data from json document - def importConnector = new CouchbaseConnector( + def connector = new CouchbaseConnector( couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, @@ -61,7 +61,7 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine boolean ready = false for (int i = 0; i < maxTries; i++) { try { - importConnector.getSourceFields() + connector.getSourceFields() ready = true println "Couchbase bucket ready" break @@ -90,23 +90,14 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine def coordinateId = doc["coordinateid"] def time = doc["time"] def key = "weather::${coordinateId}::${time}" - importConnector.persist(key, doc).join() + connector.persist(key, doc).join() insertCount++ } catch (Exception ex) { println "WARNING: Failed to insert document for coordinateid ${doc["coordinateid"]} and time ${doc["time"]}: ${ex.message}" } } println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" - importConnector.shutdown() - System.out.flush() - // increased timeout to deal with CI under high load - def connector = new CouchbaseConnector( - couchbaseContainer.connectionString, - bucketDefinition.name, - couchbaseContainer.username, - couchbaseContainer.password, - Duration.ofSeconds(20)) def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" def weatherFactory = new IconTimeBasedWeatherValueFactory() source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) From 75eb5aceb22d46bd0a5694f245c6b08d87891cac Mon Sep 17 00:00:00 2001 From: danielfeismann Date: Fri, 27 Mar 2026 14:22:10 +0100 Subject: [PATCH 3/5] build gradle --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 051811e320..a6738ec5be 100644 --- a/build.gradle +++ b/build.gradle @@ -73,7 +73,7 @@ dependencies { // testing testImplementation "org.apache.groovy:groovy:$groovyBinaryVersion" - testImplementation 'org.apache.groovy:groovy-json:5.0.4' + testImplementation "org.apache.groovy:groovy-json:$groovyBinaryVersion" testImplementation "org.junit.platform:junit-platform-launcher:$junitVersion" testImplementation "org.junit.jupiter:junit-jupiter:$junitVersion" From ef3739bdfb6b6ff3c76f3f7529391401f7691188 Mon Sep 17 00:00:00 2001 From: danielfeismann Date: Fri, 27 Mar 2026 14:44:49 +0100 Subject: [PATCH 4/5] abstract couchbase test setup --- .../AbstractCouchbaseWeatherSourceIT.groovy | 115 ++++++++++++++++++ .../CouchbaseWeatherSourceCosmoIT.groovy | 106 ++-------------- .../CouchbaseWeatherSourceIconIT.groovy | 106 ++-------------- 3 files changed, 137 insertions(+), 190 deletions(-) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy new file mode 100644 index 0000000000..147234bd64 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy @@ -0,0 +1,115 @@ +/* + * © 2026. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source.couchbase + +import edu.ie3.datamodel.io.connectors.CouchbaseConnector +import edu.ie3.test.helper.TestContainerHelper +import edu.ie3.test.helper.WeatherSourceTestHelper +import groovy.json.JsonSlurper +import org.testcontainers.couchbase.BucketDefinition +import org.testcontainers.couchbase.CouchbaseContainer +import org.testcontainers.spock.Testcontainers +import spock.lang.Shared +import spock.lang.Specification + +import java.time.Duration + +@Testcontainers +abstract class AbstractCouchbaseWeatherSourceIT extends Specification implements TestContainerHelper, WeatherSourceTestHelper { + + @Shared + BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") + + @Shared + CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:8.0.0") + .withBucket(bucketDefinition) + .withExposedPorts(8091, 8092, 8093, 8094, 11210) + .withStartupAttempts(3) + + @Shared + CouchbaseWeatherSource source + + static String coordinateIdColumnName = "coordinateid" + + abstract String getJsonResourcePath() + abstract Object getWeatherFactory() + abstract Object getCoordinateSource() + String getDtfPattern() { + return "yyyy-MM-dd'T'HH:mm:ssxxx" + } + + def setupSpec() { + // create an index for the document keys + couchbaseContainer.execInContainer("cbq", + "-e", "http://localhost:8093", + "-u", couchbaseContainer.username, + "-p", couchbaseContainer.password, + "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") + + // Create connector to import the data from json document + def connector = new CouchbaseConnector( + couchbaseContainer.connectionString, + bucketDefinition.name, + couchbaseContainer.username, + couchbaseContainer.password, + Duration.ofSeconds(30)) + + // Wait for bucket to be ready + println "Waiting for Couchbase bucket to be ready..." + int maxTries = 10 + boolean ready = false + for (int i = 0; i < maxTries; i++) { + try { + connector.getSourceFields() + ready = true + println "Couchbase bucket ready" + break + } catch (Exception ignored) { + if (i % 10 == 0) { + println "Waiting for bucket... (try ${i+1})" + } + sleep(1000) + } + } + + if (!ready) { + println "Couchbase bucket did not become ready in time!" + System.out.flush() + throw new RuntimeException("Couchbase bucket did not become ready in time!") + } + + // Insert test data from JSON file + println "Inserting test data from JSON file..." + def jsonFile = new File(getJsonResourcePath()) + def jsonSlurper = new JsonSlurper() + def weatherDocs = jsonSlurper.parse(jsonFile) as List + def insertCount = 0 + weatherDocs.each { doc -> + try { + def coordinateId = doc[coordinateIdColumnName] + def time = doc["time"] + def key = "weather::${coordinateId}::${time}" + connector.persist(key, doc).join() + insertCount++ + } catch (Exception ex) { + println "WARNING: Failed to insert document for coordinateid ${doc[coordinateIdColumnName]} and time ${doc["time"]}: ${ex.message}" + } + } + println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" + + source = new CouchbaseWeatherSource(connector, getCoordinateSource(), coordinateIdColumnName, getWeatherFactory(), getDtfPattern()) + println "setupSpec completed" + System.out.flush() + } + + def "The test container can establish a valid connection"() { + when: + def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + + then: + connector.connectionValid + } +} diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy index 2e8803efb4..1f62b15ba6 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy @@ -5,114 +5,30 @@ */ package edu.ie3.datamodel.io.source.couchbase -import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.CosmoTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.datamodel.models.value.WeatherValue import edu.ie3.test.common.CosmoWeatherTestData -import edu.ie3.test.helper.TestContainerHelper -import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.interval.ClosedInterval import org.locationtech.jts.geom.Point -import org.testcontainers.couchbase.BucketDefinition -import org.testcontainers.couchbase.CouchbaseContainer import org.testcontainers.spock.Testcontainers -import org.testcontainers.utility.MountableFile -import spock.lang.Shared -import spock.lang.Specification - -import java.time.Duration @Testcontainers -class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContainerHelper, WeatherSourceTestHelper { - - @Shared - BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") - - @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:8.0.0") - .withBucket(bucketDefinition) - .withExposedPorts(8091, 8092, 8093, 8094, 11210) - .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early - - @Shared - CouchbaseWeatherSource source - - static String coordinateIdColumnName = "coordinateid" - - def setupSpec() { - // create an index for the document keys - couchbaseContainer.execInContainer("cbq", - "-e", "http://localhost:8093", - "-u", couchbaseContainer.username, - "-p", couchbaseContainer.password, - "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") - - // Create connector to import the data from json document - def connector = new CouchbaseConnector( - couchbaseContainer.connectionString, - bucketDefinition.name, - couchbaseContainer.username, - couchbaseContainer.password, - Duration.ofSeconds(30)) - - // Wait for bucket to be ready - println "Waiting for Couchbase bucket to be ready..." - int maxTries = 10 - boolean ready = false - for (int i = 0; i < maxTries; i++) { - try { - connector.getSourceFields() - ready = true - println "Couchbase bucket ready" - break - } catch (Exception ex) { - if (i % 10 == 0) { - println "Waiting for bucket... (try ${i+1})" - } - sleep(1000) - } - } - - if (!ready) { - println "Couchbase bucket did not become ready in time!" - System.out.flush() - throw new RuntimeException("Couchbase bucket did not become ready in time!") - } - - // Insert test data from JSON file - println "Inserting test data from JSON file..." - def jsonFile = new File("src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/cosmo/weather.json") - def jsonSlurper = new groovy.json.JsonSlurper() - def weatherDocs = jsonSlurper.parse(jsonFile) as List - def insertCount = 0 - weatherDocs.each { doc -> - try { - def coordinateId = doc["coordinateid"] - def time = doc["time"] - def key = "weather::${coordinateId}::${time}" - connector.persist(key, doc).join() - insertCount++ - } catch (Exception ex) { - println "WARNING: Failed to insert document for coordinateid ${doc["coordinateid"]} and time ${doc["time"]}: ${ex.message}" - } - } - println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" - - def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new CosmoTimeBasedWeatherValueFactory() - source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) - println "setupSpec completed" - System.out.flush() +class CouchbaseWeatherSourceCosmoIT extends AbstractCouchbaseWeatherSourceIT { + @Override + String getJsonResourcePath() { + return "src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/cosmo/weather.json" } - def "The test container can establish a valid connection"() { - when: - def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + @Override + Object getWeatherFactory() { + return new CosmoTimeBasedWeatherValueFactory() + } - then: - connector.connectionValid + @Override + Object getCoordinateSource() { + return CosmoWeatherTestData.coordinateSource } def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy index 6c2244cfdb..bc1e5ed2d1 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy @@ -5,112 +5,28 @@ */ package edu.ie3.datamodel.io.source.couchbase -import edu.ie3.datamodel.io.connectors.CouchbaseConnector import edu.ie3.datamodel.io.factory.timeseries.IconTimeBasedWeatherValueFactory import edu.ie3.datamodel.models.timeseries.individual.IndividualTimeSeries import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue import edu.ie3.test.common.IconWeatherTestData -import edu.ie3.test.helper.TestContainerHelper -import edu.ie3.test.helper.WeatherSourceTestHelper import edu.ie3.util.interval.ClosedInterval -import groovy.json.JsonSlurper -import org.testcontainers.couchbase.BucketDefinition -import org.testcontainers.couchbase.CouchbaseContainer import org.testcontainers.spock.Testcontainers -import spock.lang.Shared -import spock.lang.Specification - -import java.time.Duration @Testcontainers -class CouchbaseWeatherSourceIconIT extends Specification implements TestContainerHelper, WeatherSourceTestHelper { - - @Shared - BucketDefinition bucketDefinition = new BucketDefinition("ie3_in") - - @Shared - CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:8.0.0") - .withBucket(bucketDefinition) - .withExposedPorts(8091, 8092, 8093, 8094, 11210) - .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early - - @Shared - CouchbaseWeatherSource source - - static String coordinateIdColumnName = "coordinateid" - - def setupSpec() { - // create an index for the document keys - couchbaseContainer.execInContainer("cbq", - "-e", "http://localhost:8093", - "-u", couchbaseContainer.username, - "-p", couchbaseContainer.password, - "-s", "CREATE index id_idx ON `" + bucketDefinition.name + "` (META().id);") - - // Create connector to import the data from json document - def connector = new CouchbaseConnector( - couchbaseContainer.connectionString, - bucketDefinition.name, - couchbaseContainer.username, - couchbaseContainer.password, - Duration.ofSeconds(30)) - - // Wait for bucket to be ready - println "Waiting for Couchbase bucket to be ready..." - int maxTries = 10 - boolean ready = false - for (int i = 0; i < maxTries; i++) { - try { - connector.getSourceFields() - ready = true - println "Couchbase bucket ready" - break - } catch (Exception ex) { - if (i % 10 == 0) { - println "Waiting for bucket... (try ${i+1})" - } - sleep(1000) - } - } - - if (!ready) { - println "Couchbase bucket did not become ready in time!" - System.out.flush() - throw new RuntimeException("Couchbase bucket did not become ready in time!") - } - - // Insert test data from JSON file - println "Inserting test data from JSON file..." - def jsonFile = new File("src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/icon/weather.json") - def jsonSlurper = new groovy.json.JsonSlurper() - def weatherDocs = jsonSlurper.parse(jsonFile) as List - def insertCount = 0 - weatherDocs.each { doc -> - try { - def coordinateId = doc["coordinateid"] - def time = doc["time"] - def key = "weather::${coordinateId}::${time}" - connector.persist(key, doc).join() - insertCount++ - } catch (Exception ex) { - println "WARNING: Failed to insert document for coordinateid ${doc["coordinateid"]} and time ${doc["time"]}: ${ex.message}" - } - } - println "Inserted ${insertCount}/${weatherDocs.size()} test documents from JSON file" - - def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx" - def weatherFactory = new IconTimeBasedWeatherValueFactory() - source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern) - println "setupSpec completed" - System.out.flush() +class CouchbaseWeatherSourceIconIT extends AbstractCouchbaseWeatherSourceIT { + @Override + String getJsonResourcePath() { + return "src/test/resources/edu/ie3/datamodel/io/source/couchbase/_weather/icon/weather.json" } - def "The test container can establish a valid connection"() { - when: - def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password) + @Override + Object getWeatherFactory() { + return new IconTimeBasedWeatherValueFactory() + } - then: - connector.connectionValid + @Override + Object getCoordinateSource() { + return IconWeatherTestData.coordinateSource } def "A CouchbaseWeatherSource can read and correctly parse a single value for a specific date and coordinate"() { From 56038d5e3f1dac18eb1d6e588fe871c022446325 Mon Sep 17 00:00:00 2001 From: danielfeismann Date: Tue, 31 Mar 2026 12:32:00 +0200 Subject: [PATCH 5/5] include feedback from review --- .../io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy index 147234bd64..8bbfef3075 100644 --- a/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy +++ b/src/test/groovy/edu/ie3/datamodel/io/source/couchbase/AbstractCouchbaseWeatherSourceIT.groovy @@ -27,7 +27,7 @@ abstract class AbstractCouchbaseWeatherSourceIT extends Specification implements CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:8.0.0") .withBucket(bucketDefinition) .withExposedPorts(8091, 8092, 8093, 8094, 11210) - .withStartupAttempts(3) + .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early @Shared CouchbaseWeatherSource source