Add MBTiles benchmark and refactor TileStore

apache · Jun 13, 2023 · 7cf110e · 7cf110e
1 parent 42e3c83
commit 7cf110e
Show file tree

Hide file tree

Showing 12 changed files with 137 additions and 33 deletions.
diff --git a/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java b/baremaps-benchmark/src/main/java/org/apache/baremaps/benchmarks/MBTilesBenchmark.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.baremaps.benchmarks;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import org.apache.baremaps.tilestore.TileCoord;
+import org.apache.baremaps.tilestore.TileStoreException;
+import org.apache.baremaps.tilestore.mbtiles.MBTiles;
+import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.RunnerException;
+import org.openjdk.jmh.runner.options.Options;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+@State(Scope.Benchmark)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Fork(value = 1, warmups = 1)
+public class MBTilesBenchmark {
+
+  public Random random = new Random(0);
+
+  @Param({"10", "100", "1000"})
+  public int iterations;
+
+  private MBTiles mbTiles;
+
+  @Setup
+  public void setup() throws IOException, TileStoreException {
+    var sqliteFile = File.createTempFile("baremaps", ".sqlite");
+    sqliteFile.deleteOnExit();
+    var sqliteDataSource = ExportVectorTiles.createDataSource(sqliteFile.toPath());
+    mbTiles = new MBTiles(sqliteDataSource);
+    mbTiles.initializeDatabase();
+  }
+
+  @Benchmark
+  @BenchmarkMode(Mode.SingleShotTime)
+  public void writeMBTiles(MBTilesBenchmark benchmark) throws TileStoreException {
+    for (int i = 0; i < benchmark.iterations; i++) {
+      var bytes = new byte[1 << 16];
+      random.nextBytes(bytes);
+      mbTiles.put(new TileCoord(0, 0, i), ByteBuffer.wrap(bytes));
+    }
+  }
+
+  public static void main(String[] args) throws RunnerException {
+    Options opt =
+        new OptionsBuilder().include(MBTilesBenchmark.class.getSimpleName()).forks(1).build();
+    new Runner(opt).run();
+  }
+
+
+}
diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileCache.java
@@ -50,10 +50,10 @@ public TileCache(TileStore tileStore, CaffeineSpec spec) {
 
   /** {@inheritDoc} */
   @Override
-  public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+  public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
     return cache.get(tileCoord, t -> {
       try {
-        var buffer = tileStore.read(t);
+        var buffer = tileStore.get(t);
         if (buffer == null) {
           return null;
         } else {
@@ -68,8 +68,8 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
 
   /** {@inheritDoc} */
   @Override
-  public void write(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
-    tileStore.write(tileCoord, bytes);
+  public void put(TileCoord tileCoord, ByteBuffer bytes) throws TileStoreException {
+    tileStore.put(tileCoord, bytes);
     cache.invalidate(tileCoord);
   }
 

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileChannel.java
@@ -53,9 +53,9 @@ public TileChannel(TileStore source, TileStore target, boolean deleteEmptyTiles)
   @Override
   public void accept(TileCoord tileCoord) {
     try {
-      ByteBuffer blob = source.read(tileCoord);
+      ByteBuffer blob = source.get(tileCoord);
       if (blob != null) {
-        target.write(tileCoord, blob);
+        target.put(tileCoord, blob);
       } else if (deleteEmptyTiles) {
         target.delete(tileCoord);
       }

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/TileStore.java
@@ -15,27 +15,57 @@
 
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
 
 /** Represents a store for tiles. */
 public interface TileStore {
 
   /**
-   * Reads the content of a tile.
+   * Gets the content of a tile.
    *
    * @param tileCoord the tile coordinate
    * @return the content of the tile
    * @throws TileStoreException
    */
-  ByteBuffer read(TileCoord tileCoord) throws TileStoreException;
+  ByteBuffer get(TileCoord tileCoord) throws TileStoreException;
 
   /**
-   * Writes the content of a tile.
+   * Gets the content of several tiles.
+   *
+   * @param tileCoords the tile coordinates
+   * @return the content of the tiles
+   * @throws TileStoreException
+   */
+  default List<ByteBuffer> get(List<TileCoord> tileCoords) throws TileStoreException {
+    var blobs = new ArrayList<ByteBuffer>(tileCoords.size());
+    for (var tileCoord : tileCoords) {
+      blobs.add(get(tileCoord));
+    }
+    return blobs;
+  }
+
+  /**
+   * Puts the content of a tile.
    *
    * @param tileCoord the tile coordinate
    * @param blob the content of the tile
    * @throws TileStoreException
    */
-  void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
+  void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException;
+
+  /**
+   * Puts the content of several tiles.
+   *
+   * @param tileCoords the tile coordinates
+   * @param blobs the content of the tiles
+   * @throws TileStoreException
+   */
+  default void put(List<TileCoord> tileCoords, List<ByteBuffer> blobs) throws TileStoreException {
+    for (int i = 0; i < tileCoords.size(); i++) {
+      put(tileCoords.get(i), blobs.get(i));
+    }
+  }
 
   /**
    * Deletes the content of a tile.

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/file/FileTileStore.java
@@ -38,7 +38,7 @@ public FileTileStore(Path path) {
 
   /** {@inheritDoc} */
   @Override
-  public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+  public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
     try {
       return ByteBuffer.wrap(Files.readAllBytes(resolve(tileCoord)));
     } catch (IOException e) {
@@ -48,7 +48,7 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
 
   /** {@inheritDoc} */
   @Override
-  public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+  public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
     try {
       var file = resolve(tileCoord);
       Files.createDirectories(file.getParent());

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/mbtiles/MBTiles.java
@@ -72,7 +72,7 @@ public MBTiles(DataSource dataSource) {
 
   /** {@inheritDoc} */
   @Override
-  public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+  public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
     try (Connection connection = dataSource.getConnection();
         PreparedStatement statement = connection.prepareStatement(SELECT_TILE)) {
       statement.setInt(1, tileCoord.z());
@@ -92,7 +92,7 @@ public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
 
   /** {@inheritDoc} */
   @Override
-  public void write(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
+  public void put(TileCoord tileCoord, ByteBuffer blob) throws TileStoreException {
     try (Connection connection = dataSource.getConnection();
         PreparedStatement statement = connection.prepareStatement(INSERT_TILE)) {
       statement.setInt(1, tileCoord.z());

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java b/baremaps-core/src/main/java/org/apache/baremaps/tilestore/postgres/PostgresTileStore.java
@@ -99,7 +99,7 @@ public PostgresTileStore(DataSource datasource, Tileset tileset) {
 
   /** {@inheritDoc} */
   @Override
-  public ByteBuffer read(TileCoord tileCoord) throws TileStoreException {
+  public ByteBuffer get(TileCoord tileCoord) throws TileStoreException {
     try (Connection connection = datasource.getConnection();
         Statement statement = connection.createStatement();
         ByteArrayOutputStream data = new ByteArrayOutputStream()) {
@@ -239,7 +239,7 @@ protected String tileEnvelope(TileCoord tileCoord) {
   }
 
   /** This operation is not supported. */
-  public void write(TileCoord tileCoord, ByteBuffer blob) {
+  public void put(TileCoord tileCoord, ByteBuffer blob) {
     throw new UnsupportedOperationException("The postgis tile store is read only");
   }
 

diff --git a/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java b/baremaps-core/src/main/java/org/apache/baremaps/workflow/tasks/ExportVectorTiles.java
@@ -87,17 +87,7 @@ private TileStore targetTileStore(Tileset source) throws TileStoreException, IOE
     if (mbtiles) {
       Files.deleteIfExists(repository);
 
-      var sqliteConfig = new SQLiteConfig();
-      sqliteConfig.setCacheSize(1000000);
-      sqliteConfig.setPageSize(65536);
-      sqliteConfig.setJournalMode(JournalMode.OFF);
-      sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
-      sqliteConfig.setSynchronous(SynchronousMode.OFF);
-      sqliteConfig.setTempStore(TempStore.MEMORY);
-
-      var sqliteDataSource = new SQLiteDataSource();
-      sqliteDataSource.setConfig(sqliteConfig);
-      sqliteDataSource.setUrl("jdbc:sqlite:" + repository);
+      var sqliteDataSource = createDataSource(repository);
 
       var hikariConfig = new HikariConfig();
       hikariConfig.setDataSource(sqliteDataSource);
@@ -114,6 +104,22 @@ private TileStore targetTileStore(Tileset source) throws TileStoreException, IOE
     }
   }
 
+  public static SQLiteDataSource createDataSource(Path path) {
+    var sqliteConfig = new SQLiteConfig();
+    sqliteConfig.setCacheSize(1000000);
+    sqliteConfig.setPageSize(65536);
+    sqliteConfig.setJournalMode(JournalMode.OFF);
+    sqliteConfig.setLockingMode(LockingMode.EXCLUSIVE);
+    sqliteConfig.setSynchronous(SynchronousMode.OFF);
+    sqliteConfig.setTempStore(TempStore.MEMORY);
+
+    var sqliteDataSource = new SQLiteDataSource();
+    sqliteDataSource.setConfig(sqliteConfig);
+    sqliteDataSource.setUrl("jdbc:sqlite:" + path);
+
+    return sqliteDataSource;
+  }
+
   private Map<String, String> metadata(Tileset tileset) throws JsonProcessingException {
     var metadata = new HashMap<String, String>();
 

diff --git a/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java b/baremaps-core/src/test/java/org/apache/baremaps/tilestore/TileStoreTest.java
@@ -31,15 +31,15 @@ void readWriteDeleteTile() throws Exception {
     ByteBuffer blob = ByteBuffer.wrap("tile_content".getBytes());
 
     // Write data
-    tileStore.write(tileCoord, blob);
+    tileStore.put(tileCoord, blob);
 
     // Read the data
-    ByteBuffer inputStream = tileStore.read(tileCoord);
+    ByteBuffer inputStream = tileStore.get(tileCoord);
     assertArrayEquals(blob.array(), inputStream.array());
 
     // Delete the data
     tileStore.delete(tileCoord);
-    assertThrows(TileStoreException.class, () -> tileStore.read(tileCoord));
+    assertThrows(TileStoreException.class, () -> tileStore.get(tileCoord));
   }
 
   public abstract TileStore createTileStore() throws Exception;

diff --git a/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java b/baremaps-ogcapi/src/main/java/org/apache/baremaps/ogcapi/TilesResource.java
@@ -100,7 +100,7 @@ public Response getTile(String tileSetId, String tileMatrix, Integer tileRow, In
     int y = tileCol;
     TileCoord tileCoord = new TileCoord(x, y, z);
     try {
-      ByteBuffer blob = tileStore.read(tileCoord);
+      ByteBuffer blob = tileStore.get(tileCoord);
       if (blob != null) {
         return Response.status(200) // lgtm [java/xss]
             .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")

diff --git a/baremaps-server/src/main/java/org/apache/baremaps/server/DevResources.java b/baremaps-server/src/main/java/org/apache/baremaps/server/DevResources.java
@@ -134,7 +134,7 @@ public Response getTile(@PathParam("z") int z, @PathParam("x") int x, @PathParam
       var tileStore = new PostgresTileStore(dataSource,
           objectMapper.readValue(configReader.read(tileset), Tileset.class));
       var tileCoord = new TileCoord(x, y, z);
-      var blob = tileStore.read(tileCoord);
+      var blob = tileStore.get(tileCoord);
       if (blob != null) {
         return Response.status(200)
             .header(CONTENT_TYPE, TILE_TYPE)

diff --git a/baremaps-server/src/main/java/org/apache/baremaps/server/ServerResources.java b/baremaps-server/src/main/java/org/apache/baremaps/server/ServerResources.java
@@ -78,7 +78,7 @@ public TileJSON getTileset() {
   public Response getTile(@PathParam("z") int z, @PathParam("x") int x, @PathParam("y") int y) {
     TileCoord tileCoord = new TileCoord(x, y, z);
     try {
-      ByteBuffer blob = tileStore.read(tileCoord);
+      ByteBuffer blob = tileStore.get(tileCoord);
       if (blob != null) {
         byte[] bytes = new byte[blob.remaining()];
         blob.get(bytes);