Raster processing and digital elevation models

.run/basemap-dem.run.xml

@@ -0,0 +1,17 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="basemap-dem" type="Application" factoryName="Application">
+    <option name="MAIN_CLASS_NAME" value="org.apache.baremaps.cli.Baremaps" />
+    <module name="baremaps-cli" />
+    <option name="PROGRAM_PARAMETERS" value="dem serve --path $USER_HOME$/Downloads/gebco_2024.tif" />
+    <projectPathOnTarget name="projectPathOnTarget" value="/app" />
+    <target name="@@@LOCAL@@@" />
+    <extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
+      <option name="credential" />
+      <option name="region" />
+      <option name="useCurrentConnection" value="false" />
+    </extension>
+    <method v="2">
+      <option name="Make" enabled="true" />
+    </method>
+  </configuration>
+</component>

baremaps-cli/pom.xml

@@ -43,6 +43,10 @@ limitations under the License.
       <groupId>org.apache.baremaps</groupId>
       <artifactId>baremaps-core</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.baremaps</groupId>
+      <artifactId>baremaps-dem</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.baremaps</groupId>
       <artifactId>baremaps-server</artifactId>

baremaps-cli/src/main/java/org/apache/baremaps/cli/Baremaps.java

@@ -25,6 +25,7 @@
 import java.util.concurrent.Callable;
 import org.apache.baremaps.cli.Baremaps.VersionProvider;
 import org.apache.baremaps.cli.database.Database;
+import org.apache.baremaps.cli.dem.DEM;
 import org.apache.baremaps.cli.geocoder.Geocoder;
 import org.apache.baremaps.cli.iploc.IpLoc;
 import org.apache.baremaps.cli.map.Map;
@@ -42,7 +43,14 @@
     name = "baremaps",
     description = "A toolkit for producing vector tiles.",
     versionProvider = VersionProvider.class,
-    subcommands = {Workflow.class, Database.class, Map.class, Geocoder.class, IpLoc.class},
+    subcommands = {
+        Workflow.class,
+        Database.class,
+        Map.class,
+        Geocoder.class,
+        IpLoc.class,
+        DEM.class
+    },
     sortOptions = false)
 @SuppressWarnings("squid:S106")
 public class Baremaps implements Callable<Integer> {

baremaps-cli/src/main/java/org/apache/baremaps/cli/dem/DEM.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.cli.dem;
+
+
+
+import picocli.CommandLine;
+import picocli.CommandLine.Command;
+
+@Command(name = "dem", description = "DEM processing commands.",
+    subcommands = {
+        Serve.class,
+        VectorTileContours.class
+    },
+    sortOptions = false)
+@SuppressWarnings("squid:S106")
+public class DEM implements Runnable {
+
+  @Override
+  public void run() {
+    CommandLine.usage(this, System.out);
+  }
+}

baremaps-cli/src/main/java/org/apache/baremaps/cli/dem/Serve.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.cli.dem;
+
+import static org.apache.baremaps.utils.ObjectMapperUtils.objectMapper;
+
+import com.linecorp.armeria.common.*;
+import com.linecorp.armeria.server.Server;
+import com.linecorp.armeria.server.annotation.JacksonResponseConverterFunction;
+import com.linecorp.armeria.server.cors.CorsService;
+import com.linecorp.armeria.server.docs.DocService;
+import com.linecorp.armeria.server.file.HttpFile;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+import org.apache.baremaps.dem.ElevationUtils;
+import org.apache.baremaps.server.BufferedImageResource;
+import org.apache.baremaps.server.VectorTileResource;
+import org.apache.baremaps.tilestore.raster.*;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+@Command(name = "serve", description = "Start a tile server that serves elevation data.")
+public class Serve implements Callable<Integer> {
+
+  @Option(names = {"--host"}, paramLabel = "HOST", description = "The host of the server.")
+  private String host = "localhost";
+
+  @Option(names = {"--port"}, paramLabel = "PORT", description = "The port of the server.")
+  private int port = 9000;
+
+  @Option(names = {"--path"}, paramLabel = "PATH", description = "The path of a geoTIFF file.")
+  private Path path;
+
+  @Override
+  public Integer call() throws Exception {
+    // Initialize the tile stores
+    var geoTiffReader = new GeoTiffReader(path);
+    var rasterElevationTileStore = new TerrariumTileStore(geoTiffReader);
+    var rasterHillshadeTileStore =
+        new RasterHillshadeTileStore(
+            geoTiffReader,
+            ElevationUtils::terrariumToElevation);
+    var vectorHillshadeTileStore =
+        new VectorHillshadeTileStore(
+            geoTiffReader);
+    var vectorContourTileStore =
+        new VectorContourTileStore(geoTiffReader);
+
+    // Initialize the server
+    var objectMapper = objectMapper();
+    var jsonResponseConverter = new JacksonResponseConverterFunction(objectMapper);
+    var serverBuilder = Server.builder();
+    serverBuilder.http(port);
+
+    // Register the services
+    serverBuilder.annotatedService(
+        "/raster/elevation",
+        new BufferedImageResource(() -> rasterElevationTileStore),
+        jsonResponseConverter);
+    serverBuilder.annotatedService(
+        "/raster/hillshade",
+        new BufferedImageResource(() -> rasterHillshadeTileStore),
+        jsonResponseConverter);
+    serverBuilder.annotatedService(
+        "/vector/contour",
+        new VectorTileResource(() -> vectorContourTileStore),
+        jsonResponseConverter);
+    serverBuilder.annotatedService(
+        "/vector/hillshade",
+        new VectorTileResource(() -> vectorHillshadeTileStore),
+        jsonResponseConverter);
+
+    var index = HttpFile.of(ClassLoader.getSystemClassLoader(), "/dem/index.html");
+    serverBuilder.service("/", index.asService());
+
+    serverBuilder.decorator(CorsService.builderForAnyOrigin()
+        .allowAllRequestHeaders(true)
+        .allowRequestMethods(
+            HttpMethod.GET,
+            HttpMethod.POST,
+            HttpMethod.PUT,
+            HttpMethod.DELETE,
+            HttpMethod.OPTIONS,
+            HttpMethod.HEAD)
+        .allowCredentials()
+        .exposeHeaders(HttpHeaderNames.LOCATION)
+        .newDecorator());
+
+    serverBuilder.serviceUnder("/docs", new DocService());
+
+    serverBuilder.disableServerHeader();
+    serverBuilder.disableDateHeader();
+    var server = serverBuilder.build();
+
+    var startFuture = server.start();
+    startFuture.join();
+
+    var shutdownFuture = server.closeOnJvmShutdown();
+    shutdownFuture.join();
+
+    return 0;
+  }
+}

baremaps-cli/src/main/java/org/apache/baremaps/cli/dem/VectorTileContours.java

@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.baremaps.cli.dem;
+
+
+
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.baremaps.maplibre.tileset.Tileset;
+import org.apache.baremaps.maplibre.tileset.TilesetLayer;
+import org.apache.baremaps.openstreetmap.stream.ProgressLogger;
+import org.apache.baremaps.openstreetmap.stream.StreamUtils;
+import org.apache.baremaps.tilestore.TileCoord;
+import org.apache.baremaps.tilestore.TileEntry;
+import org.apache.baremaps.tilestore.TileStoreException;
+import org.apache.baremaps.tilestore.pmtiles.PMTilesStore;
+import org.apache.baremaps.tilestore.raster.*;
+import org.apache.baremaps.workflow.WorkflowException;
+import org.apache.baremaps.workflow.tasks.ExportVectorTiles;
+import org.locationtech.jts.geom.Envelope;
+import picocli.CommandLine.Command;
+import picocli.CommandLine.Option;
+
+@Command(name = "vector-contours", description = "Generate vector contours from a DEM.")
+@SuppressWarnings({"squid:S106", "squid:S3864"})
+public class VectorTileContours implements Callable<Integer> {
+
+  @Option(names = {"--path"}, paramLabel = "PATH", description = "The path of a geoTIFF file.")
+  private Path path;
+
+  @Option(names = {"--repository"}, paramLabel = "REPOSITORY", description = "The tile repository.",
+      required = true)
+  private Path repository;
+
+  @Option(names = {"--format"}, paramLabel = "FORMAT",
+      description = "The format of the repository.")
+  private ExportVectorTiles.Format format = ExportVectorTiles.Format.FILE;
+
+
+  @Override
+  public Integer call() throws Exception {
+    var contourLayer = new TilesetLayer();
+    contourLayer.setId("contours");
+
+    var tileset = new Tileset();
+    tileset.setName("contours");
+    tileset.setMinzoom(2);
+    tileset.setMaxzoom(10);
+    tileset.setCenter(List.of(0d, 0d, 1d));
+    tileset.setBounds(List.of(-180d, -85.0511d, 180d, 85.0511d));
+    tileset.setVectorLayers(List.of(contourLayer));
+
+    // Initialize the tile stores
+    try (var geoTiffReader = new GeoTiffReader(path);
+        var sourceTileStore = new VectorContourTileStore(geoTiffReader);
+        var targetTileStore = new PMTilesStore(repository, tileset);) {
+
+      var envelope = new Envelope(-180, 180, -85.0511, 85.0511);
+      var count = TileCoord.count(envelope, 0, 10);
+
+      var tileCoordIterator =
+          TileCoord.iterator(envelope, 2, 10);
+      var tileCoordStream =
+          StreamUtils.stream(tileCoordIterator).peek(new ProgressLogger<>(count, 1000));
+
+
+      var bufferedTileEntryStream = StreamUtils.bufferInCompletionOrder(tileCoordStream, tile -> {
+
+        try {
+          return new TileEntry<>(tile, sourceTileStore.read(tile));
+        } catch (TileStoreException e) {
+          throw new WorkflowException(e);
+        } finally {
+          System.out.println("Processing tile " + tile);
+        }
+      }, 8);
+
+      var partitionedTileEntryStream = StreamUtils.partition(bufferedTileEntryStream, 8);
+      partitionedTileEntryStream.forEach(batch -> {
+        try {
+          targetTileStore.write(batch);
+        } catch (TileStoreException e) {
+          throw new WorkflowException(e);
+        }
+      });
+      return 0;
+    }
+  }
+}

baremaps-cli/src/main/java/org/apache/baremaps/cli/map/Dev.java

@@ -27,6 +27,7 @@
 import com.linecorp.armeria.server.file.FileService;
 import com.linecorp.armeria.server.file.HttpFile;
 import java.io.IOException;
+import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.util.concurrent.Callable;
 import java.util.function.Supplier;
@@ -37,8 +38,8 @@
 import org.apache.baremaps.maplibre.tileset.Tileset;
 import org.apache.baremaps.server.ChangeResource;
 import org.apache.baremaps.server.StyleResource;
-import org.apache.baremaps.server.TileResource;
 import org.apache.baremaps.server.TilesetResource;
+import org.apache.baremaps.server.VectorTileResource;
 import org.apache.baremaps.tilestore.TileStore;
 import org.apache.baremaps.tilestore.postgres.PostgresTileStore;
 import org.apache.baremaps.utils.PostgresUtils;
@@ -96,7 +97,7 @@ public Integer call() throws Exception {
       }
     };
 
-    var tileStoreSupplier = (Supplier<TileStore>) () -> {
+    var tileStoreSupplier = (Supplier<TileStore<ByteBuffer>>) () -> {
       var tileJSON = tilesetSupplier.get();
       return new PostgresTileStore(datasource, tileJSON);
     };
@@ -116,7 +117,8 @@ public Integer call() throws Exception {
     var jsonResponseConverter = new JacksonResponseConverterFunction(objectMapper);
     serverBuilder.annotatedService(new ChangeResource(tilesetPath, stylePath),
         jsonResponseConverter);
-    serverBuilder.annotatedService(new TileResource(tileStoreSupplier), jsonResponseConverter);
+    serverBuilder.annotatedService("/tiles", new VectorTileResource(tileStoreSupplier),
+        jsonResponseConverter);
     serverBuilder.annotatedService(new StyleResource(styleSupplier), jsonResponseConverter);
     serverBuilder.annotatedService(new TilesetResource(tilesetSupplier), jsonResponseConverter);
 

baremaps-cli/src/main/java/org/apache/baremaps/cli/map/MBTiles.java

@@ -28,6 +28,7 @@
 import com.linecorp.armeria.server.docs.DocService;
 import com.linecorp.armeria.server.file.FileService;
 import com.linecorp.armeria.server.file.HttpFile;
+import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.util.concurrent.Callable;
 import java.util.function.Supplier;
@@ -36,9 +37,9 @@
 import org.apache.baremaps.maplibre.style.Style;
 import org.apache.baremaps.maplibre.tilejson.TileJSON;
 import org.apache.baremaps.server.*;
-import org.apache.baremaps.tilestore.TileCache;
 import org.apache.baremaps.tilestore.TileStore;
 import org.apache.baremaps.tilestore.mbtiles.MBTilesStore;
+import org.apache.baremaps.tilestore.vector.VectorTileCache;
 import org.apache.baremaps.utils.SqliteUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -83,8 +84,8 @@ public Integer call() throws Exception {
 
     var datasource = SqliteUtils.createDataSource(mbtilesPath, true);
     try (var tileStore = new MBTilesStore(datasource);
-        var tileCache = new TileCache(tileStore, caffeineSpec)) {
-      var tileStoreSupplier = (Supplier<TileStore>) () -> tileCache;
+        var tileCache = new VectorTileCache(tileStore, caffeineSpec)) {
+      var tileStoreSupplier = (Supplier<TileStore<ByteBuffer>>) () -> tileCache;
 
       var style = objectMapper.readValue(configReader.read(stylePath), Style.class);
       var styleSupplier = (Supplier<Style>) () -> style;
@@ -96,7 +97,8 @@ public Integer call() throws Exception {
       serverBuilder.http(port);
 
       var jsonResponseConverter = new JacksonResponseConverterFunction(objectMapper);
-      serverBuilder.annotatedService(new TileResource(tileStoreSupplier), jsonResponseConverter);
+      serverBuilder.annotatedService("/tiles", new VectorTileResource(tileStoreSupplier),
+          jsonResponseConverter);
       serverBuilder.annotatedService(new StyleResource(styleSupplier), jsonResponseConverter);
       serverBuilder.annotatedService(new TileJSONResource(tileJSONSupplier), jsonResponseConverter);