get projjson from metadata

roundtrip projjson sort of

actually pass the value of extensions enabled to the Parquet reader

remove accidentally added file

rename files from geometry to geospatial

rename geometry to geospatial

more geometry to geospatial

maybe better geospatial statistics python output

use string view for algorithm_name

schema_internal tweaks

column writer and metadata

geometry_stats -> geospatial_stats

fix build

reader test comments

clarify the point size constant

refactor test util for generating WKB

better test utils

maybe fix test

lint

slightly better handling of wraparound

collapse missing header

remove unneded constructors

fewer constructors

improve comments, equality

lowercase trivial methods

add experimental note

correct comment

throw instead of attempt to correctly write stats for dictionaries

two more WKB utility edits

add upper bound/lower bound

more iterating on the geospatial statistics

remove one more constructor

AppendValues -> Append

rename constants

fix comment

better tostring for boundingbox

all the spans, better

fix  whitespace

check bounds empty before emitting

offload enum converts ro thrift_internal

ensure stats set is checked at least once

remove unused variable

snake case geospatial stat to geo_stats

GeospatialStatistics -> GeoStatistics

deparameterize test that no longer depends on data page version

remove write_geospatial_types option

Update python build for no geospatial logical types and renamed statistics

separate JSON-requiring code and ensure tests pass

separate json-requiring things to their own file

conditionally link to rapidjson

fix comment

fix tidy

link privately where possible

add the crs context

maybe better context

remember to include config to not skip tests

Add crs context to writer

move crs logic in the other direction to internal_json

fix comment

use a hash in the projjson key

dev lint

fix accidental diffs

use enum class for EdgeInterpolationAlgorithm

pragma once the offending header

use k syntax for constant prefixes

add an is_empty()

revert checked cast

export classes for Windows

try something for MingW

remove unnecessary constructors

don't test multiple data page version in reader test

Author: paleolimbot

cpp/src/arrow/dataset/file_parquet.cc

@@ -133,6 +133,8 @@ parquet::ArrowReaderProperties MakeArrowReaderProperties(
   arrow_properties.set_io_context(
       parquet_scan_options.arrow_reader_properties->io_context());
   arrow_properties.set_use_threads(options.use_threads);
+  arrow_properties.set_arrow_extensions_enabled(
+      parquet_scan_options.arrow_reader_properties->get_arrow_extensions_enabled());
   return arrow_properties;
 }
 

cpp/src/parquet/CMakeLists.txt

@@ -171,8 +171,9 @@ set(PARQUET_SRCS
     exception.cc
     file_reader.cc
     file_writer.cc
-    geometry_statistics.cc
-    geometry_util_internal.cc
+    geospatial_statistics.cc
+    geospatial_util_internal.cc
+    geospatial_util_internal_json.cc
     level_comparison.cc
     level_conversion.cc
     metadata.cc
@@ -261,9 +262,11 @@ endif()
 if(NOT PARQUET_MINIMAL_DEPENDENCY)
   list(APPEND PARQUET_SHARED_LINK_LIBS arrow_shared)
 
-  # TODO(paleolimbot): Remove once sample files are generated
-  list(APPEND PARQUET_SHARED_LINK_LIBS RapidJSON)
-  list(APPEND PARQUET_STATIC_LINK_LIBS RapidJSON)
+  # TODO(paleolimbot): Make sure this is OK or remove if not!
+  if(ARROW_JSON)
+    list(APPEND PARQUET_SHARED_PRIVATE_LINK_LIBS RapidJSON)
+    list(APPEND PARQUET_STATIC_LINK_LIBS RapidJSON)
+  endif()
 
   # These are libraries that we will link privately with parquet_shared (as they
   # do not need to be linked transitively by other linkers)
@@ -378,8 +381,8 @@ add_parquet_test(internals-test
                  statistics_test.cc
                  encoding_test.cc
                  metadata_test.cc
-                 geometry_statistics_test.cc
-                 geometry_util_internal_test.cc
+                 geospatial_statistics_test.cc
+                 geospatial_util_internal_test.cc
                  page_index_test.cc
                  public_api_test.cc
                  size_statistics_test.cc

cpp/src/parquet/api/reader.h

@@ -22,7 +22,7 @@
 #include "parquet/column_scanner.h"
 #include "parquet/exception.h"
 #include "parquet/file_reader.h"
-#include "parquet/geometry_statistics.h"
+#include "parquet/geospatial_statistics.h"
 #include "parquet/metadata.h"
 #include "parquet/platform.h"
 #include "parquet/printer.h"

cpp/src/parquet/arrow/arrow_reader_writer_test.cc

@@ -1489,15 +1489,17 @@ TEST_F(TestGeoArrowParquetIO, GeoArrowExtension) {
 
   // Build a binary WKB array with at least one null value
   ::arrow::BinaryBuilder builder;
-  std::array<char, test::kWkbPointSize> item;
+
   for (int k = 0; k < 10; k++) {
-    test::GenerateWKBPoint(reinterpret_cast<uint8_t*>(item.data()), k, k + 1);
-    ASSERT_OK(builder.AppendValues({std::string(item.data(), item.size())}));
+    std::string item = test::MakeWKBPoint(
+        {static_cast<double>(k), static_cast<double>(k + 1)}, false, false);
+    ASSERT_OK(builder.Append(item));
   }
   ASSERT_OK(builder.AppendNull());
   for (int k = 0; k < 5; k++) {
-    test::GenerateWKBPoint(reinterpret_cast<uint8_t*>(item.data()), k, k + 1);
-    ASSERT_OK(builder.AppendValues({std::string(item.data(), item.size())}));
+    std::string item = test::MakeWKBPoint(
+        {static_cast<double>(k), static_cast<double>(k + 1)}, false, false);
+    ASSERT_OK(builder.Append(item));
   }
 
   ASSERT_OK_AND_ASSIGN(const auto binary_array, builder.Finish());
@@ -1512,9 +1514,7 @@ TEST_F(TestGeoArrowParquetIO, GeoArrowExtension) {
 
   // When the original Arrow schema isn't stored and Arrow extensions are disabled,
   // LogicalType::GEOMETRY is read as utf8.
-  auto writer_properties = ::parquet::ArrowWriterProperties::Builder()
-                               .write_geospatial_logical_types()
-                               ->build();
+  auto writer_properties = default_arrow_writer_properties();
   this->RoundTripSingleColumn(wkb_array, binary_array, writer_properties);
   this->RoundTripSingleColumn(large_wkb_array, binary_array, writer_properties);
 

cpp/src/parquet/arrow/arrow_schema_test.cc

@@ -19,7 +19,6 @@
 #include <vector>
 
 #include "gmock/gmock-matchers.h"
-#include "gmock/gmock.h"
 #include "gtest/gtest.h"
 
 #include "parquet/arrow/reader.h"
@@ -32,6 +31,7 @@
 #include "parquet/thrift_internal.h"
 
 #include "arrow/array.h"
+#include "arrow/config.h"
 #include "arrow/extension/json.h"
 #include "arrow/ipc/writer.h"
 #include "arrow/testing/extension_type.h"
@@ -1251,15 +1251,12 @@ TEST_F(TestConvertArrowSchema, ParquetFlatPrimitivesAsDictionaries) {
 }
 
 TEST_F(TestConvertArrowSchema, ParquetGeoArrowCrsLonLat) {
+#ifdef ARROW_JSON
   // All the Arrow Schemas below should convert to the type defaults for GEOMETRY
   // and GEOGRAPHY when GeoArrow extension types are registered and the appropriate
   // writer option is set.
   ::arrow::ExtensionTypeGuard guard(test::geoarrow_wkb());
 
-  ArrowWriterProperties::Builder builder;
-  builder.write_geospatial_logical_types();
-  auto arrow_properties = builder.build();
-
   std::vector<NodePtr> parquet_fields;
   parquet_fields.push_back(PrimitiveNode::Make("geometry", Repetition::OPTIONAL,
                                                LogicalType::Geometry(),
@@ -1289,22 +1286,22 @@ TEST_F(TestConvertArrowSchema, ParquetGeoArrowCrsLonLat) {
                                           R"(, "edges": "spherical"})"),
                        true)};
 
-    ASSERT_OK(ConvertSchema(arrow_fields, arrow_properties));
+    ASSERT_OK(ConvertSchema(arrow_fields));
     ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields));
   }
+#else
+  GTEST_SKIP() << "non-default CRS testing requires ARROW_JSON";
+#endif
 }
 
 TEST_F(TestConvertArrowSchema, ParquetGeoArrowCrsSrid) {
+#ifdef ARROW_JSON
   // Checks the conversion between GeoArrow's crs_type: srid and Parquet's srid:XXX.
   // SRID (spatial reference identifier) is an opaque application specific identifier
   // that GeoArrow will transport but refuse to resolve if required for a spatial
   // operation.
   ::arrow::ExtensionTypeGuard guard(test::geoarrow_wkb());
 
-  ArrowWriterProperties::Builder builder;
-  builder.write_geospatial_logical_types();
-  auto arrow_properties = builder.build();
-
   std::vector<NodePtr> parquet_fields;
   parquet_fields.push_back(PrimitiveNode::Make("geometry", Repetition::OPTIONAL,
                                                LogicalType::Geometry("srid:1234"),
@@ -1321,40 +1318,57 @@ TEST_F(TestConvertArrowSchema, ParquetGeoArrowCrsSrid) {
                          R"({"crs": "5678", "crs_type": "srid", "edges": "spherical"})"),
                      true)};
 
-  ASSERT_OK(ConvertSchema(arrow_fields, arrow_properties));
+  ASSERT_OK(ConvertSchema(arrow_fields));
   ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields));
+#else
+  GTEST_SKIP() << "non-default CRS testing requires ARROW_JSON";
+#endif
 }
 
 TEST_F(TestConvertArrowSchema, ParquetGeoArrowCrsProjjson) {
-  GTEST_SKIP() << "GeoArrow/PROJJSON support not yet implemented";
-
+#ifdef ARROW_JSON
   // Checks the conversion between GeoArrow that contains non-lon/lat PROJJSON
   // to Parquet. Almost all GeoArrow types that arrive at the Parquet reader
   // will have their CRS expressed in this way.
   ::arrow::ExtensionTypeGuard guard(test::geoarrow_wkb());
 
-  ArrowWriterProperties::Builder builder;
-  builder.write_geospatial_logical_types();
-  auto arrow_properties = builder.build();
+  std::vector<std::shared_ptr<Field>> arrow_fields = {
+      ::arrow::field(
+          "geometry",
+          test::geoarrow_wkb(R"({"crs": {"key0": "value0"}, "crs_type": "projjson"})"),
+          true),
+      ::arrow::field(
+          "geography",
+          test::geoarrow_wkb(
+              R"({"crs": {"key1": "value1"}, "crs_type": "projjson", "edges": "spherical"})"),
+          true)};
+
+  auto arrow_properties = default_arrow_writer_properties();
+  ASSERT_OK(ConvertSchema(arrow_fields, arrow_properties));
 
-  std::vector<NodePtr> parquet_fields;
-  parquet_fields.push_back(PrimitiveNode::Make("geometry", Repetition::OPTIONAL,
-                                               LogicalType::Geometry("projjson:1234"),
-                                               ParquetType::BYTE_ARRAY));
-  parquet_fields.push_back(PrimitiveNode::Make("geography", Repetition::OPTIONAL,
-                                               LogicalType::Geography("projjson:5678"),
-                                               ParquetType::BYTE_ARRAY));
+  std::shared_ptr<::arrow::KeyValueMetadata> crs_metadata =
+      ::arrow::key_value_metadata({}, {});
+  arrow_properties->geo_crs_context()->AddProjjsonCrsFieldsToFileMetadata(
+      crs_metadata.get());
 
-  std::vector<std::shared_ptr<Field>> arrow_fields = {
-      ::arrow::field("geometry",
-                     test::geoarrow_wkb(R"({"crs": "1234", "crs_type": "srid"})"), true),
-      ::arrow::field("geography",
-                     test::geoarrow_wkb(
-                         R"({"crs": "5678", "crs_type": "srid", "edges": "spherical"})"),
-                     true)};
+  ASSERT_EQ(crs_metadata->size(), 2);
+  ASSERT_EQ(crs_metadata->value(0), R"({"key0":"value0"})");
+  ASSERT_EQ(crs_metadata->value(1), R"({"key1":"value1"})");
+
+  std::vector<NodePtr> parquet_fields;
+  parquet_fields.push_back(
+      PrimitiveNode::Make("geometry", Repetition::OPTIONAL,
+                          LogicalType::Geometry("projjson:" + crs_metadata->key(0)),
+                          ParquetType::BYTE_ARRAY));
+  parquet_fields.push_back(
+      PrimitiveNode::Make("geography", Repetition::OPTIONAL,
+                          LogicalType::Geography("projjson:" + crs_metadata->key(1)),
+                          ParquetType::BYTE_ARRAY));
 
-  ASSERT_OK(ConvertSchema(arrow_fields, arrow_properties));
   ASSERT_NO_FATAL_FAILURE(CheckFlatSchema(parquet_fields));
+#else
+  GTEST_SKIP() << "non-default CRS testing requires ARROW_JSON";
+#endif
 }
 
 TEST_F(TestConvertArrowSchema, ParquetLists) {

cpp/src/parquet/arrow/schema.cc

@@ -21,12 +21,7 @@
 #include <string>
 #include <vector>
 
-// TODO(paleolimbot): Remove once example files are generated
-#include "arrow/json/rapidjson_defs.h"  // IWYU pragma: keep
-
-#include <rapidjson/document.h>
-#include <rapidjson/writer.h>
-
+#include "arrow/config.h"
 #include "arrow/extension/json.h"
 #include "arrow/extension_type.h"
 #include "arrow/io/memory.h"
@@ -42,6 +37,7 @@
 
 #include "parquet/arrow/schema_internal.h"
 #include "parquet/exception.h"
+#include "parquet/geospatial_util_internal_json.h"
 #include "parquet/metadata.h"
 #include "parquet/properties.h"
 #include "parquet/types.h"
@@ -249,90 +245,21 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type,
   return Status::OK();
 }
 
-// TODO(paleolimbot): Remove once example files are written
-Result<std::string> GeospatialGeoArrowCrsToParquetCrs(
-    const ::arrow::rapidjson::Document& document,
-    const ArrowWriterProperties& arrow_properties) {
-  namespace rj = ::arrow::rapidjson;
-
-  std::string crs_type;
-  if (document.HasMember("crs_type")) {
-    crs_type = document["crs_type"].GetString();
-  }
-
-  if (!document.HasMember("crs") || document["crs"].IsNull()) {
-    // Parquet GEOMETRY/GEOGRAPHY do not have a concept of a null/missing
-    // CRS, but an omitted one is more likely to have meant "lon/lat" than
-    // a truly unspecified one (i.e., Engineering CRS with arbitrary XY units)
-    return "";
-  }
-
-  const auto& json_crs = document["crs"];
-  if (json_crs.IsString() && crs_type == "srid") {
-    // srid is an application-specific identifier. GeoArrow lets this be propagated via
-    // "crs_type": "srid".
-    return std::string("srid:") + json_crs.GetString();
-  } else if (json_crs.IsString() &&
-             (json_crs == "EPSG:4326" || json_crs == "OGC:CRS84")) {
-    // crs can be left empty because these cases both correspond to
-    // longitude/latitude in WGS84 according to the Parquet specification
-    return "";
-  } else if (json_crs.IsObject()) {
-    if (json_crs.HasMember("id")) {
-      const auto& identifier = json_crs["id"];
-      if (identifier.HasMember("authority") && identifier.HasMember("code")) {
-        if (identifier["authority"] == "OGC" && identifier["code"] == "CRS84") {
-          // longitude/latitude
-          return "";
-        } else if (identifier["authority"] == "EPSG" && identifier["code"] == 4326) {
-          // longitude/latitude
-          return "";
-        }
-      }
-    }
-
-    // TODO(paleolimbot) this is not quite correct because we're supposed to put this
-    // in the metadata according to the spec. We need to find a way to put
-    // this in the arrow_properties/file metadata via a CrsProvider or something.
-    rj::StringBuffer buffer;
-    rj::Writer<rj::StringBuffer> writer(buffer);
-    document.Accept(writer);
-    return std::string("projjson:") + buffer.GetString();
-  } else {
-    // e.g., authority:code, WKT2, arbitrary string. A pluggable CrsProvider
-    // could handle these and return something we're allowed to write here.
-    return Status::Invalid("Unsupported GeoArrow CRS for Parquet");
-  }
-}
-
 Result<std::shared_ptr<const LogicalType>> GeospatialLogicalTypeFromArrow(
     const std::string& serialized_data, const ArrowWriterProperties& arrow_properties) {
-  // Parquet has no way to interpret a null or missing CRS; however, it is more likely
-  // to induce confusion insert the fully specified equivalent of a null CRS (custom
-  // engineering CRS with unspecified units)
+  // Without a JSON parser, we can still handle a few trivial cases
   if (serialized_data.empty() || serialized_data == "{}") {
     return LogicalType::Geometry();
+  } else if (
+      serialized_data ==
+      R"({"edges": "spherical", "crs": "OGC:CRS84", "crs_type": "authority_code"})") {
+    return LogicalType::Geography();
+  } else if (serialized_data == R"({"crs": "OGC:CRS84", "crs_type": "authority_code"})") {
+    return LogicalType::Geometry();
+  } else {
+    // Will return an error status if Parquet was not built with ARROW_JSON
+    return GeospatialLogicalTypeFromGeoArrowJSON(serialized_data, arrow_properties);
   }
-
-  namespace rj = ::arrow::rapidjson;
-  rj::Document document;
-  if (document.Parse(serialized_data.data(), serialized_data.length()).HasParseError()) {
-    return Status::Invalid("Invalid serialized JSON data: ", serialized_data);
-  }
-
-  ARROW_ASSIGN_OR_RAISE(std::string crs,
-                        GeospatialGeoArrowCrsToParquetCrs(document, arrow_properties));
-
-  if (document.HasMember("edges") && document["edges"] == "planar") {
-    return LogicalType::Geometry(crs);
-  } else if (document.HasMember("edges") && document["edges"] == "spherical") {
-    return LogicalType::Geography(crs,
-                                  LogicalType::EdgeInterpolationAlgorithm::SPHERICAL);
-  } else if (document.HasMember("edges")) {
-    return Status::NotImplemented("GeoArrow edge type: ", serialized_data);
-  }
-
-  return LogicalType::Geometry(crs);
 }
 
 static constexpr char FIELD_ID_KEY[] = "PARQUET:field_id";
@@ -526,8 +453,7 @@ Status FieldToNode(const std::string& name, const std::shared_ptr<Field>& field,
         type = ParquetType::BYTE_ARRAY;
         logical_type = LogicalType::JSON();
         break;
-      } else if (arrow_properties.write_geospatial_logical_types() &&
-                 ext_type->extension_name() == std::string("geoarrow.wkb")) {
+      } else if (ext_type->extension_name() == std::string("geoarrow.wkb")) {
         type = ParquetType::BYTE_ARRAY;
         ARROW_ASSIGN_OR_RAISE(logical_type, GeospatialLogicalTypeFromArrow(
                                                 ext_type->Serialize(), arrow_properties));
@@ -562,6 +488,7 @@ struct SchemaTreeContext {
   SchemaManifest* manifest;
   ArrowReaderProperties properties;
   const SchemaDescriptor* schema;
+  std::shared_ptr<const KeyValueMetadata> metadata;
 
   void LinkParent(const SchemaField* child, const SchemaField* parent) {
     manifest->child_to_parent[child] = parent;
@@ -584,7 +511,7 @@ ::arrow::Result<std::shared_ptr<ArrowType>> GetTypeForNode(
     int column_index, const schema::PrimitiveNode& primitive_node,
     SchemaTreeContext* ctx) {
   ARROW_ASSIGN_OR_RAISE(std::shared_ptr<ArrowType> storage_type,
-                        GetArrowType(primitive_node, ctx->properties));
+                        GetArrowType(primitive_node, ctx->properties, ctx->metadata));
   if (ctx->properties.read_dictionary(column_index) &&
       IsDictionaryReadSupported(*storage_type)) {
     return ::arrow::dictionary(::arrow::int32(), storage_type);
@@ -1187,6 +1114,10 @@ Status ToParquetSchema(const ::arrow::Schema* arrow_schema,
                        const WriterProperties& properties,
                        const ArrowWriterProperties& arrow_properties,
                        std::shared_ptr<SchemaDescriptor>* out) {
+  // TODO(paleolimbot): I'm wondering if this geo_crs_context is reused when testing
+  // on MINGW, where we get some failures indicating non-empty metadata where it was
+  // expected
+  arrow_properties.geo_crs_context()->Clear();
   std::vector<NodePtr> nodes(arrow_schema->num_fields());
   for (int i = 0; i < arrow_schema->num_fields(); i++) {
     RETURN_NOT_OK(
@@ -1249,6 +1180,7 @@ Status SchemaManifest::Make(const SchemaDescriptor* schema,
   ctx.manifest = manifest;
   ctx.properties = properties;
   ctx.schema = schema;
+  ctx.metadata = metadata;
   const GroupNode& schema_node = *schema->group_node();
   manifest->descr = schema;
   manifest->schema_fields.resize(schema_node.field_count());