Skip to content

Commit 7ee1a33

Browse files
authored
Merge pull request #88 from paulmach/ewkb
Add ewkb encoding/decoding support
2 parents f35347a + ce48d35 commit 7ee1a33

31 files changed

+4710
-671
lines changed

README.md

+10-9
Original file line numberDiff line numberDiff line change
@@ -144,31 +144,32 @@ data, err := layers.Marshal() // this data is NOT gzipped.
144144
data, err := layers.MarshalGzipped()
145145
```
146146

147-
## Decoding WKB from a database query
147+
## Decoding WKB/EWKB from a database query
148148

149-
Geometries are usually returned from databases in WKB format. The [encoding/wkb](encoding/wkb)
149+
Geometries are usually returned from databases in WKB or EWKB format. The [encoding/ewkb](encoding/ewkb)
150150
sub-package offers helpers to "scan" the data into the base types directly.
151151
For example:
152152

153153
```go
154+
db.Exec(
155+
"INSERT INTO postgis_table (point_column) VALUES (ST_GeomFromEWKB(?))",
156+
ewkb.Value(orb.Point{1, 2}, 4326),
157+
)
158+
154159
row := db.QueryRow("SELECT ST_AsBinary(point_column) FROM postgis_table")
155160
156161
var p orb.Point
157-
err := row.Scan(wkb.Scanner(&p))
158-
159-
db.Exec("INSERT INTO table (point_column) VALUES (ST_GeomFromWKB(?))", wkb.Value(p))
162+
err := row.Scan(ewkb.Scanner(&p))
160163
```
161164

162-
Scanning directly from MySQL columns is supported. By default MySQL returns geometry
163-
data as WKB but prefixed with a 4 byte SRID. To support this, if the data is not
164-
valid WKB, the code will strip the first 4 bytes, the SRID, and try again.
165-
This works for most use cases.
165+
For more information see the readme in the [encoding/ewkb](encoding/ewkb) package.
166166

167167
## List of sub-package utilities
168168

169169
- [`clip`](clip) - clipping geometry to a bounding box
170170
- [`encoding/mvt`](encoding/mvt) - encoded and decoding from [Mapbox Vector Tiles](https://www.mapbox.com/vector-tiles/)
171171
- [`encoding/wkb`](encoding/wkb) - well-known binary as well as helpers to decode from the database queries
172+
- [`encoding/ewkb`](encoding/ewkb) - extended well-known binary format that includes the SRID
172173
- [`encoding/wkt`](encoding/wkt) - well-known text encoding
173174
- [`geojson`](geojson) - working with geojson and the types in this package
174175
- [`maptile`](maptile) - working with mercator map tiles and quadkeys

encoding/ewkb/README.md

+107
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# encoding/ewkb [![Godoc Reference](https://pkg.go.dev/badge/github.com/paulmach/orb)](https://pkg.go.dev/github.com/paulmach/orb/encoding/ewkb)
2+
3+
This package provides encoding and decoding of [extended WKB](https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Format_variations)
4+
data. This format includes the [SRID](https://en.wikipedia.org/wiki/Spatial_reference_system) in the data.
5+
If the SRID is not needed use the [wkb](../wkb) package for a simpler interface.
6+
The interface is defined as:
7+
8+
```go
9+
func Marshal(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) ([]byte, error)
10+
func MarshalToHex(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) (string, error)
11+
func MustMarshal(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) []byte
12+
func MustMarshalToHex(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) string
13+
14+
func NewEncoder(w io.Writer) *Encoder
15+
func (e *Encoder) SetByteOrder(bo binary.ByteOrder) *Encoder
16+
func (e *Encoder) SetSRID(srid int) *Encoder
17+
func (e *Encoder) Encode(geom orb.Geometry) error
18+
19+
func Unmarshal(b []byte) (orb.Geometry, int, error)
20+
21+
func NewDecoder(r io.Reader) *Decoder
22+
func (d *Decoder) Decode() (orb.Geometry, int, error)
23+
```
24+
25+
## Inserting geometry into a database
26+
27+
Depending on the database different formats and functions are supported.
28+
29+
### PostgreSQL and PostGIS
30+
31+
PostGIS stores geometry as EWKB internally. As a result it can be inserted without
32+
a wrapper function.
33+
34+
```go
35+
db.Exec("INSERT INTO geodata(geom) VALUES (ST_GeomFromEWKB($1))", ewkb.Value(coord, 4326))
36+
37+
db.Exec("INSERT INTO geodata(geom) VALUES ($1)", ewkb.Value(coord, 4326))
38+
```
39+
40+
### MySQL/MariaDB
41+
42+
MySQL and MariaDB
43+
[store geometry](https://dev.mysql.com/doc/refman/5.7/en/gis-data-formats.html)
44+
data in WKB format with a 4 byte SRID prefix.
45+
46+
```go
47+
coord := orb.Point{1, 2}
48+
49+
// as WKB in hex format
50+
data := wkb.MustMarshalToHex(coord)
51+
db.Exec("INSERT INTO geodata(geom) VALUES (ST_GeomFromWKB(UNHEX(?), 4326))", data)
52+
53+
// relying on the raw encoding
54+
db.Exec("INSERT INTO geodata(geom) VALUES (?)", ewkb.ValuePrefixSRID(coord, 4326))
55+
```
56+
57+
## Reading geometry from a database query
58+
59+
As stated above, different databases supported different formats and functions.
60+
61+
### PostgreSQL and PostGIS
62+
63+
When working with PostGIS the raw format is EWKB so the wrapper function is not necessary
64+
65+
```go
66+
// both of these queries return the same data
67+
row := db.QueryRow("SELECT ST_AsEWKB(geom) FROM geodata")
68+
row := db.QueryRow("SELECT geom FROM geodata")
69+
70+
// if you don't need the SRID
71+
p := orb.Point{}
72+
err := row.Scan(ewkb.Scanner(&p))
73+
log.Printf("geom: %v", p)
74+
75+
// if you need the SRID
76+
p := orb.Point{}
77+
gs := ewkb.Scanner(&p)
78+
err := row.Scan(gs)
79+
80+
log.Printf("srid: %v", gs.SRID)
81+
log.Printf("geom: %v", gs.Geometry)
82+
log.Printf("also geom: %v", p)
83+
```
84+
85+
### MySQL/MariaDB
86+
87+
```go
88+
// using the ST_AsBinary function
89+
row := db.QueryRow("SELECT st_srid(geom), ST_AsBinary(geom) FROM geodata")
90+
row.Scan(&srid, ewkb.Scanner(&data))
91+
92+
// relying on the raw encoding
93+
row := db.QueryRow("SELECT geom FROM geodata")
94+
95+
// if you don't need the SRID
96+
p := orb.Point{}
97+
err := row.Scan(ewkb.ScannerPrefixSRID(&p))
98+
log.Printf("geom: %v", p)
99+
100+
// if you need the SRID
101+
p := orb.Point{}
102+
gs := ewkb.ScannerPrefixSRID(&p)
103+
err := row.Scan(gs)
104+
105+
log.Printf("srid: %v", gs.SRID)
106+
log.Printf("geom: %v", gs.Geometry)
107+
```

encoding/ewkb/collection_test.go

+64
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
package ewkb
2+
3+
import (
4+
"testing"
5+
6+
"github.com/paulmach/orb"
7+
"github.com/paulmach/orb/encoding/internal/wkbcommon"
8+
)
9+
10+
func TestCollection(t *testing.T) {
11+
large := orb.Collection{}
12+
for i := 0; i < wkbcommon.MaxMultiAlloc+100; i++ {
13+
large = append(large, orb.Point{float64(i), float64(-i)})
14+
}
15+
16+
cases := []struct {
17+
name string
18+
srid int
19+
data []byte
20+
expected orb.Collection
21+
}{
22+
{
23+
name: "large",
24+
srid: 123,
25+
data: MustMarshal(large, 123),
26+
expected: large,
27+
},
28+
{
29+
name: "collection with point",
30+
data: MustDecodeHex("0107000020e6100000010000000101000000000000000000f03f0000000000000040"),
31+
srid: 4326,
32+
expected: orb.Collection{orb.Point{1, 2}},
33+
},
34+
{
35+
name: "collection with point and line",
36+
data: MustDecodeHex("0020000007000010e60000000200000000013ff000000000000040000000000000000000000002000000023ff0000000000000400000000000000040080000000000004010000000000000"),
37+
srid: 4326,
38+
expected: orb.Collection{
39+
orb.Point{1, 2},
40+
orb.LineString{{1, 2}, {3, 4}},
41+
},
42+
},
43+
{
44+
name: "collection with point and line and polygon",
45+
data: MustDecodeHex("0107000020e6100000030000000101000000000000000000f03f0000000000000040010200000002000000000000000000f03f00000000000000400000000000000840000000000000104001030000000300000004000000000000000000f03f00000000000000400000000000000840000000000000104000000000000014400000000000001840000000000000f03f000000000000004004000000000000000000264000000000000028400000000000002a400000000000002c400000000000002e4000000000000030400000000000002640000000000000284004000000000000000000354000000000000036400000000000003740000000000000384000000000000039400000000000003a4000000000000035400000000000003640"),
46+
srid: 4326,
47+
expected: orb.Collection{
48+
orb.Point{1, 2},
49+
orb.LineString{{1, 2}, {3, 4}},
50+
orb.Polygon{
51+
{{1, 2}, {3, 4}, {5, 6}, {1, 2}},
52+
{{11, 12}, {13, 14}, {15, 16}, {11, 12}},
53+
{{21, 22}, {23, 24}, {25, 26}, {21, 22}},
54+
},
55+
},
56+
},
57+
}
58+
59+
for _, tc := range cases {
60+
t.Run(tc.name, func(t *testing.T) {
61+
compare(t, tc.expected, tc.srid, tc.data)
62+
})
63+
}
64+
}

encoding/ewkb/ewkb.go

+179
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
package ewkb
2+
3+
import (
4+
"bytes"
5+
"encoding/binary"
6+
"encoding/hex"
7+
"errors"
8+
"io"
9+
10+
"github.com/paulmach/orb"
11+
"github.com/paulmach/orb/encoding/internal/wkbcommon"
12+
)
13+
14+
var (
15+
// ErrUnsupportedDataType is returned by Scan methods when asked to scan
16+
// non []byte data from the database. This should never happen
17+
// if the driver is acting appropriately.
18+
ErrUnsupportedDataType = errors.New("wkb: scan value must be []byte")
19+
20+
// ErrNotEWKB is returned when unmarshalling EWKB and the data is not valid.
21+
ErrNotEWKB = errors.New("wkb: invalid data")
22+
23+
// ErrIncorrectGeometry is returned when unmarshalling EWKB data into the wrong type.
24+
// For example, unmarshaling linestring data into a point.
25+
ErrIncorrectGeometry = errors.New("wkb: incorrect geometry")
26+
27+
// ErrUnsupportedGeometry is returned when geometry type is not supported by this lib.
28+
ErrUnsupportedGeometry = errors.New("wkb: unsupported geometry")
29+
)
30+
31+
var commonErrorMap = map[error]error{
32+
wkbcommon.ErrUnsupportedDataType: ErrUnsupportedDataType,
33+
wkbcommon.ErrNotWKB: ErrNotEWKB,
34+
wkbcommon.ErrNotWKBHeader: ErrNotEWKB,
35+
wkbcommon.ErrIncorrectGeometry: ErrIncorrectGeometry,
36+
wkbcommon.ErrUnsupportedGeometry: ErrUnsupportedGeometry,
37+
}
38+
39+
func mapCommonError(err error) error {
40+
e, ok := commonErrorMap[err]
41+
if ok {
42+
return e
43+
}
44+
45+
return err
46+
}
47+
48+
// DefaultByteOrder is the order used for marshalling or encoding is none is specified.
49+
var DefaultByteOrder binary.ByteOrder = binary.LittleEndian
50+
51+
// DefaultSRID is set to 4326, a common SRID, which represents spatial data using
52+
// longitude and latitude coordinates on the Earth's surface as defined in the WGS84 standard,
53+
// which is also used for the Global Positioning System (GPS).
54+
// This will be used by the encoder if non is specified.
55+
var DefaultSRID int = 4326
56+
57+
// An Encoder will encode a geometry as EWKB to the writer given at creation time.
58+
type Encoder struct {
59+
srid int
60+
e *wkbcommon.Encoder
61+
}
62+
63+
// MustMarshal will encode the geometry and panic on error.
64+
// Currently there is no reason to error during geometry marshalling.
65+
func MustMarshal(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) []byte {
66+
d, err := Marshal(geom, srid, byteOrder...)
67+
if err != nil {
68+
panic(err)
69+
}
70+
71+
return d
72+
}
73+
74+
// Marshal encodes the geometry with the given byte order.
75+
// An SRID of 0 will not be included in the encoding and the result will be a wkb encoding of the geometry.
76+
func Marshal(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) ([]byte, error) {
77+
buf := bytes.NewBuffer(make([]byte, 0, wkbcommon.GeomLength(geom, srid != 0)))
78+
79+
e := NewEncoder(buf)
80+
e.SetSRID(srid)
81+
82+
if len(byteOrder) > 0 {
83+
e.SetByteOrder(byteOrder[0])
84+
}
85+
86+
err := e.Encode(geom)
87+
if err != nil {
88+
return nil, err
89+
}
90+
91+
if buf.Len() == 0 {
92+
return nil, nil
93+
}
94+
95+
return buf.Bytes(), nil
96+
}
97+
98+
// MarshalToHex will encode the geometry into a hex string representation of the binary ewkb.
99+
func MarshalToHex(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) (string, error) {
100+
data, err := Marshal(geom, srid, byteOrder...)
101+
if err != nil {
102+
return "", err
103+
}
104+
105+
return hex.EncodeToString(data), nil
106+
}
107+
108+
// MustMarshalToHex will encode the geometry and panic on error.
109+
// Currently there is no reason to error during geometry marshalling.
110+
func MustMarshalToHex(geom orb.Geometry, srid int, byteOrder ...binary.ByteOrder) string {
111+
d, err := MarshalToHex(geom, srid, byteOrder...)
112+
if err != nil {
113+
panic(err)
114+
}
115+
116+
return d
117+
}
118+
119+
// NewEncoder creates a new Encoder for the given writer.
120+
func NewEncoder(w io.Writer) *Encoder {
121+
e := wkbcommon.NewEncoder(w)
122+
e.SetByteOrder(DefaultByteOrder)
123+
return &Encoder{e: e, srid: DefaultSRID}
124+
}
125+
126+
// SetByteOrder will override the default byte order set when
127+
// the encoder was created.
128+
func (e *Encoder) SetByteOrder(bo binary.ByteOrder) *Encoder {
129+
e.e.SetByteOrder(bo)
130+
return e
131+
}
132+
133+
// SetSRID will override the default srid.
134+
func (e *Encoder) SetSRID(srid int) *Encoder {
135+
e.srid = srid
136+
return e
137+
}
138+
139+
// Encode will write the geometry encoded as EWKB to the given writer.
140+
func (e *Encoder) Encode(geom orb.Geometry, srid ...int) error {
141+
s := e.srid
142+
if len(srid) > 0 {
143+
s = srid[0]
144+
}
145+
146+
return e.e.Encode(geom, s)
147+
}
148+
149+
// Decoder can decoder WKB geometry off of the stream.
150+
type Decoder struct {
151+
d *wkbcommon.Decoder
152+
}
153+
154+
// Unmarshal will decode the type into a Geometry.
155+
func Unmarshal(data []byte) (orb.Geometry, int, error) {
156+
g, srid, err := wkbcommon.Unmarshal(data)
157+
if err != nil {
158+
return nil, 0, mapCommonError(err)
159+
}
160+
161+
return g, srid, nil
162+
}
163+
164+
// NewDecoder will create a new EWKB decoder.
165+
func NewDecoder(r io.Reader) *Decoder {
166+
return &Decoder{
167+
d: wkbcommon.NewDecoder(r),
168+
}
169+
}
170+
171+
// Decode will decode the next geometry off of the stream.
172+
func (d *Decoder) Decode() (orb.Geometry, int, error) {
173+
g, srid, err := d.d.Decode()
174+
if err != nil {
175+
return nil, 0, mapCommonError(err)
176+
}
177+
178+
return g, srid, nil
179+
}

0 commit comments

Comments
 (0)