CC - Add flatbuffers

FlatBuffersSchema/Stations/Coordinates.py

@@ -0,0 +1,23 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: Stations
+
+import flatbuffers
+
+class Coordinates(object):
+    __slots__ = ['_tab']
+
+    # Coordinates
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Coordinates
+    def Lat(self): return self._tab.Get(flatbuffers.number_types.Float32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(0))
+    # Coordinates
+    def Lon(self): return self._tab.Get(flatbuffers.number_types.Float32Flags, self._tab.Pos + flatbuffers.number_types.UOffsetTFlags.py_type(4))
+
+def CreateCoordinates(builder, lat, lon):
+    builder.Prep(4, 8)
+    builder.PrependFloat32(lon)
+    builder.PrependFloat32(lat)
+    return builder.Offset()

FlatBuffersSchema/Stations/Features.py

@@ -0,0 +1,54 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: Stations
+
+import flatbuffers
+
+class Features(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFeatures(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Features()
+        x.Init(buf, n + offset)
+        return x
+
+    # Features
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Features
+    def Geometry(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            x = self._tab.Indirect(o + self._tab.Pos)
+            from .Geometry import Geometry
+            obj = Geometry()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+    # Features
+    def Type(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # Features
+    def Properties(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            x = self._tab.Indirect(o + self._tab.Pos)
+            from .Properties import Properties
+            obj = Properties()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+def FeaturesStart(builder): builder.StartObject(3)
+def FeaturesAddGeometry(builder, geometry): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(geometry), 0)
+def FeaturesAddType(builder, type): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(type), 0)
+def FeaturesAddProperties(builder, properties): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(properties), 0)
+def FeaturesEnd(builder): return builder.EndObject()

FlatBuffersSchema/Stations/FlatBuffersDocument.py

@@ -0,0 +1,52 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: Stations
+
+import flatbuffers
+
+class FlatBuffersDocument(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsFlatBuffersDocument(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = FlatBuffersDocument()
+        x.Init(buf, n + offset)
+        return x
+
+    # FlatBuffersDocument
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # FlatBuffersDocument
+    def Type(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # FlatBuffersDocument
+    def Features(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            x = self._tab.Vector(o)
+            x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
+            x = self._tab.Indirect(x)
+            from .Features import Features
+            obj = Features()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+    # FlatBuffersDocument
+    def FeaturesLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+def FlatBuffersDocumentStart(builder): builder.StartObject(2)
+def FlatBuffersDocumentAddType(builder, type): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(type), 0)
+def FlatBuffersDocumentAddFeatures(builder, features): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(features), 0)
+def FlatBuffersDocumentStartFeaturesVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def FlatBuffersDocumentEnd(builder): return builder.EndObject()

FlatBuffersSchema/Stations/Geometry.py

@@ -0,0 +1,42 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: Stations
+
+import flatbuffers
+
+class Geometry(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsGeometry(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Geometry()
+        x.Init(buf, n + offset)
+        return x
+
+    # Geometry
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Geometry
+    def Type(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # Geometry
+    def Coordinates(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            x = o + self._tab.Pos
+            from .Coordinates import Coordinates
+            obj = Coordinates()
+            obj.Init(self._tab.Bytes, x)
+            return obj
+        return None
+
+def GeometryStart(builder): builder.StartObject(2)
+def GeometryAddType(builder, type): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(type), 0)
+def GeometryAddCoordinates(builder, coordinates): builder.PrependStructSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(coordinates), 0)
+def GeometryEnd(builder): return builder.EndObject()

FlatBuffersSchema/Stations/Properties.py

@@ -0,0 +1,79 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace: Stations
+
+import flatbuffers
+
+class Properties(object):
+    __slots__ = ['_tab']
+
+    @classmethod
+    def GetRootAsProperties(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Properties()
+        x.Init(buf, n + offset)
+        return x
+
+    # Properties
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Properties
+    def Description(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # Properties
+    def MarkerSymbol(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # Properties
+    def Title(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # Properties
+    def Url(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # Properties
+    def Lines(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.String(a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
+        return ""
+
+    # Properties
+    def LinesLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Properties
+    def Address(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+def PropertiesStart(builder): builder.StartObject(6)
+def PropertiesAddDescription(builder, description): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(description), 0)
+def PropertiesAddMarkerSymbol(builder, markerSymbol): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(markerSymbol), 0)
+def PropertiesAddTitle(builder, title): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(title), 0)
+def PropertiesAddUrl(builder, url): builder.PrependUOffsetTRelativeSlot(3, flatbuffers.number_types.UOffsetTFlags.py_type(url), 0)
+def PropertiesAddLines(builder, lines): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(lines), 0)
+def PropertiesStartLinesVector(builder, numElems): return builder.StartVector(4, numElems, 4)
+def PropertiesAddAddress(builder, address): builder.PrependUOffsetTRelativeSlot(5, flatbuffers.number_types.UOffsetTFlags.py_type(address), 0)
+def PropertiesEnd(builder): return builder.EndObject()

FlatBuffersSchema/stations.fbs

@@ -0,0 +1,33 @@
+namespace FlatBuffersSchema.Stations;
+
+struct Coordinates {
+    lat:float;
+    lon:float;
+}
+
+table Geometry {
+    type:string;
+    coordinates:Coordinates;
+}
+
+table Properties {
+    description:string;
+    marker_symbol:string;
+    title:string;
+    url:string;
+    lines:[string];
+    address:string;
+}
+
+table Features {
+    geometry:Geometry;
+    type:string;
+    properties:Properties;
+}
+
+table FlatBuffersDocument {
+    type:string;
+    features:[Features];
+}
+
+root_type FlatBuffersDocument;

README.md

@@ -3,4 +3,14 @@
 # python-serialization-benchmark
 Performance comparison of different serialization formats with Python
 
-[Launch interactive notebook](https://mybinder.org/v2/gh/charleswhchan/python-serialization-benchmark/master)
+[Launch interactive notebook](https://mybinder.org/v2/gh/charleswhchan/python-serialization-benchmark/master)
+
+## FlatBuffers
+- To re-generate schema for FlatBuffers: 
+```
+flatc --python FlatBuffersSchema/stations.fbs
+```
+- Then convert JSON to FlatBuffers binary format:
+```
+flatc --binary FlatBuffersSchema/stations.fbs stations.geojson && mv stations.bin FlatBuffersSchema/
+```

index.ipynb

@@ -107,6 +107,65 @@
     "%timeit bson.loads(stations_bson)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## flatbuffers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import flatbuffers\n",
+    "from FlatBuffersSchema.Stations.FlatBuffersDocument import FlatBuffersDocument as fbDoc"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "buf = open('./FlatBuffersSchema/stations.bin', 'rb').read()\n",
+    "buf = bytearray(buf)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stations_fb = fbDoc.GetRootAsFlatBuffersDocument(buf, 0)\n",
+    "print(stations_fb.Type())\n",
+    "print(stations_fb.FeaturesLength())\n",
+    "print(stations_fb.Features(0).Geometry().Coordinates().Lat())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# obj to flatbuffers\n",
+    "# TODO"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# flatbuffers to object\n",
+    "%timeit fbDoc.GetRootAsFlatBuffersDocument(buf, 0)"
+   ]
+  },
   {
    "cell_type": "markdown",
    "metadata": {},

requirements.txt

@@ -1,3 +1,4 @@
 bson
+flatbuffers
 msgpack
 pyyaml