Version 0.0.1, setup.py compatibility, another xy fix.

Author: Tom MacWright

CHANGES.txt

@@ -0,0 +1 @@
+v0.0.1, March 14, 2011 -- Initial release

LICENSE.md

@@ -0,0 +1,25 @@
+Copyright (c), Development Seed  
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+- Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+- Neither the name "Development Seed" nor the names of its contributors may be
+  used to endorse or promote products derived from this software without
+  specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1 @@
+include *.md

Makefile

@@ -0,0 +1,16 @@
+test/tmp:
+	mkdir test/tmp
+
+test/data:
+	mkdir test/data
+	wget http://mapbox-tilesets.s3.amazonaws.com/mbtiles/World_Glass_1.1.mbtiles.zip -o test/data/World_Glass_1.1.mbtiles.zip
+	unzip test/data/World_Glass_1.1.mbtiles.zip
+	mv World_Glass_1.1.mbtiles test/data
+
+test/tmp/world_glass:
+	time ./mbutil.py test/data/World_Glass_1.1.mbtiles test/tmp/world_glass
+
+test/tmp/world_glass_out.mbtiles:
+	time ./mbutil.py test/tmp/world_glass test/tmp/world_glass_out.mbtiles
+
+test: test/data test/tmp test/tmp/world_glass test/tmp/world_glass_out.mbtiles

mbutil.py

@@ -34,11 +34,10 @@ def mbtiles_connect(mbtiles_file):
         print e
         sys.exit(1)
 
-def optimize_connection(con):
-    con.execute("""PRAGMA synchronous=0""")
-    con.execute("""PRAGMA locking_mode=EXCLUSIVE""")
-    con.execute("""PRAGMA journal_mode=TRUNCATE""")
-    con.commit()
+def optimize_connection(cur):
+    cur.execute("""PRAGMA synchronous=0""")
+    cur.execute("""PRAGMA locking_mode=EXCLUSIVE""")
+    cur.execute("""PRAGMA journal_mode=TRUNCATE""")
 
 def compression_prepare(cur, con):
     cur.execute("""
@@ -53,14 +52,12 @@ def compression_prepare(cur, con):
         tile_row integer, 
         tile_id VARCHAR(256));
     """)
-    con.commit()
 
-def optimize_database(con):
+def optimize_database(cur):
     print 'analyzing db'
-    con.execute("""ANALYZE;""")
+    cur.execute("""ANALYZE;""")
     print 'cleaning db'
-    con.execute("""VACUUM;""")
-    con.commit()
+    cur.execute("""VACUUM;""")
 
 def compression_do(cur, con, chunk):
     overlapping = 0
@@ -110,16 +107,14 @@ def compression_do(cur, con, chunk):
                 print "insert into map: %s" % (time.time() - start)
         con.commit()
 
-def compression_finalize(cur, con):
+def compression_finalize(cur):
     cur.execute("""drop table tiles;""")
-    con.commit()
     cur.execute("""create view tiles as
         select map.zoom_level as zoom_level,
         map.tile_column as tile_column,
         map.tile_row as tile_row,
         images.tile_data as tile_data FROM
         map JOIN images on images.tile_id = map.tile_id;""")
-    con.commit()
     cur.execute("""
           CREATE UNIQUE INDEX map_index on map 
             (zoom_level, tile_column, tile_row);""")
@@ -128,47 +123,44 @@ def compression_finalize(cur, con):
             (tile_id);""")
     cur.execute("""vacuum;""")
     cur.execute("""analyze;""")
-    con.commit()
 
 def disk_to_mbtiles(directory_path, mbtiles_file):
     print "Importing disk to MBTiles"
     print "%s --> %s" % (directory_path, mbtiles_file)
     con = mbtiles_connect(mbtiles_file)
-    optimize_connection(con)
-    mbtiles_setup(con.cursor())
-    con.commit()
+    cur = con.cursor()
+    optimize_connection(cur)
+    mbtiles_setup(cur)
     try:
         metadata = json.load(open('%s/metadata.json' % directory_path, 'r'))
         for name, value in metadata.items():
-            con.execute('insert into metadata (name, value) values (?, ?)',
+            cur.execute('insert into metadata (name, value) values (?, ?)',
                     (name, value))
-        con.commit()
         print 'metadata from metadata.json restored'
     except Exception, e:
         print e
         print 'metadata.json not found'
 
     count = 0
+    start_time = time.time()
     msg = ""
     for r1, zs, ignore in os.walk(directory_path):
         for z in zs:
-            for r2, ys, ignore in os.walk(os.path.join(r1, z)):
-                for y in ys:
-                    for r2, ignore, xs in os.walk(os.path.join(r1, z, y)):
-                        for x in xs:
-                            f = open(os.path.join(r1, z, y, x), 'r')
-                            con.execute("""insert into tiles (zoom_level,
+            for r2, xs, ignore in os.walk(os.path.join(r1, z)):
+                for x in xs:
+                    for r2, ignore, ys in os.walk(os.path.join(r1, z, x)):
+                        for y in ys:
+                            f = open(os.path.join(r1, z, x, y), 'rb')
+                            cur.execute("""insert into tiles (zoom_level,
                                 tile_row, tile_column, tile_data) values
                                 (?, ?, ?, ?);""",
                                 (z, x, y, sqlite3.Binary(f.read())))
                             f.close()
                             count = count + 1
-                            for c in msg: sys.stdout.write(chr(8))
-                            msg = "%s tiles inserted" % count
-                            sys.stdout.write(msg)
-                            if (count % 5000):
-                                con.commit()
-    con.commit()
+                            if (count % 100) == 0:
+                                for c in msg: sys.stdout.write(chr(8))
+                                msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time))
+                                sys.stdout.write(msg)
     print 'tiles inserted.'
     optimize_database(con)
 

setup.py

@@ -0,0 +1,14 @@
+from distutils.core import setup
+
+setup(
+    name='mbutil',
+    version='0.0.1',
+    author='Tom MacWright',
+    author_email='[email protected]',
+    packages=[],
+    scripts=['mbutil.py'],
+    url='https://github.com/mapbox/mbutil',
+    license='LICENSE.md',
+    description='An importer and exporter for MBTiles',
+    long_description=open('README.md').read(),
+)