Skip to content

Commit be0f75f

Browse files
leplatremTom MacWright
leplatrem
authored and
Tom MacWright
committed
package mbutil as a module
1 parent b46ae06 commit be0f75f

File tree

3 files changed

+194
-182
lines changed

3 files changed

+194
-182
lines changed

‎mbutil.py

+3-181
Original file line numberDiff line numberDiff line change
@@ -1,193 +1,15 @@
11
#!/usr/bin/env python
2-
32
# MBUtil: a tool for MBTiles files
43
# Supports importing, exporting, and more
54
#
65
# (c) Development Seed 2011
76
# Licensed under BSD
8-
9-
import sqlite3, uuid, sys, logging, time, os, json
7+
import logging
8+
import os, sys
109
from optparse import OptionParser
1110

12-
logger = logging.getLogger(__name__)
13-
14-
15-
def mbtiles_setup(cur):
16-
cur.execute("""
17-
create table tiles (
18-
zoom_level integer,
19-
tile_column integer,
20-
tile_row integer,
21-
tile_data blob);
22-
""")
23-
cur.execute("""create table metadata
24-
(name text, value text);""")
25-
cur.execute("""create unique index name on metadata (name);""")
26-
cur.execute("""create unique index tile_index on tiles
27-
(zoom_level, tile_column, tile_row);""")
28-
29-
def mbtiles_connect(mbtiles_file):
30-
try:
31-
con = sqlite3.connect(mbtiles_file)
32-
return con
33-
except Exception, e:
34-
logger.error("Could not connect to database")
35-
logger.exception(e)
36-
sys.exit(1)
37-
38-
def optimize_connection(cur):
39-
cur.execute("""PRAGMA synchronous=0""")
40-
cur.execute("""PRAGMA locking_mode=EXCLUSIVE""")
41-
cur.execute("""PRAGMA journal_mode=TRUNCATE""")
42-
43-
def compression_prepare(cur, con):
44-
cur.execute("""
45-
CREATE TABLE if not exists images (
46-
tile_data blob,
47-
tile_id VARCHAR(256));
48-
""")
49-
cur.execute("""
50-
CREATE TABLE if not exists map (
51-
zoom_level integer,
52-
tile_column integer,
53-
tile_row integer,
54-
tile_id VARCHAR(256));
55-
""")
56-
57-
def optimize_database(cur):
58-
logger.debug('analyzing db')
59-
cur.execute("""ANALYZE;""")
60-
logger.debug('cleaning db')
61-
cur.execute("""VACUUM;""")
62-
63-
def compression_do(cur, con, chunk):
64-
overlapping = 0
65-
unique = 0
66-
total = 0
67-
cur.execute("select count(zoom_level) from tiles")
68-
res = cur.fetchone()
69-
total_tiles = res[0]
70-
logging.debug("%d total tiles to fetch" % total_tiles)
71-
for i in range(total_tiles / chunk):
72-
logging.debug("%d / %d rounds done" % (i, (total_tiles / chunk)))
73-
ids = []
74-
files = []
75-
start = time.time()
76-
cur.execute("""select zoom_level, tile_column, tile_row, tile_data
77-
from tiles where rowid > ? and rowid <= ?""", ((i * chunk), ((i + 1) * chunk)))
78-
logger.debug("select: %s" % (time.time() - start))
79-
rows = cur.fetchall()
80-
for r in rows:
81-
total = total + 1
82-
if r[3] in files:
83-
overlapping = overlapping + 1
84-
start = time.time()
85-
query = """insert into map
86-
(zoom_level, tile_column, tile_row, tile_id)
87-
values (?, ?, ?, ?)"""
88-
logger.debug("insert: %s" % (time.time() - start))
89-
cur.execute(query, (r[0], r[1], r[2], ids[files.index(r[3])]))
90-
else:
91-
unique = unique + 1
92-
id = str(uuid.uuid4())
93-
94-
ids.append(id)
95-
files.append(r[3])
96-
97-
start = time.time()
98-
query = """insert into images
99-
(tile_id, tile_data)
100-
values (?, ?)"""
101-
cur.execute(query, (str(id), sqlite3.Binary(r[3])))
102-
logger.debug("insert into images: %s" % (time.time() - start))
103-
start = time.time()
104-
query = """insert into map
105-
(zoom_level, tile_column, tile_row, tile_id)
106-
values (?, ?, ?, ?)"""
107-
cur.execute(query, (r[0], r[1], r[2], id))
108-
logger.debug("insert into map: %s" % (time.time() - start))
109-
con.commit()
110-
111-
def compression_finalize(cur):
112-
cur.execute("""drop table tiles;""")
113-
cur.execute("""create view tiles as
114-
select map.zoom_level as zoom_level,
115-
map.tile_column as tile_column,
116-
map.tile_row as tile_row,
117-
images.tile_data as tile_data FROM
118-
map JOIN images on images.tile_id = map.tile_id;""")
119-
cur.execute("""
120-
CREATE UNIQUE INDEX map_index on map
121-
(zoom_level, tile_column, tile_row);""")
122-
cur.execute("""
123-
CREATE UNIQUE INDEX images_id on images
124-
(tile_id);""")
125-
cur.execute("""vacuum;""")
126-
cur.execute("""analyze;""")
127-
128-
def disk_to_mbtiles(directory_path, mbtiles_file):
129-
logger.info("Importing disk to MBTiles")
130-
logger.debug("%s --> %s" % (directory_path, mbtiles_file))
131-
con = mbtiles_connect(mbtiles_file)
132-
cur = con.cursor()
133-
optimize_connection(cur)
134-
mbtiles_setup(cur)
135-
try:
136-
metadata = json.load(open('%s/metadata.json' % directory_path, 'r'))
137-
for name, value in metadata.items():
138-
cur.execute('insert into metadata (name, value) values (?, ?)',
139-
(name, value))
140-
logger.info('metadata from metadata.json restored')
141-
except IOError, e:
142-
logger.warning('metadata.json not found')
143-
144-
count = 0
145-
start_time = time.time()
146-
msg = ""
147-
for r1, zs, ignore in os.walk(directory_path):
148-
for z in zs:
149-
for r2, xs, ignore in os.walk(os.path.join(r1, z)):
150-
for x in xs:
151-
for r2, ignore, ys in os.walk(os.path.join(r1, z, x)):
152-
for y in ys:
153-
f = open(os.path.join(r1, z, x, y), 'rb')
154-
cur.execute("""insert into tiles (zoom_level,
155-
tile_column, tile_row, tile_data) values
156-
(?, ?, ?, ?);""",
157-
(z, x, y.split('.')[0], sqlite3.Binary(f.read())))
158-
f.close()
159-
count = count + 1
160-
if (count % 100) == 0:
161-
for c in msg: sys.stdout.write(chr(8))
162-
msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time))
163-
sys.stdout.write(msg)
164-
logger.debug('tiles inserted.')
165-
optimize_database(con)
11+
from mbutil import mbtiles_to_disk, disk_to_mbtiles
16612

167-
def mbtiles_to_disk(mbtiles_file, directory_path):
168-
logger.debug("Exporting MBTiles to disk")
169-
logger.debug("%s --> %s" % (mbtiles_file, directory_path))
170-
con = mbtiles_connect(mbtiles_file)
171-
cur = con.cursor()
172-
os.mkdir("%s" % directory_path)
173-
metadata = dict(con.execute('select name, value from metadata;').fetchall())
174-
json.dump(metadata, open('%s/metadata.json' % directory_path, 'w'))
175-
count = con.execute('select count(zoom_level) from tiles;').fetchone()[0]
176-
done = 0
177-
msg =''
178-
tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
179-
t = tiles.fetchone()
180-
while t:
181-
if not os.path.isdir("%s/%s/%s/" % (directory_path, t[0], t[1])):
182-
os.makedirs("%s/%s/%s/" % (directory_path, t[0], t[1]))
183-
f = open('%s/%s/%s/%s.%s' %
184-
(directory_path, t[0], t[1], t[2], metadata.get('format', 'png')), 'wb')
185-
f.write(t[3])
186-
f.close()
187-
done = done + 1
188-
for c in msg: sys.stdout.write(chr(8))
189-
logger.info('%s / %s tiles exported' % (done, count))
190-
t = tiles.fetchone()
19113

19214
if __name__ == '__main__':
19315

‎mbutil/__init__.py

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
#!/usr/bin/env python
2+
3+
# MBUtil: a tool for MBTiles files
4+
# Supports importing, exporting, and more
5+
#
6+
# (c) Development Seed 2011
7+
# Licensed under BSD
8+
9+
import sqlite3, uuid, sys, logging, time, os, json
10+
11+
logger = logging.getLogger(__name__)
12+
13+
14+
def mbtiles_setup(cur):
15+
cur.execute("""
16+
create table tiles (
17+
zoom_level integer,
18+
tile_column integer,
19+
tile_row integer,
20+
tile_data blob);
21+
""")
22+
cur.execute("""create table metadata
23+
(name text, value text);""")
24+
cur.execute("""create unique index name on metadata (name);""")
25+
cur.execute("""create unique index tile_index on tiles
26+
(zoom_level, tile_column, tile_row);""")
27+
28+
def mbtiles_connect(mbtiles_file):
29+
try:
30+
con = sqlite3.connect(mbtiles_file)
31+
return con
32+
except Exception, e:
33+
logger.error("Could not connect to database")
34+
logger.exception(e)
35+
sys.exit(1)
36+
37+
def optimize_connection(cur):
38+
cur.execute("""PRAGMA synchronous=0""")
39+
cur.execute("""PRAGMA locking_mode=EXCLUSIVE""")
40+
cur.execute("""PRAGMA journal_mode=TRUNCATE""")
41+
42+
def compression_prepare(cur, con):
43+
cur.execute("""
44+
CREATE TABLE if not exists images (
45+
tile_data blob,
46+
tile_id VARCHAR(256));
47+
""")
48+
cur.execute("""
49+
CREATE TABLE if not exists map (
50+
zoom_level integer,
51+
tile_column integer,
52+
tile_row integer,
53+
tile_id VARCHAR(256));
54+
""")
55+
56+
def optimize_database(cur):
57+
logger.debug('analyzing db')
58+
cur.execute("""ANALYZE;""")
59+
logger.debug('cleaning db')
60+
cur.execute("""VACUUM;""")
61+
62+
def compression_do(cur, con, chunk):
63+
overlapping = 0
64+
unique = 0
65+
total = 0
66+
cur.execute("select count(zoom_level) from tiles")
67+
res = cur.fetchone()
68+
total_tiles = res[0]
69+
logging.debug("%d total tiles to fetch" % total_tiles)
70+
for i in range(total_tiles / chunk):
71+
logging.debug("%d / %d rounds done" % (i, (total_tiles / chunk)))
72+
ids = []
73+
files = []
74+
start = time.time()
75+
cur.execute("""select zoom_level, tile_column, tile_row, tile_data
76+
from tiles where rowid > ? and rowid <= ?""", ((i * chunk), ((i + 1) * chunk)))
77+
logger.debug("select: %s" % (time.time() - start))
78+
rows = cur.fetchall()
79+
for r in rows:
80+
total = total + 1
81+
if r[3] in files:
82+
overlapping = overlapping + 1
83+
start = time.time()
84+
query = """insert into map
85+
(zoom_level, tile_column, tile_row, tile_id)
86+
values (?, ?, ?, ?)"""
87+
logger.debug("insert: %s" % (time.time() - start))
88+
cur.execute(query, (r[0], r[1], r[2], ids[files.index(r[3])]))
89+
else:
90+
unique = unique + 1
91+
id = str(uuid.uuid4())
92+
93+
ids.append(id)
94+
files.append(r[3])
95+
96+
start = time.time()
97+
query = """insert into images
98+
(tile_id, tile_data)
99+
values (?, ?)"""
100+
cur.execute(query, (str(id), sqlite3.Binary(r[3])))
101+
logger.debug("insert into images: %s" % (time.time() - start))
102+
start = time.time()
103+
query = """insert into map
104+
(zoom_level, tile_column, tile_row, tile_id)
105+
values (?, ?, ?, ?)"""
106+
cur.execute(query, (r[0], r[1], r[2], id))
107+
logger.debug("insert into map: %s" % (time.time() - start))
108+
con.commit()
109+
110+
def compression_finalize(cur):
111+
cur.execute("""drop table tiles;""")
112+
cur.execute("""create view tiles as
113+
select map.zoom_level as zoom_level,
114+
map.tile_column as tile_column,
115+
map.tile_row as tile_row,
116+
images.tile_data as tile_data FROM
117+
map JOIN images on images.tile_id = map.tile_id;""")
118+
cur.execute("""
119+
CREATE UNIQUE INDEX map_index on map
120+
(zoom_level, tile_column, tile_row);""")
121+
cur.execute("""
122+
CREATE UNIQUE INDEX images_id on images
123+
(tile_id);""")
124+
cur.execute("""vacuum;""")
125+
cur.execute("""analyze;""")
126+
127+
def disk_to_mbtiles(directory_path, mbtiles_file):
128+
logger.info("Importing disk to MBTiles")
129+
logger.debug("%s --> %s" % (directory_path, mbtiles_file))
130+
con = mbtiles_connect(mbtiles_file)
131+
cur = con.cursor()
132+
optimize_connection(cur)
133+
mbtiles_setup(cur)
134+
try:
135+
metadata = json.load(open('%s/metadata.json' % directory_path, 'r'))
136+
for name, value in metadata.items():
137+
cur.execute('insert into metadata (name, value) values (?, ?)',
138+
(name, value))
139+
logger.info('metadata from metadata.json restored')
140+
except IOError, e:
141+
logger.warning('metadata.json not found')
142+
143+
count = 0
144+
start_time = time.time()
145+
msg = ""
146+
for r1, zs, ignore in os.walk(directory_path):
147+
for z in zs:
148+
for r2, xs, ignore in os.walk(os.path.join(r1, z)):
149+
for x in xs:
150+
for r2, ignore, ys in os.walk(os.path.join(r1, z, x)):
151+
for y in ys:
152+
f = open(os.path.join(r1, z, x, y), 'rb')
153+
cur.execute("""insert into tiles (zoom_level,
154+
tile_column, tile_row, tile_data) values
155+
(?, ?, ?, ?);""",
156+
(z, x, y.split('.')[0], sqlite3.Binary(f.read())))
157+
f.close()
158+
count = count + 1
159+
if (count % 100) == 0:
160+
for c in msg: sys.stdout.write(chr(8))
161+
msg = "%s tiles inserted (%d tiles/sec)" % (count, count / (time.time() - start_time))
162+
sys.stdout.write(msg)
163+
logger.debug('tiles inserted.')
164+
optimize_database(con)
165+
166+
def mbtiles_to_disk(mbtiles_file, directory_path):
167+
logger.debug("Exporting MBTiles to disk")
168+
logger.debug("%s --> %s" % (mbtiles_file, directory_path))
169+
con = mbtiles_connect(mbtiles_file)
170+
cur = con.cursor()
171+
os.mkdir("%s" % directory_path)
172+
metadata = dict(con.execute('select name, value from metadata;').fetchall())
173+
json.dump(metadata, open('%s/metadata.json' % directory_path, 'w'))
174+
count = con.execute('select count(zoom_level) from tiles;').fetchone()[0]
175+
done = 0
176+
msg =''
177+
tiles = con.execute('select zoom_level, tile_column, tile_row, tile_data from tiles;')
178+
t = tiles.fetchone()
179+
while t:
180+
if not os.path.isdir("%s/%s/%s/" % (directory_path, t[0], t[1])):
181+
os.makedirs("%s/%s/%s/" % (directory_path, t[0], t[1]))
182+
f = open('%s/%s/%s/%s.%s' %
183+
(directory_path, t[0], t[1], t[2], metadata.get('format', 'png')), 'wb')
184+
f.write(t[3])
185+
f.close()
186+
done = done + 1
187+
for c in msg: sys.stdout.write(chr(8))
188+
logger.info('%s / %s tiles exported' % (done, count))
189+
t = tiles.fetchone()
190+

‎setup.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
version='0.0.1',
66
author='Tom MacWright',
77
author_email='[email protected]',
8-
packages=[],
8+
packages=['mbutil'],
99
scripts=['mbutil.py'],
1010
url='https://github.com/mapbox/mbutil',
1111
license='LICENSE.md',

0 commit comments

Comments
 (0)