Merge branch 'master' into branch-0.11

Conflicts:
	CHANGELOG.creole
	pgmapcss/version.py

Author: plepe

CHANGELOG.creole

@@ -1,6 +1,11 @@
 0.11 release upcoming
 * Mode standalone: specify ID or list of IDs instead of a bounding box
 
+0.10.5 release 2015-05-05
+* DB/Overpass: download from API in blocks, to avoid huge memory consumption. specify {{{db.overpass-blocksize}}} (default: 64MB).
+* Icon handling from 0.10.4 also for ol4pgm
+* Update Maki icons
+
 0.10.4 release 2015-03-11
 * DB/Overpass: set max allowed memory by config option {{{db.overpass-memory}}}
 * DB/Overpass: load full response at once; this greatly improves performance, but needs more memory.

README.md

@@ -10,7 +10,7 @@ Alternatively there's a standalone mode, where the MapCSS style is compiled into
 
 Since version 0.10 you don't even need a local database import (a local database is needed nonetheless), as the new default data source is Overpass API. Alternatively, osm2pgsql and osmosis pgsnapshot are supported. See [doc/database.md](doc/database.md) for details.
 
-Stable version: [0.10.4](https://github.com/plepe/pgmapcss) ([CHANGELOG](https://github.com/plepe/pgmapcss/blob/master/CHANGELOG.creole)), development versions: [0.11-dev](https://github.com/plepe/pgmapcss/tree/branch-0.11) ([open issues](https://github.com/plepe/pgmapcss/milestones/Version%200.11)) resp. [NG-dev](https://github.com/plepe/pgmapcss/tree/pgmapcss-NG)([open issues](https://github.com/plepe/pgmapcss/milestones/pgmapcss%20NG)).
+Stable version: [0.10.5](https://github.com/plepe/pgmapcss) ([CHANGELOG](https://github.com/plepe/pgmapcss/blob/master/CHANGELOG.creole)), development versions: [0.11-dev](https://github.com/plepe/pgmapcss/tree/branch-0.11) ([open issues](https://github.com/plepe/pgmapcss/milestones/Version%200.11)) resp. [NG-dev](https://github.com/plepe/pgmapcss/tree/pgmapcss-NG)([open issues](https://github.com/plepe/pgmapcss/milestones/pgmapcss%20NG)).
 
 Features
 --------

doc/config_options.md

@@ -33,3 +33,4 @@ Advances options:
 | db.overpass-timeout | overpass only: specify timeout in seconds for queries to the Overpass API (make sure that the Overpass server accepts timeouts that long, e.g. by adding a "Timeout" parameter to Apache2). As a render job usually needs several queries, the total rendering time might be a multiple of this value. | 180
 | db.overpass-memory | overpass only: Sets the 'maxsize' parameter for an overpass query, which indicates the maximum allowed memory for the query in bytes RAM on the server. | 536870912 (512 MB)
 | db.overpass-profiler | overpass only: prints all queries, their duration (incl. parsing json) and the count of returned features. | true/**false**
+| db.overpass-blocksize | overpass only: When reading from Overpass API read in blocks of specified size. Reduces memory impact. | 67108864 (64 MB)

doc/database.md

@@ -30,6 +30,7 @@ Behaviour can be influenced with the following config options:
 | db.overpass-timeout | specify timeout in seconds for queries to the Overpass API (make sure that the Overpass server accepts timeouts that long, e.g. by adding a "Timeout" parameter to Apache2). As a render job usually needs several queries, the total rendering time might be a multiple of this value. | 180
 | db.overpass-memory | Sets the 'maxsize' parameter for an overpass query, which indicates the maximum allowed memory for the query in bytes RAM on the server. | 536870912 (512 MB)
 | db.overpass-profiler | overpass only: prints all queries, their duration (incl. parsing json) and the count of returned features. | true/**false**
+| db.overpass-blocksize | overpass only: When reading from Overpass API read in blocks of specified size. Reduces memory impact. | 67108864 (64 MB)
 
 Example usage:
 ```sh

pgmapcss/db/overpass/db.py

@@ -19,6 +19,9 @@ def __init__(self, conn, stat):
         if not 'db.overpass-memory' in self.stat['config']:
             self.stat['config']['db.overpass-memory'] = '536870912'
 
+        if not 'db.overpass-blocksize' in self.stat['config']:
+            self.stat['config']['db.overpass-blocksize'] = '67108864'
+
         self.parent_queries = []
 
     def tag_type(self, key, condition):

pgmapcss/db/overpass/db_functions.py

@@ -7,43 +7,120 @@ def overpass_query(query):
 
 # START db.overpass-profiler
     time_start = datetime.datetime.now()
+    time_duration = datetime.timedelta(0)
+    download_size = 0
+    count = 0
+    count_blocks = 0
 # END db.overpass-profiler
     url = '{db.overpass-url}/interpreter'
     data = urllib.parse.urlencode({ 'data': query })
     data = data.encode('utf-8')
-    count = 0
 
     try:
         f = urllib.request.urlopen(url, data)
     except urllib.error.HTTPError as err:
         plpy.warning('Overpass query failed:\n' + query)
         raise
 
-    try:
-        r = f.read().decode('utf-8')
-    except urllib.error.HTTPError as err:
-        plpy.warning('Overpass query failed (after {} features):\n'.format(count) + query)
-        raise
+    first = True
+    done = False
+    to_parse = None
+    result = None
+    block_remains = ''
 
-    # areas not initialized -> ignore
-    if re.search('osm3s_v[0-9\.]+_areas', r):
-        f.close()
-        return
+    while not done:
+# START db.overpass-profiler
+        count_blocks += 1
+# END db.overpass-profiler
+        try:
+            r = f.read({db.overpass-blocksize})
+# START db.overpass-profiler
+            download_size += len(r)
+# END db.overpass-profiler
 
-    data = json.loads(r)
+            # make sure that result is valid UTF-8
+            # thanks to http://rosettacode.org/wiki/Read_a_file_character_by_character/UTF8#Python
+            while True:
+                try:
+                    r = r.decode('utf-8')
+                except UnicodeDecodeError:
+                    r += f.read(1)
+# START db.overpass-profiler
+                    download_size += 1
+# END db.overpass-profiler
+                else:
+                    break
+        except urllib.error.HTTPError as err:
+            plpy.warning('Overpass query failed (after {} features):\n'.format(count) + query)
+            raise
+
+        # EOF detected
+        if not r:
+            done = True
+
+        # areas not initialized -> ignore
+        if first and re.search('osm3s_v[0-9\.]+_areas', r):
+            f.close()
+            return
+
+        if first:
+            # maybe we could read whole result in blocksize
+            first = False
+            try:
+                result = json.loads(r)
+            except ValueError:
+                pass
+            else:
+                done = True
 
-    if 'remark' in data:
+        # try to read complete elements from block, remember surroundings for
+        # later (block_remains)
+        if done:
+            to_parse = block_remains
+        else:
+            block_remains += r
+            end_last = block_remains.rfind('\n},')
+            if end_last != -1:
+                start = block_remains.find('"elements": [')
+                to_parse = block_remains[0 : end_last] + '}]\n}'
+                block_remains = block_remains[0 : start + 13] + block_remains[end_last + 3 :]
+
+        # try to load JSON - if not complete, try after next reading
+        if to_parse:
+            try:
+                result = json.loads(to_parse)
+                to_parse = None
+            except ValueError:
+                pass
+
+# START db.overpass-profiler
+        time_duration += (datetime.datetime.now() - time_start)
+# END db.overpass-profiler
+
+        # found a result
+        if result:
+# START db.overpass-profiler
+            count += len(result['elements'])
+# END db.overpass-profiler
+            for e in result['elements']:
+                yield e
+            result['elements'] = []
+
+# START db.overpass-profiler
+        time_start = datetime.datetime.now()
+# END db.overpass-profiler
+
+    if 'remark' in result:
         # ignore timeout if it happens in "print"
         if not re.search("Query timed out in \"print\"", data['remark']):
           raise Exception('Error in Overpass API (after {} features): {}\nFailed query was:\n{}'.format(count, data['remark'], query))
 
+    f.close()
+
 # START db.overpass-profiler
-    plpy.warning('%s\nquery took %.2fs for %d features' % (query, (datetime.datetime.now() - time_start).total_seconds(), len(data['elements'])))
+    plpy.warning('%s\nquery took %.2fs for %d features (%.1f MB, %d blocks)' % (query, time_duration.total_seconds(), count, download_size / 1024.0 / 1024, count_blocks))
 # END db.overpass-profiler
 
-    for e in data['elements']:
-        yield e
-
 def node_geom(lat, lon):
     global geom_plan