Merge pull request #171 from developmentseed/endpoint

tms tile image format

Author: Martha Morrissey

docs/parameters.rst

@@ -44,7 +44,7 @@ Here is the full list of configuration parameters you can specify in a ``config.
 	One of ``'classification'``, ``'object-detection'``, or ``'segmentation'``. This defines the output format for the final label numpy arrays (``y_train`` and ``y_test``).
 
  	``'classification'``
- 		Output is an array of ``len(classes) + 1``. Each array value will be either `1` or `0` based on whether it matches the class at the same index. The additional array element belongs to the background class, which will always be the first element. 
+ 		Output is an array of ``len(classes) + 1``. Each array value will be either `1` or `0` based on whether it matches the class at the same index. The additional array element belongs to the background class, which will always be the first element.
 
  	``'object-detection'``
  		Output is an array of bounding boxes of the form ``[xmin, ymin, width, height, class_index]``. In this case, the values are pixel values measured from the upper left-hand corner (not latitude and longitude values). Each feature is tested against each class, so if a feature matches two or more classes, it will have the corresponding number of bounding boxes created.
@@ -65,3 +65,6 @@ Here is the full list of configuration parameters you can specify in a ``config.
 
 **imagery_offset**:  list of ints
 	An optional list of integers representing the number of pixels to offset imagery. For example ``[15, -5]`` will move the images 15 pixels right and 5 pixels up relative to the requested tile bounds.
+
+**tms_image_format**: string
+ An option string that has the downloaded imagery's format such as `.jpg` or `.png` when it isn't provided by the endpoint

label_maker/package.py

@@ -6,7 +6,7 @@
 import numpy as np
 from PIL import Image
 
-from label_maker.utils import is_tif
+from label_maker.utils import is_tif, get_image_format
 
 
 def package_directory(dest_folder, classes, imagery, ml_type, seed=False,
@@ -72,10 +72,12 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False,
     y_vals = []
 
     # open the images and load those plus the labels into the final arrays
-    o = urlparse(imagery)
-    _, image_format = op.splitext(o.path)
     if is_tif(imagery):  # if a TIF is provided, use jpg as tile format
         image_format = '.jpg'
+
+    else:
+        image_format = get_image_format(imagery, kwargs)
+
     for tile in tiles:
         image_file = op.join(dest_folder, 'tiles', '{}{}'.format(tile, image_format))
         try:

label_maker/utils.py

@@ -27,10 +27,19 @@ def class_match(ml_type, label, i):
         return np.count_nonzero(label == i)
     return None
 
+def get_image_format(imagery, kwargs):
+    if kwargs.get('tms_image_format'):
+        image_format =  kwargs.get('tms_image_format')
+    else:
+        o = urlparse(imagery)
+        _, image_format = op.splitext(o.path)
+    return image_format
+
 def download_tile_tms(tile, imagery, folder, kwargs):
     """Download a satellite image tile from a tms endpoint"""
-    o = urlparse(imagery)
-    _, image_format = op.splitext(o.path)
+
+    image_format = get_image_format(imagery, kwargs)
+
     r = requests.get(url(tile.split('-'), imagery),
                      auth=kwargs.get('http_auth'))
     tile_img = op.join(folder, '{}{}'.format(tile, image_format))

label_maker/validate.py

@@ -33,5 +33,6 @@
     'seed': {'type': 'integer'},
     'imagery_offset': {'type': 'list', 'schema': {'type': 'integer'}, 'minlength': 2, 'maxlength': 2},
     'split_vals': {'type': 'list', 'schema': {'type': 'float'}},
-    'split_names': {'type': 'list', 'schema': {'type': 'string'}}
+    'split_names': {'type': 'list', 'schema': {'type': 'string'}},
+    'tms_image_format': {'type': 'string'}
 }

test/fixtures/integration/config_tms_format_img.json

@@ -0,0 +1,13 @@
+{
+  "country": "togo",
+  "bounding_box": [1.09725, 6.05520, 1.34582, 6.30915],
+  "zoom": 12,
+  "classes": [
+    { "name": "Roads", "filter": ["has", "highway"] },
+    { "name": "Buildings", "filter": ["has", "building"] }
+  ],
+  "imagery": "http://a.tiles.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg?access_token=ACCESS_TOKEN",
+  "background_ratio": 1,
+  "ml_type": "classification",
+  "tms_image_format": ".png"
+}

test/fixtures/integration/labels-cl-img-f.npz

@@ -21,10 +21,15 @@ def setUpClass(cls):
         copyfile('test/fixtures/integration/labels-cl.npz', 'integration-cl-split/labels.npz')
         copytree('test/fixtures/integration/tiles', 'integration-cl-split/tiles')
 
+        makedirs('integration-cl-img-f')
+        copyfile('test/fixtures/integration/labels-cl-img-f.npz', 'integration-cl-img-f/labels.npz')
+        copytree('test/fixtures/integration/tiles_png', 'integration-cl-img-f/tiles')
+
     @classmethod
     def tearDownClass(cls):
         rmtree('integration-cl')
         rmtree('integration-cl-split')
+        rmtree('integration-cl-img-f')
 
     def test_cli(self):
         """Verify data.npz produced by CLI"""
@@ -73,4 +78,21 @@ def test_cli_3way_split(self):
         # validate label data with shapes
         self.assertEqual(data['y_train'].shape, (5, 7))
         self.assertEqual(data['y_test'].shape, (2, 7))
-        self.assertEqual(data['y_val'].shape, (1, 7))
+        self.assertEqual(data['y_val'].shape, (1, 7))
+
+    def test_tms_img_format(self):
+        """Verify data.npz produced by CLI"""
+
+        cmd = 'label-maker package --dest integration-cl-img-f --config test/fixtures/integration/config_tms_format_img.json'
+        cmd = cmd.split(' ')
+        subprocess.run(cmd, universal_newlines=True)
+
+        data = np.load('integration-cl-img-f/data.npz')
+
+        # validate our image data with shapes
+        self.assertEqual(data['x_train'].shape, (9, 256, 256, 3))
+        self.assertEqual(data['x_test'].shape, (3, 256, 256, 3))
+
+        # validate label data with shapes
+        self.assertEqual(data['y_train'].shape, (9, 3))
+        self.assertEqual(data['y_test'].shape, (3, 3))