Skip to content

Commit

Permalink
Replace gdalinfo-json.js with node-gdal
Browse files Browse the repository at this point in the history
This patch derived from a bug where non-metric projections didn't
generate the correct `gsd` resolution. `gdalinfo-json` parses gdal's CLI
output rather than interface with the underlying gdal ABI. So using
`node-gdal` gives a more powerful and better maintained interface.

This patch also removes support for batch processing of s3 files as that
is no longer used with the OAM ecosystem.

Tests, Travis and HoundCI (linting) integration are now also included.

Fixes hotosm/oam-uploader-api#42
  • Loading branch information
tombh committed Apr 25, 2017
1 parent bd4b3cf commit 3a4e160
Show file tree
Hide file tree
Showing 17 changed files with 2,095 additions and 409 deletions.
5 changes: 0 additions & 5 deletions .env-sample

This file was deleted.

7 changes: 7 additions & 0 deletions .eslintrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"env": {
"node": true,
"es6": true
},
"extends": "semistandard"
}
2 changes: 2 additions & 0 deletions .hound.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
eslint:
enabled: true
92 changes: 0 additions & 92 deletions .jshintrc

This file was deleted.

1 change: 1 addition & 0 deletions .nvmrc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
6.10.2
2 changes: 2 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
language: node_js

53 changes: 5 additions & 48 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,28 +1,18 @@
### Metadata Generator for OIN

This node app reads objects from a S3 bucket and generates metadata json files based on OIN metadata standard.
The Open Imagery Network standardizes on a single format for imagery. This small CLI tool generates
a JSON string of the standard metadata for a given OIN geo image.

### Dependencies

You must have GDAL installed.

### Setup
### Installation

- `$ npm install`
- Copy the `.env-sample` file to `.env` and edit with your own values or set via a source like `$HOME/.aws/credentials`.
- Copy the `config-sample.json` file to `config.json` and edit with your own values.


### Usage

$ npm start

The default setting runs 20 tasks in parallel. You can change the number by playing around with `var limitParallel = 20;` on line 13 of `index.js`

#### Command Line

Metadata may also be generated for individual files by running the command line tool. If installed via `npm install`, it will be available as `node_modules/.bin/oin-meta-generator`, otherwise `bin/index.js`.

```
Usage: oin-meta-generator [args] <file>
Expand Down Expand Up @@ -83,38 +73,5 @@ $ oin-meta-generator \
}
```


### Expected Output

If you've provided a `config.json` and have provided AWS credentials, you should see a successful connection message. At this point, the app will be querying all the data files in the bucket and generating metadata for each matching file. When it finds a matching file and creates the metadata, you will see something like below in the terminal

`1 - The file saved!: 339.tiff_meta.json`

Output data is saved to the `meta` directory by default (changeable) and will look similar to

```json
{
"uuid": "http://bucket.s3.amazonaws.com/4326/srtm_01_02.tiff",
"title": "srtm_01_02.tiff",
"projection": "GEOGCS[\"WGS84\",DATUM[\"WGS_1984\",SPHEROID[\"WGS84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],AUTHORITY[\"EPSG\",\"6326\"]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433],AUTHORITY[\"EPSG\",\"4326\"]]",
"bbox": [
-180.00041666666667,
49.999583333333334,
-174.99958333333333,
55.000416666666666
],
"footprint": "POLYGON((-180.00041666666667 55.000416666666666,-174.99958333333333 55.000416666666666,-174.99958333333333 49.999583333333334,-180.00041666666667 49.999583333333334,-180.00041666666667 55.000416666666666))",
"gsd": 0.000833333333333,
"file_size": 953117,
"acquisition_start":"2015-03-02T00:00:00.000",
"acquisition_end":"2015-03-03T00:00:00.000",
"platform": "satellite",
"provider": "some satellite company",
"contact": "[email protected]",
"properties": {
"thumbnail": "link to thumbnail if available",
"tms": "link to TMS if available",
"sensor": "name of satellite"
}
}
```
### Testing
Run `npm test`
33 changes: 15 additions & 18 deletions bin/index.js
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
#!/usr/bin/env node

var fs = require('fs');
const fs = require('fs');

var gdalinfo = require('gdalinfo-json');
require('epipebomb')();
var yargs = require('yargs');
const yargs = require('yargs');

var applyGdalinfo = require('../lib/apply-gdalinfo');
const GdalInfo = require('../lib/GdalInfo');

var argv = yargs.usage('Usage: $0 [args] <file>')
const argv = yargs.usage('Usage: $0 [args] <file>')
.option('u', {
alias: 'uuid',
describe: 'Source UUID',
Expand Down Expand Up @@ -64,16 +63,16 @@ var argv = yargs.usage('Usage: $0 [args] <file>')
.version()
.argv;

var filename = argv._[0];
const filename = argv._[0];

var metadata = {
let metadata = {
uuid: argv.uuid,
title: argv.title,
platform: argv.platform,
provider: argv.provider,
contact: argv.contact,
properties: (argv.additionalMetadata || []).reduce(function (obj, pair) {
var parts = pair.split('=', 2);
const parts = pair.split('=', 2);

obj[parts[0]] = parts[1];

Expand All @@ -93,7 +92,7 @@ if (argv.uploadedAt) {
metadata.uploaded_at = new Date(argv.uploadedAt);
}

// filter out null values
// Filter out null values
metadata = Object.keys(metadata)
.filter(function (k) {
return metadata[k] != null;
Expand All @@ -104,15 +103,13 @@ metadata = Object.keys(metadata)
return obj;
}, {});

var stats = fs.statSync(filename);
const stats = fs.statSync(filename);
metadata.file_size = stats.size;

gdalinfo.local(filename, function (err, oin) {
if (err) {
throw err;
}
const imagery = new GdalInfo(filename);
metadata.projection = imagery.projectionAsWKT();
metadata.gsd = imagery.calculatePixelSize();
metadata.bbox = imagery.bboxAsArray();
metadata.footprint = imagery.bboxAsWKT();

applyGdalinfo(metadata, oin);

process.stdout.write(JSON.stringify(metadata));
});
process.stdout.write(JSON.stringify(metadata));
12 changes: 0 additions & 12 deletions config-sample.json

This file was deleted.

Loading

0 comments on commit 3a4e160

Please sign in to comment.