From 59579fc13fecf52ec05c8a9e43d94a814e786e34 Mon Sep 17 00:00:00 2001 From: Scott Bender Date: Fri, 16 Mar 2018 20:30:55 -0400 Subject: [PATCH 01/12] feature: support 'meta' deltas --- src/fullsignalk.js | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/fullsignalk.js b/src/fullsignalk.js index 90bd8b5c..188b5d9f 100644 --- a/src/fullsignalk.js +++ b/src/fullsignalk.js @@ -188,6 +188,32 @@ function addValues(context, contextPath, source, timestamp, pathValues) { } } +function addMetas(context, contextPath, source, timestamp, metas) { + var len = metas.length; + for (var i = 0; i < len; ++i) { + addMeta(context, contextPath, source, timestamp, metas[i]); + } +} + +function addMeta(context, contextPath, source, timestamp, pathValue) { + if (_.isUndefined(pathValue.path) || _.isUndefined(pathValue.value)) { + console.error("Illegal value in delta:" + JSON.stringify(pathValue)); + return; + } + var valueLeaf; + + const splitPath = pathValue.path.split('.'); + + valueLeaf = splitPath.reduce(function(previous, pathPart, i) { + if (!previous[pathPart]) { + previous[pathPart] = {}; + } + return previous[pathPart]; + }, context); + + valueLeaf.meta = _.merge(valueLeaf.meta || {}, pathValue.value) +} + function addValue(context, contextPath, source, timestamp, pathValue) { let errMessage = "" if(_.isUndefined(pathValue.path)){ From ebd2dd6db9f984a1503778e7dba4e7a599923a53 Mon Sep 17 00:00:00 2001 From: Scott Bender Date: Thu, 14 Jun 2018 00:33:03 -0400 Subject: [PATCH 02/12] update: delta schema to include meta --- schemas/delta.json | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/schemas/delta.json b/schemas/delta.json index c05ecde1..a9b6988c 100644 --- a/schemas/delta.json +++ b/schemas/delta.json @@ -18,7 +18,14 @@ "description": "The updates", "items": { "type": "object", - "required": ["values"], + "oneOf": [ + { + "required": ["values"] + }, + { + "required": ["meta"] + } + ], "not": { "allOf": [{ @@ -64,6 +71,33 @@ } } } + }, + "meta": { + "type": "array", + "items": { + "type": "object", + "required": [ + "path", + "value" + ], + "properties": { + "path": { + "type": "string", + "description": "The local path to the data value", + "example": "navigation.courseOverGroundMagnetic" + }, + "value": { + "type": [ + "string", + "number", + "object", + "boolean", + "null" + ], + "additionalProperties": true + } + } + } } } } From c6ad87b48a7be025d86fa287cc4b86b1af8d2d55 Mon Sep 17 00:00:00 2001 From: Scott Bender Date: Thu, 14 Jun 2018 00:35:14 -0400 Subject: [PATCH 03/12] chore: add tests for meta deltas --- test/data/delta-invalid/meta-missing-value.json | 12 ++++++++++++ test/data/delta-valid/meta-delta.json | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 test/data/delta-invalid/meta-missing-value.json create mode 100644 test/data/delta-valid/meta-delta.json diff --git a/test/data/delta-invalid/meta-missing-value.json b/test/data/delta-invalid/meta-missing-value.json new file mode 100644 index 00000000..1430b26c --- /dev/null +++ b/test/data/delta-invalid/meta-missing-value.json @@ -0,0 +1,12 @@ +{ + "context": "bar", + "updates": [{ + "source": { + "label": "schema" + }, + "timestamp": "2013-10-08T15:47:28.263Z", + "meta": [{ + "path": "a.b.c" + }] + }] +} diff --git a/test/data/delta-valid/meta-delta.json b/test/data/delta-valid/meta-delta.json new file mode 100644 index 00000000..683289c9 --- /dev/null +++ b/test/data/delta-valid/meta-delta.json @@ -0,0 +1,15 @@ +{ + "context": "bar", + "updates": [{ + "source": { + "label": "schema" + }, + "timestamp": "2013-10-08T15:47:28.263Z", + "meta": [{ + "path": "a.b.c", + "value": { + "units": "m" + } + }] + }] +} From 802fa0b9c1665f9f3a66c028c043c39e0f6a5df9 Mon Sep 17 00:00:00 2001 From: Scott Bender Date: Thu, 14 Jun 2018 14:00:47 -0400 Subject: [PATCH 04/12] update: add the meta delta to the delta documentation --- gitbook-docs/data_model.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/gitbook-docs/data_model.md b/gitbook-docs/data_model.md index 6d45e393..0f234043 100644 --- a/gitbook-docs/data_model.md +++ b/gitbook-docs/data_model.md @@ -263,6 +263,32 @@ the value should be merged to the full model mounted where the delta‘s context ] } ``` +## Delta Format For Metadata + +Metadata can also be specified via a delta. The delta is very similiar to the `values` deltas above, but instead of having `values` key, it will have a `meta` key. Note that a client could multiple meta deltas for any given path from different sources, so the client merge the meta information. + +```json +{ + "context": "vessels.urn:mrn:imo:mmsi:234567890", + "updates": [ + { + "source": {...}, + "timestamp": "2014-08-15T19:02:31.507Z", + "meta":[ + { + "path": "environment.wind.speedApparent", + "value": + { + "units": "m/s", + "description": "Apparent wind speed" + } + } + ] + } + ] +} +``` + [<]: # ## Data Quality From c41d8dbe7d63e104dc34227241569c1d3217edbb Mon Sep 17 00:00:00 2001 From: Scott Bender Date: Fri, 22 Jun 2018 10:44:17 -0400 Subject: [PATCH 05/12] update: use ./definitions.json#/definitions/meta for the meta delta value --- schemas/delta.json | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/schemas/delta.json b/schemas/delta.json index a9b6988c..80de53c8 100644 --- a/schemas/delta.json +++ b/schemas/delta.json @@ -87,13 +87,7 @@ "example": "navigation.courseOverGroundMagnetic" }, "value": { - "type": [ - "string", - "number", - "object", - "boolean", - "null" - ], + "$ref": "./definitions.json#/definitions/meta" "additionalProperties": true } } From f8e25b615f23f07e68ac8a1a3b516c6f01278067 Mon Sep 17 00:00:00 2001 From: Scott Bender Date: Wed, 11 Jul 2018 08:47:49 -0400 Subject: [PATCH 06/12] fix: syntax error in delta schema --- schemas/delta.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemas/delta.json b/schemas/delta.json index 80de53c8..7e2ae5d6 100644 --- a/schemas/delta.json +++ b/schemas/delta.json @@ -87,7 +87,7 @@ "example": "navigation.courseOverGroundMagnetic" }, "value": { - "$ref": "./definitions.json#/definitions/meta" + "$ref": "./definitions.json#/definitions/meta", "additionalProperties": true } } From 7aef93e63194cdb259b241138f5ad58273b3aa06 Mon Sep 17 00:00:00 2001 From: Scott Bender Date: Wed, 11 Jul 2018 08:58:55 -0400 Subject: [PATCH 07/12] fix: support both meta and values in a delta --- schemas/delta.json | 14 +++++++++++--- test/data/delta-valid/meta-and-values.json | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 test/data/delta-valid/meta-and-values.json diff --git a/schemas/delta.json b/schemas/delta.json index 7e2ae5d6..b8228823 100644 --- a/schemas/delta.json +++ b/schemas/delta.json @@ -20,13 +20,21 @@ "type": "object", "oneOf": [ { - "required": ["values"] + "required": ["values"], + "properties": {"$source":{}, "source": {}, "timestamp": {}, "values": {}}, + "additionalProperties": false }, { - "required": ["meta"] + "required": ["meta"], + "properties": {"$source":{}, "source": {}, "timestamp": {}, "meta": {}}, + "additionalProperties": false + }, + { + "required": ["values", "meta"], + "properties": {"$source":{}, "source": {}, "timestamp": {}, "values": {}, "meta": {}}, + "additionalProperties": false } ], - "not": { "allOf": [{ "required": ["source"] diff --git a/test/data/delta-valid/meta-and-values.json b/test/data/delta-valid/meta-and-values.json new file mode 100644 index 00000000..1fa735c4 --- /dev/null +++ b/test/data/delta-valid/meta-and-values.json @@ -0,0 +1,19 @@ +{ + "context": "bar", + "updates": [{ + "source": { + "label": "schema" + }, + "timestamp": "2013-10-08T15:47:28.263Z", + "meta": [{ + "path": "a.b.c", + "value": { + "units": "m" + } + }], + "values": [{ + "path": "a.b.c", + "value": 1234 + }] + }] +} From dc873a3da783e53081867f3892f5e008383bd75a Mon Sep 17 00:00:00 2001 From: Barry Pitfield Date: Fri, 13 Jul 2018 16:37:40 +0100 Subject: [PATCH 08/12] fix: add mandatory description to meta deltas (#489) --- test/data/delta-valid/meta-and-values.json | 1 + test/data/delta-valid/meta-delta.json | 1 + 2 files changed, 2 insertions(+) diff --git a/test/data/delta-valid/meta-and-values.json b/test/data/delta-valid/meta-and-values.json index 1fa735c4..4f8ded2d 100644 --- a/test/data/delta-valid/meta-and-values.json +++ b/test/data/delta-valid/meta-and-values.json @@ -8,6 +8,7 @@ "meta": [{ "path": "a.b.c", "value": { + "description": "...", "units": "m" } }], diff --git a/test/data/delta-valid/meta-delta.json b/test/data/delta-valid/meta-delta.json index 683289c9..ad80a379 100644 --- a/test/data/delta-valid/meta-delta.json +++ b/test/data/delta-valid/meta-delta.json @@ -8,6 +8,7 @@ "meta": [{ "path": "a.b.c", "value": { + "description": "...", "units": "m" } }] From 65a573cbd712cf11f7447dbedbcb69ca248afe20 Mon Sep 17 00:00:00 2001 From: Barry Pitfield Date: Wed, 25 Jul 2018 18:42:57 +0100 Subject: [PATCH 09/12] docs: modify re meta deltas (#491) --- gitbook-docs/data_model.md | 28 +++++++++++++------ gitbook-docs/subscription_protocol.md | 9 +++++- .../delta/docs-data_model_meta_deltas.json | 25 +++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) create mode 100644 samples/delta/docs-data_model_meta_deltas.json diff --git a/gitbook-docs/data_model.md b/gitbook-docs/data_model.md index 0f234043..519bd60d 100644 --- a/gitbook-docs/data_model.md +++ b/gitbook-docs/data_model.md @@ -263,32 +263,42 @@ the value should be merged to the full model mounted where the delta‘s context ] } ``` +[<]: # ## Delta Format For Metadata -Metadata can also be specified via a delta. The delta is very similiar to the `values` deltas above, but instead of having `values` key, it will have a `meta` key. Note that a client could multiple meta deltas for any given path from different sources, so the client merge the meta information. +Metadata can also be updated via a delta within the `meta` key. + +Since meta data is not often updated it is only sent when there has been a change. See [Subscription Protocol](subscription_protocol.md) for details. +[>]: # (mdpInsert ```json fsnip ../samples/delta/docs-data_model_meta_deltas.json --prettify 2 20) ```json { "context": "vessels.urn:mrn:imo:mmsi:234567890", "updates": [ { - "source": {...}, "timestamp": "2014-08-15T19:02:31.507Z", - "meta":[ + "meta": [ { "path": "environment.wind.speedApparent", - "value": - { - "units": "m/s", - "description": "Apparent wind speed" - } + "value": { + "units": "m/s", + "description": "Apparent wind speed", + "displayName": "Apparent Wind Speed", + "shortName": "AWS", + "zones": [ + { + "upper": 15.4333, + "state": "warn", + "message": "high wind speed" + } + ] + } } ] } ] } ``` - [<]: # ## Data Quality diff --git a/gitbook-docs/subscription_protocol.md b/gitbook-docs/subscription_protocol.md index 184bcfe0..61bf3301 100644 --- a/gitbook-docs/subscription_protocol.md +++ b/gitbook-docs/subscription_protocol.md @@ -86,7 +86,7 @@ The following are optional, included above only for example as it uses defaults * `period=[millisecs]` becomes the transmission rate, e.g. every `period/1000` seconds. Default: 1000 * `format=[delta|full]` specifies delta or full format. Default: delta -* `policy=[instant|ideal|fixed]`. Default: ideal +* `policy=[instant|ideal|fixed]`. Default: ideal. (does not apply to meta - see below) * `instant` means send all changes as fast as they are received, but no faster than `minPeriod`. With this policy the client has an immediate copy of the current state of the server. * `ideal` means use `instant` policy, but if no changes are received before `period`, then resend the last known @@ -101,6 +101,13 @@ duplication as it prefers on a per connection basis. At the same time it is good connections necessary, for instance one WebSocket connection shared between an instrument panel with many gauges, rather then one WebSocket connection per gauge. +## Meta data +Meta is updated via the `meta` section within the delta message. As meta changes infrequently it is only sent when it has changed. + +Servers implementing the subscription model (ie. using deltas) SHOULD implement meta deltas. Where meta deltas are implemented, servers MUST only ever send full copies of the meta for a leaf, ie. they MUST NEVER send a partial meta. + +Upon receiving a new subscription a server MUST send the meta for each leaf subscribed to; this MAY be in the same JSON document as the values, or in a separate one prior to sending values for that leaf or leaves. Subsequently the server MUST resend the full meta for a leaf each time any item in that meta is changed. This is equivalent to the `instant` subscription for values. Therefore meta is never subscribed on an `ideal` or `fixed` policy, irrespective of the policy requested by the consumer (which applies to values only). + ## Multiple value handling in subscriptions A subscription to a key is for all the updates to that key. If there are multiple sources generating data for that key diff --git a/samples/delta/docs-data_model_meta_deltas.json b/samples/delta/docs-data_model_meta_deltas.json new file mode 100644 index 00000000..f683d158 --- /dev/null +++ b/samples/delta/docs-data_model_meta_deltas.json @@ -0,0 +1,25 @@ +{ + "context": "vessels.urn:mrn:imo:mmsi:234567890", + "updates": [ + { + "timestamp": "2014-08-15T19:02:31.507Z", + "meta": [ + { + "path": "environment.wind.speedApparent", + "value": { + "units": "m/s", + "description": "Apparent wind speed", + "displayName": "Apparent Wind Speed", + "shortName": "AWS", + "zones": [{ + "upper": 15.4333, + "state": "warn", + "message": "high wind speed" + } + ] + } + } + ] + } + ] +} \ No newline at end of file From ebb46a930f090c59e8686194f139b102edb3e34b Mon Sep 17 00:00:00 2001 From: Barry Pitfield Date: Wed, 3 Nov 2021 19:19:38 +0000 Subject: [PATCH 10/12] fix: remove additionalProperties from meta deltas (#492) --- schemas/delta.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/schemas/delta.json b/schemas/delta.json index b8228823..c4b056e7 100644 --- a/schemas/delta.json +++ b/schemas/delta.json @@ -95,8 +95,7 @@ "example": "navigation.courseOverGroundMagnetic" }, "value": { - "$ref": "./definitions.json#/definitions/meta", - "additionalProperties": true + "$ref": "./definitions.json#/definitions/meta" } } } From 6a8e2e1656778924ee3fd3e8641e8bb4ae67c96d Mon Sep 17 00:00:00 2001 From: Barry Pitfield Date: Mon, 13 Dec 2021 20:06:49 +0000 Subject: [PATCH 11/12] style: modify and reformat json test files (#493) --- .../delta-invalid/meta-missing-value.json | 22 ++++----- test/data/delta-valid/meta-and-values.json | 45 +++++++++++-------- test/data/delta-valid/meta-delta.json | 37 +++++++++------ 3 files changed, 61 insertions(+), 43 deletions(-) diff --git a/test/data/delta-invalid/meta-missing-value.json b/test/data/delta-invalid/meta-missing-value.json index 1430b26c..65d83097 100644 --- a/test/data/delta-invalid/meta-missing-value.json +++ b/test/data/delta-invalid/meta-missing-value.json @@ -1,12 +1,12 @@ { - "context": "bar", - "updates": [{ - "source": { - "label": "schema" - }, - "timestamp": "2013-10-08T15:47:28.263Z", - "meta": [{ - "path": "a.b.c" - }] - }] -} + "context": "vessels.urn:mrn:imo:mmsi:234567890", + "updates": [ + { + "meta": [ + { + "path": "propulsion.0.revolutions" + } + ] + } + ] +} \ No newline at end of file diff --git a/test/data/delta-valid/meta-and-values.json b/test/data/delta-valid/meta-and-values.json index 4f8ded2d..608a8d2a 100644 --- a/test/data/delta-valid/meta-and-values.json +++ b/test/data/delta-valid/meta-and-values.json @@ -1,20 +1,29 @@ { - "context": "bar", - "updates": [{ - "source": { - "label": "schema" - }, - "timestamp": "2013-10-08T15:47:28.263Z", - "meta": [{ - "path": "a.b.c", - "value": { - "description": "...", - "units": "m" + "context": "vessels.urn:mrn:imo:mmsi:234567890", + "updates": [ + { + "source": { + "label": "N2000-01", + "type": "NMEA2000", + "src": "017", + "pgn": 127488 + }, + "timestamp": "2013-10-08T15:47:28.263Z", + "meta": [ + { + "path": "propulsion.0.revolutions", + "value": { + "description": "Engine revolutions (x60 for RPM)", + "units": "Hz" } - }], - "values": [{ - "path": "a.b.c", - "value": 1234 - }] - }] -} + } + ], + "values": [ + { + "path": "propulsion.0.revolutions", + "value": 11 + } + ] + } + ] +} \ No newline at end of file diff --git a/test/data/delta-valid/meta-delta.json b/test/data/delta-valid/meta-delta.json index ad80a379..3e41a653 100644 --- a/test/data/delta-valid/meta-delta.json +++ b/test/data/delta-valid/meta-delta.json @@ -1,16 +1,25 @@ { - "context": "bar", - "updates": [{ - "source": { - "label": "schema" - }, - "timestamp": "2013-10-08T15:47:28.263Z", - "meta": [{ - "path": "a.b.c", - "value": { - "description": "...", - "units": "m" + "context": "vessels.urn:mrn:imo:mmsi:234567890", + "updates": [ + { + "meta": [ + { + "path": "environment.depth.belowTransducer", + "value": { + "description": "Depth below Transducer", + "units": "m", + "timeout": 2.5, + "zones": [ + { + "upper": 5.0, + "state": "warn", + "message": "Shallow water" + } + ], + "warnMethod": ["sound"] } - }] - }] -} + } + ] + } + ] +} \ No newline at end of file From d4b3e596b748bc939d7c829e7665f8c3b8502074 Mon Sep 17 00:00:00 2001 From: Teppo Kurki Date: Mon, 13 Dec 2021 22:23:34 +0200 Subject: [PATCH 12/12] chore: remove addMetas already implemented in master --- src/fullsignalk.js | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/fullsignalk.js b/src/fullsignalk.js index 188b5d9f..90bd8b5c 100644 --- a/src/fullsignalk.js +++ b/src/fullsignalk.js @@ -188,32 +188,6 @@ function addValues(context, contextPath, source, timestamp, pathValues) { } } -function addMetas(context, contextPath, source, timestamp, metas) { - var len = metas.length; - for (var i = 0; i < len; ++i) { - addMeta(context, contextPath, source, timestamp, metas[i]); - } -} - -function addMeta(context, contextPath, source, timestamp, pathValue) { - if (_.isUndefined(pathValue.path) || _.isUndefined(pathValue.value)) { - console.error("Illegal value in delta:" + JSON.stringify(pathValue)); - return; - } - var valueLeaf; - - const splitPath = pathValue.path.split('.'); - - valueLeaf = splitPath.reduce(function(previous, pathPart, i) { - if (!previous[pathPart]) { - previous[pathPart] = {}; - } - return previous[pathPart]; - }, context); - - valueLeaf.meta = _.merge(valueLeaf.meta || {}, pathValue.value) -} - function addValue(context, contextPath, source, timestamp, pathValue) { let errMessage = "" if(_.isUndefined(pathValue.path)){