Skip to content

Commit cac4d5c

Browse files
jeffkalajeffkala
committed
JSON data reconstruction for remediation
1 parent a870990 commit cac4d5c

File tree

3 files changed

+175
-39
lines changed

3 files changed

+175
-39
lines changed

‎docs/user/usage.md‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,3 +610,120 @@ Can you guess what would be the outcome for an `int`, `float` operator?
610610
```
611611

612612
See `tests` folder in the repo for more examples.
613+
614+
## Putting a Result Back Together
615+
616+
Jdiff results are very helpful in determining what is wrong with the outputs. What if you want to reconstruct the results in order to fix the problem. The `parse_diff` helper does just that. Imagine you have a `jdiff` result such as:
617+
618+
Examples of jdiff evaluated results:
619+
620+
```python
621+
ex1 = {'bar-2': 'missing', 'bar-1': 'new'}
622+
ex2 = {
623+
'hostname': {'new_value': 'veos-actual', 'old_value': 'veos-intended'},
624+
'domain-name': 'new'
625+
}
626+
ex3 = {
627+
'hostname': {'new_value': 'veos-0', 'old_value': 'veos'},
628+
"index_element['ip name']": 'missing',
629+
'domain-name': 'new'
630+
}
631+
ex4 = {
632+
'servers':
633+
{
634+
'server': defaultdict(<class 'list'>,
635+
{
636+
'missing': [
637+
{
638+
'address': '1.us.pool.ntp.org',
639+
'config': {'address': '1.us.pool.ntp.org'},
640+
'state': {'address': '1.us.pool.ntp.org'}
641+
}
642+
]
643+
}
644+
)
645+
}
646+
}
647+
```
648+
649+
And you need to understand what is extra and what is missing from the result. (Think configuration compliance on a JSON/JSON-RPC system).
650+
651+
Well running the `parse_diff` will give you what is extra (in the comparison data) and missing from the reference data, and also the reverse. What is missing (in the reference data) that is missing from the comparison data.
652+
653+
An example will help visualize the results.
654+
655+
```python
656+
In [1]: from jdiff import extract_data_from_json
657+
...: from jdiff.check_types import CheckType
658+
...: from jdiff.utils.diff_helpers import parse_diff
659+
660+
In [2]: reference_data = {"foo": {"bar-2": "baz2"}}
661+
...: comparison_data = {"foo": {"bar-1": "baz1"}}
662+
...: match_key = "foo"
663+
664+
In [3]: extracted_comparison_data = extract_data_from_json(comparison_data, match_key)
665+
666+
In [4]: extracted_comparison_data
667+
Out[4]: {'bar-1': 'baz1'}
668+
669+
In [5]: extracted_reference_data = extract_data_from_json(reference_data, match_key)
670+
671+
In [6]: extracted_reference_data
672+
Out[6]: {'bar-2': 'baz2'}
673+
674+
In [7]: jdiff_exact_match = CheckType.create("exact_match")
675+
...: jdiff_evaluate_response, _ = jdiff_exact_match.evaluate(extracted_reference_data, extracted_comparison_data)
676+
677+
In [8]: jdiff_evaluate_response
678+
Out[8]: {'bar-2': 'missing', 'bar-1': 'new'}
679+
680+
In [9]: parsed_extra, parsed_missing = parse_diff(
681+
...: jdiff_evaluate_response,
682+
...: comparison_data,
683+
...: reference_data,
684+
...: match_key,
685+
...: )
686+
...:
687+
688+
In [10]: parsed_extra
689+
Out[10]: {'bar-1': 'baz1'}
690+
691+
In [10]: parsed_missing
692+
Out[10]: {'bar-2': 'baz2'}
693+
```
694+
695+
What about one with a more true JSON data structure. Like this RESTCONF YANG response.
696+
697+
```python
698+
from jdiff import extract_data_from_json
699+
from jdiff.check_types import CheckType
700+
from jdiff.utils.diff_helpers import parse_diff
701+
702+
reference_data = {"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}
703+
comparison_data = {"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}
704+
match_key = '"openconfig-system:config"'
705+
extracted_comparison_data = extract_data_from_json(comparison_data, match_key)
706+
extracted_reference_data = extract_data_from_json(reference_data, match_key)
707+
jdiff_exact_match = CheckType.create("exact_match")
708+
jdiff_evaluate_response, _ = jdiff_exact_match.evaluate(extracted_reference_data, extracted_comparison_data)
709+
710+
parsed_extra, parsed_missing = parse_diff(
711+
jdiff_evaluate_response,
712+
comparison_data,
713+
reference_data,
714+
match_key,
715+
)
716+
```
717+
Which results in:
718+
719+
```python
720+
In [24]: parsed_extra
721+
{'hostname': 'veos-0', 'domain-name': 'ntc.com'}
722+
723+
In [25]: parsed_missing
724+
Out[25]: {'hostname': 'veos', 'ip name': 'ntc.com'}
725+
```
726+
727+
Now you can see how valuable this data can be to reconstruct, or remediate a out of compliant JSON object.
728+
729+
For more detailed examples see the `test_diff_helpers.py` file.

‎jdiff/utils/diff_helpers.py‎

Lines changed: 37 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -122,43 +122,58 @@ def set_nested_value(data, keys, value):
122122
set_nested_value(data[keys[0]], keys[1:], value)
123123

124124

125-
# {'foo': {'bar-1': 'missing', 'bar-2': 'new'}}
126125
def parse_diff(jdiff_evaluate_response, actual, intended, match_config):
127-
"""Parse jdiff evaluate result into missing and extra dictionaries."""
128-
extra = {}
129-
missing = {}
126+
"""Parse jdiff evaluate result into missing and extra dictionaries.
127+
128+
Dict value in jdiff_evaluate_response can be:
129+
- 'missing' -> In the intended but missing from actual.
130+
- 'new' -> In the actual missing from intended.
131+
132+
Examples of jdiff_evaluate_response:
133+
- {'bar-2': 'missing', 'bar-1': 'new'}
134+
- {'hostname': {'new_value': 'veos-actual', 'old_value': 'veos-intended'}, 'domain-name': 'new'}
135+
- {'hostname': {'new_value': 'veos-0', 'old_value': 'veos'}, "index_element['ip name']": 'missing', 'domain-name': 'new'}
136+
- {'servers': {'server': defaultdict(<class 'list'>, {'missing': [{'address': '1.us.pool.ntp.org', 'config': {'address': '1.us.pool.ntp.org'}, 'state': {'address': '1.us.pool.ntp.org'}}]})}}
137+
"""
138+
# Remove surrounding double quotes if present from jmespath/config-to-match match with - in the string.
139+
match_config = match_config.strip('"')
140+
extra = {} # In the actual missing from intended.
141+
missing = {} # In the intended but missing from actual.
130142

131143
def process_diff(_map, extra_map, missing_map, previous_key=None):
144+
"""Process the diff recursively."""
132145
for key, value in _map.items():
133-
print("value", value)
134-
print("type(value)", type(value))
135-
if isinstance(value, dict) and "new_value" in value and "old_value" in value:
136-
extra_map[key] = value["old_value"]
137-
missing_map[key] = value["new_value"]
146+
if isinstance(value, dict) and all(nested_key in value for nested_key in ("new_value", "old_value")):
147+
extra_map[key] = value["new_value"]
148+
missing_map[key] = value["old_value"]
138149
elif isinstance(value, str):
139-
if "missing" in value:
140-
print("missing", value)
141-
extra_map[key] = actual.get(match_config, {}).get(key)
142-
if "new" in value:
143-
print("new", value)
150+
if "missing" in value and "index_element" in key:
144151
key_chain, _ = _parse_index_element_string(key)
145-
new_value = reduce(getitem, key_chain, intended)
146-
set_nested_value(missing_map, key_chain[1::], new_value)
152+
if len(key_chain) == 1:
153+
missing_map[key_chain[0]] = intended.get(match_config, {}).get(key_chain[0])
154+
else:
155+
new_value = reduce(getitem, key_chain, intended)
156+
set_nested_value(extra_map, key_chain[1::], new_value)
157+
elif "missing" in value:
158+
missing_map[key] = intended.get(match_config, {}).get(key)
159+
else:
160+
if "new" in value:
161+
extra_map[key] = actual.get(match_config, {}).get(key)
147162
elif isinstance(value, defaultdict):
148-
if dict(value).get("new"):
149-
missing[previous_key][key] = dict(value).get("new", {})
150-
if dict(value).get("missing"):
151-
extra_map[previous_key][key] = dict(value).get("missing", {})
163+
value_dict = dict(value)
164+
if "new" in value_dict:
165+
extra_map[previous_key][key] = value_dict.get("new", {})
166+
if "missing" in value_dict:
167+
missing_map[previous_key][key] = value_dict.get("missing", {})
152168
elif isinstance(value, dict):
153169
extra_map[key] = {}
154170
missing_map[key] = {}
155-
process_diff(value, extra_map[key], missing_map[key], previous_key=key)
171+
process_diff(value, extra_map, missing_map, previous_key=key)
156172
return extra_map, missing_map
157173

158174
extras, missing = process_diff(jdiff_evaluate_response, extra, missing)
159175
# Don't like this, but with less the performant way of doing it right now it works to clear out
160176
# Any empty dicts that are left over from the diff.
161-
# This is a bit of a hack, but it works for now.
162177
final_extras = extras.copy()
163178
final_missing = missing.copy()
164179
for key, value in extras.items():

‎tests/test_diff_helpers.py‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import pytest
44

5+
from jdiff import extract_data_from_json
56
from jdiff.check_types import CheckType
67
from jdiff.utils.diff_helpers import (
78
_parse_index_element_string,
@@ -85,46 +86,46 @@ def test__parse_index_element_string(index_element, result):
8586
{"foo": {"bar-1": "baz1"}}, # actual
8687
{"foo": {"bar-2": "baz2"}}, # intended
8788
"foo", # match_config
88-
{"foo": {"bar-1": "baz1"}}, # extra
89-
{"foo": {"bar-2": "baz2"}}, # missing
89+
{"bar-1": "baz1"}, # extra
90+
{"bar-2": "baz2"}, # missing
9091
)
9192

9293
parse_diff_case_1 = (
93-
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}},
94-
{"openconfig-system:config": {"hostname": "veos"}},
95-
"openconfig-system:config",
96-
{"hostname": "veos-0", "domain-name": "ntc.com"},
97-
{"hostname": "veos"},
94+
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-actual"}}, # actual
95+
{"openconfig-system:config": {"hostname": "veos-intended"}}, # intended
96+
'"openconfig-system:config"', # match_config
97+
{"hostname": "veos-actual", "domain-name": "ntc.com"}, # extra
98+
{"hostname": "veos-intended"}, # missing
9899
)
99100

100101
parse_diff_case_2 = (
101-
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}},
102-
{"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}},
103-
"openconfig-system:config",
104-
{"domain-name": "ntc.com", "hostname": "veos-0"},
105-
{"hostname": "veos", "ip name": "ntc.com"},
102+
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}}, # actual
103+
{"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}}, # intended
104+
'"openconfig-system:config"', # match_config
105+
{"domain-name": "ntc.com", "hostname": "veos-0"}, # extra
106+
{"hostname": "veos", "ip name": "ntc.com"}, # missing
106107
)
107108

108109
parse_diff_case_3 = (
109110
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}},
110111
{"openconfig-system:config": {"ip name": "ntc.com"}},
111-
"openconfig-system:config",
112+
'"openconfig-system:config"',
112113
{"domain-name": "ntc.com", "hostname": "veos-0"},
113114
{"ip name": "ntc.com"},
114115
)
115116

116117
parse_diff_case_4 = (
117118
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos"}},
118119
{"openconfig-system:config": {"hostname": "veos"}},
119-
"openconfig-system:config",
120+
'"openconfig-system:config"',
120121
{"domain-name": "ntc.com"},
121122
{},
122123
)
123124

124125
parse_diff_case_5 = (
125126
{"openconfig-system:config": {"domain-name": "ntc.com", "hostname": "veos-0"}},
126127
{"openconfig-system:config": {"hostname": "veos", "ip name": "ntc.com"}},
127-
"openconfig-system:config",
128+
'"openconfig-system:config"',
128129
{"hostname": "veos-0", "domain-name": "ntc.com"},
129130
{"ip name": "ntc.com", "hostname": "veos"},
130131
)
@@ -144,7 +145,7 @@ def test__parse_index_element_string(index_element, result):
144145
}
145146
}
146147
},
147-
"openconfig-system:ntp",
148+
'"openconfig-system:ntp"',
148149
{},
149150
{
150151
"servers": {
@@ -181,7 +182,10 @@ def test__parse_index_element_string(index_element, result):
181182
def test_parse_diff(actual, intended, match_config, extra, missing): # pylint: disable=too-many-arguments
182183
"""Test that index_element can be unpacked."""
183184
jdiff_param_match = CheckType.create("exact_match")
184-
jdiff_evaluate_response, _ = jdiff_param_match.evaluate(actual, intended)
185+
extracted_actual = extract_data_from_json(actual, match_config)
186+
extracted_intended = extract_data_from_json(intended, match_config)
187+
jdiff_evaluate_response, _ = jdiff_param_match.evaluate(extracted_intended, extracted_actual)
188+
print("jdiff_evaluate_response", jdiff_evaluate_response)
185189

186190
parsed_extra, parsed_missing = parse_diff(
187191
jdiff_evaluate_response,

0 commit comments

Comments
 (0)