Feature/process 410 gone revocation

.gitignore

@@ -8,3 +8,4 @@ build/
 .cache/
 README.rst
 .vscode/launch.json
+*.iml

openbadges/verifier/actions/utils.py

@@ -1,5 +1,21 @@
+import hashlib
+import six
 import uuid
 
 
 def generate_task_key():
     return uuid.uuid4().hex
+
+
+def generate_task_signature(task_name, *args):
+    content = b''
+    for arg in args:
+        try:
+            content += six.ensure_binary(str(arg))
+        except TypeError:
+            pass
+
+    return b'__'.join([
+        six.ensure_binary(task_name),
+        six.ensure_binary(hashlib.md5(content).hexdigest())
+    ]).decode('utf-8')

openbadges/verifier/tasks/__init__.py

@@ -3,7 +3,8 @@
 from .extensions import validate_extension_node, validate_single_extension
 from .images import validate_image
 from .input import detect_input_type, process_baked_resource
-from .graph import fetch_http_node, flatten_refetch_embedded_resource, intake_json, jsonld_compact_data
+from .graph import (fetch_http_node, flatten_refetch_embedded_resource, intake_json, jsonld_compact_data,
+                    process_410_gone)
 from .object_upgrades import upgrade_0_5_node, upgrade_1_0_node, upgrade_1_1_node
 from .validation import (assertion_timestamp_checks, assertion_verification_dependencies,
                          criteria_property_dependencies, detect_and_validate_node_class,
@@ -28,6 +29,7 @@
     IDENTITY_OBJECT_PROPERTY_DEPENDENCIES:     identity_object_property_dependencies,
     INTAKE_JSON:                               intake_json,
     ISSUER_PROPERTY_DEPENDENCIES:              issuer_property_dependencies,
+    PROCESS_410_GONE:                          process_410_gone,
     PROCESS_BAKED_RESOURCE:                    process_baked_resource,
     PROCESS_JWS_INPUT:                         process_jws_input,
     UPGRADE_0_5_NODE:                          upgrade_0_5_node,

openbadges/verifier/tasks/crypto.py

@@ -26,10 +26,11 @@ def process_jws_input(state, task_meta, **options):
     node_json = jws.get_unverified_claims(data).decode('utf-8')
     node_data = json.loads(node_json)
     node_id = task_meta.get('node_id', node_data.get('id'))
+    depth = task_meta.get('depth')
 
     actions = [
-        add_task(INTAKE_JSON, data=node_json, node_id=node_id),
-        add_task(VERIFY_JWS, node_id=node_id, data=data, prerequisites=SIGNING_KEY_FETCHED)
+        add_task(INTAKE_JSON, data=node_json, node_id=node_id, depth=depth),
+        add_task(VERIFY_JWS, node_id=node_id, data=data, prerequisites=SIGNING_KEY_FETCHED, depth=depth)
     ]
     if node_id:
         actions.append(set_validation_subject(node_id))
@@ -42,15 +43,16 @@ def verify_jws_signature(state, task_meta, **options):
         node_id = task_meta['node_id']
         key_node = get_node_by_path(state, [node_id, 'verification', 'creator'])
         public_pem = key_node['publicKeyPem']
+        depth = task_meta['depth']
     except (KeyError, IndexError,):
         raise TaskPrerequisitesError()
 
     actions = [
-        add_task(VERIFY_KEY_OWNERSHIP, node_id=node_id),
+        add_task(VERIFY_KEY_OWNERSHIP, node_id=node_id, depth=depth),
         add_task(
             VALIDATE_PROPERTY, node_path=[node_id, 'badge', 'issuer'], prop_name='revocationList',
             prop_type=ValueTypes.ID, expected_class=OBClasses.RevocationList, fetch=True, required=False,
-            prerequisites=[ISSUER_PROPERTY_DEPENDENCIES]
+            prerequisites=[ISSUER_PROPERTY_DEPENDENCIES], depth=depth
         ),
     ]
 
@@ -73,13 +75,17 @@ def verify_key_ownership(state, task_meta, **options):
         issuer_node = get_node_by_path(state, [node_id, 'badge', 'issuer'])
         key_node = get_node_by_path(state, [node_id, 'verification', 'creator'])
         key_id = key_node['id']
+        depth = task_meta['depth']
     except (KeyError, IndexError,):
         raise TaskPrerequisitesError()
 
     actions = []
     if issuer_node.get('revocationList'):
         actions.append(add_task(
-            VERIFY_SIGNED_ASSERTION_NOT_REVOKED, node_id=node_id, prerequisites=[VALIDATE_REVOCATIONLIST_ENTRIES]
+            VERIFY_SIGNED_ASSERTION_NOT_REVOKED,
+            node_id=node_id,
+            prerequisites=[VALIDATE_REVOCATIONLIST_ENTRIES],
+            depth=depth
         ))
 
     issuer_keys = list_of(issuer_node.get('publicKey'))

openbadges/verifier/tasks/graph.py

@@ -13,6 +13,7 @@
 from ..actions.input import store_original_resource
 from ..actions.validation_report import set_validation_subject
 from ..actions.tasks import add_task, delete_outdated_node_tasks, report_message
+from ..actions.utils import generate_task_signature
 from ..actions.validation_report import set_openbadges_version
 from ..exceptions import TaskPrerequisitesError
 from ..openbadges_context import OPENBADGES_CONTEXT_V2_URI
@@ -21,14 +22,15 @@
 from ..utils import list_of, jsonld_use_cache,make_string_from_bytes, MESSAGE_LEVEL_WARNING
 
 from .task_types import (DETECT_AND_VALIDATE_NODE_CLASS, FETCH_HTTP_NODE, INTAKE_JSON, JSONLD_COMPACT_DATA,
-                         PROCESS_BAKED_RESOURCE, UPGRADE_0_5_NODE, UPGRADE_1_0_NODE, UPGRADE_1_1_NODE,
+                         PROCESS_410_GONE, PROCESS_BAKED_RESOURCE, UPGRADE_0_5_NODE, UPGRADE_1_0_NODE, UPGRADE_1_1_NODE,
                          VALIDATE_EXPECTED_NODE_CLASS, VALIDATE_EXTENSION_NODE)
 from .utils import abbreviate_node_id as abv_node, filter_tasks, is_iri, is_url, task_result, URN_REGEX
 from .validation import OBClasses
 
 
 def fetch_http_node(state, task_meta, **options):
     url = task_meta['url']
+    depth = task_meta['depth']
 
     if options.get('cache_backend'):
         session = requests_cache.CachedSession(
@@ -39,9 +41,9 @@ def fetch_http_node(state, task_meta, **options):
     result = session.get(
         url, headers={'Accept': 'application/ld+json, application/json, image/png, image/svg+xml'}
     )
-
     try:
-        json.loads(result.text)
+        json_body = result.json()
+        response_text_with_proper_encoding = json.dumps(json_body)
     except ValueError:
         content_type = result.headers.get('Content-Type', 'UNKNOWN')
 
@@ -59,16 +61,27 @@ def fetch_http_node(state, task_meta, **options):
         b64content = b''.join([b'data:', parsed_type.encode(), b';base64,', base64.b64encode(result.content)])
         actions = [store_original_resource(node_id=url, data=b64content)]
         if task_meta.get('is_potential_baked_input', False):
-            actions += [add_task(PROCESS_BAKED_RESOURCE, node_id=url)]
+            actions += [add_task(PROCESS_BAKED_RESOURCE, node_id=url, depth=depth)]
 
         return task_result(
             True, 'Successfully fetched image from {}'.format(url), actions)
 
     actions = [
-        store_original_resource(node_id=url, data=result.text),
-        add_task(INTAKE_JSON, data=result.text, node_id=url,
+        store_original_resource(node_id=url, data=response_text_with_proper_encoding),
+        add_task(INTAKE_JSON,
+                 data=response_text_with_proper_encoding,
+                 node_id=url,
                  expected_class=task_meta.get('expected_class'),
-                 source_node_path=task_meta.get('source_node_path'))]
+                 source_node_path=task_meta.get('source_node_path'),
+                 depth=depth
+                 )
+    ]
+
+    if result.status_code == 410:
+        actions += [add_task(
+            PROCESS_410_GONE, node_id=url, prerequisites=[generate_task_signature(JSONLD_COMPACT_DATA, url)]
+        )]
+
     return task_result(message="Successfully fetched JSON data from {}".format(url), actions=actions)
 
 
@@ -91,6 +104,7 @@ def intake_json(state, task_meta, **options):
     input_data = task_meta['data']
     node_id = task_meta.get('node_id')
     expected_class = task_meta.get('expected_class')
+    depth = task_meta.get('depth')
     openbadges_version = None
     actions = []
 
@@ -104,8 +118,14 @@ def intake_json(state, task_meta, **options):
 
     if openbadges_version in ['1.1', '2.0']:
         compact_action = add_task(
-            JSONLD_COMPACT_DATA, node_id=node_id, openbadges_version=openbadges_version,
-            expected_class=expected_class, data=input_data, source_node_path=task_meta.get('source_node_path')
+            JSONLD_COMPACT_DATA,
+            task_key=generate_task_signature(JSONLD_COMPACT_DATA, node_id),
+            node_id=node_id,
+            openbadges_version=openbadges_version,
+            expected_class=expected_class,
+            data=input_data,
+            source_node_path=task_meta.get('source_node_path'),
+            depth=depth
         )
         actions.append(compact_action)
 
@@ -169,6 +189,7 @@ def jsonld_compact_data(state, task_meta, **options):
             data = data.decode()
         input_data = json.loads(data)
         expected_class = task_meta.get('expected_class')
+        depth = task_meta.get('depth')
     except TypeError:
         return task_result(False, "Could not load data")
 
@@ -184,7 +205,7 @@ def jsonld_compact_data(state, task_meta, **options):
 
     # Handle mismatch between URL node source and declared ID.
     if result.get('id') and task_meta.get('node_id') and result['id'] != task_meta['node_id']:
-        refetch_action = add_task(FETCH_HTTP_NODE, url=result['id'])
+        refetch_action = add_task(FETCH_HTTP_NODE, url=result['id'], depth=depth)
         if expected_class:
             refetch_action['expected_class'] = expected_class
         actions = [
@@ -213,10 +234,10 @@ def jsonld_compact_data(state, task_meta, **options):
     if expected_class:
         actions.append(
             add_task(VALIDATE_EXPECTED_NODE_CLASS, node_id=node_id,
-                     expected_class=expected_class)
+                     expected_class=expected_class, depth=depth)
         )
     elif task_meta.get('detectAndValidateClass', True):
-        actions.append(add_task(DETECT_AND_VALIDATE_NODE_CLASS, node_id=node_id))
+        actions.append(add_task(DETECT_AND_VALIDATE_NODE_CLASS, node_id=node_id, depth=depth))
 
     return task_result(
         True,
@@ -231,59 +252,89 @@ def flatten_refetch_embedded_resource(state, task_meta, **options):
         node = get_node_by_id(state, node_id)
         prop_name = task_meta['prop_name']
         node_class = task_meta['node_class']
+        depth = task_meta['depth']
     except (IndexError, KeyError):
         raise TaskPrerequisitesError()
 
     actions = []
     value = node.get(prop_name)
     if value is None:
-        return task_result(True, "Expected property {} was missing in node {}".format(node_id))
+        return task_result(True, "Expected property {} was missing in node {}".format(prop_name, node_id))
 
-    if isinstance(value, six.string_types):
+    if isinstance(value, six.string_types) or (isinstance(value, list) and all(isinstance(s, six.string_types) for s in value)):
         return task_result(
             True, "Property {} referenced from {} is not embedded in need of flattening".format(
                 prop_name, abv_node(node_id=node_id)
             ))
 
     if not isinstance(value, dict):
-        return task_result(
-            False, "Property {} referenced from {} is not a JSON object or string as expected".format(
-                prop_name, abv_node(node_id=node_id)
-            ))
-    embedded_node_id = value.get('id')
-
-    if embedded_node_id is None:
-        new_node = value.copy()
-        embedded_node_id = '_:{}'.format(uuid.uuid4())
-        new_node['id'] = embedded_node_id
-        new_node['@context'] = OPENBADGES_CONTEXT_V2_URI
-        actions.append(add_node(embedded_node_id, data=new_node))
-        actions.append(patch_node(node_id, {prop_name: embedded_node_id}))
-        actions.append(report_message(
-            'Node id missing at {}. A blank node ID has been assigned'.format(
-                abv_node(node_path=[node_id, prop_name], length=64)
-            ), message_level=MESSAGE_LEVEL_WARNING)
-        )
-    elif not isinstance(embedded_node_id, six.string_types) or not is_iri(embedded_node_id):
-        return task_result(False, "Embedded JSON object at {} has no proper assigned id.".format(
-            abv_node(node_path=[node_id, prop_name])))
+        if not isinstance(value, list):
+            return task_result(
+                False, "Property {} referenced from {} is not a JSON object, JSON object list, or string as expected".format(
+                    prop_name, abv_node(node_id=node_id)
+                ))
+    else:
+        value = [value]
+
+    for node in value:
+        embedded_node_id = node.get('id')
+
+        if embedded_node_id is None:
+            new_node = node.copy()
+            embedded_node_id = '_:{}'.format(uuid.uuid4())
+            new_node['id'] = embedded_node_id
+            new_node['@context'] = OPENBADGES_CONTEXT_V2_URI
+            actions.append(add_node(embedded_node_id, data=new_node))
+            actions.append(patch_node(node_id, {prop_name: embedded_node_id}))
+            actions.append(report_message(
+                'Node id missing at {}. A blank node ID has been assigned'.format(
+                    abv_node(node_path=[node_id, prop_name], length=64)
+                ), message_level=MESSAGE_LEVEL_WARNING)
+            )
+        elif not isinstance(embedded_node_id, six.string_types) or not is_iri(embedded_node_id):
+            return task_result(False, "Embedded JSON object at {} has no proper assigned id.".format(
+                abv_node(node_path=[node_id, prop_name])))
 
-    elif node_class == OBClasses.Assertion and not is_url(embedded_node_id):
+        elif node_class == OBClasses.Assertion and not is_url(embedded_node_id):
             if not re.match(URN_REGEX, embedded_node_id, re.IGNORECASE):
                 actions.append(report_message(
                     'ID format for {} at {} not in an expected HTTP or URN:UUID scheme'.format(
                         embedded_node_id, abv_node(node_path=[node_id, prop_name])
                     )))
-            new_node = value.copy()
+            new_node = node.copy()
             new_node['@context'] = OPENBADGES_CONTEXT_V2_URI
-            actions.append(add_node(embedded_node_id, data=value))
+            actions.append(add_node(embedded_node_id, data=node))
             actions.append(patch_node(node_id, {prop_name: embedded_node_id}))
 
-    else:
-        actions.append(patch_node(node_id, {prop_name: embedded_node_id}))
-        if not node_match_exists(state, embedded_node_id) and not filter_tasks(
+        else:
+            actions.append(patch_node(node_id, {prop_name: embedded_node_id}))
+            if not node_match_exists(state, embedded_node_id) and not filter_tasks(
                 state, node_id=embedded_node_id, task_type=FETCH_HTTP_NODE):
-            # fetch
-            actions.append(add_task(FETCH_HTTP_NODE, url=embedded_node_id))
+                # fetch
+                actions.append(add_task(FETCH_HTTP_NODE, url=embedded_node_id, depth=depth))
 
     return task_result(True, "Embedded {} node in {} queued for storage and/or refetching as needed", actions)
+
+
+def process_410_gone(state, task_meta, **options):
+    try:
+        node_id = task_meta['node_id']
+        node = get_node_by_id(state, node_id)
+    except (IndexError, KeyError):
+        raise TaskPrerequisitesError()
+
+    node_type = node.get('type')
+    if 'Assertion' not in list_of(node_type):
+        return task_result(
+            True, "Fetched resource returned 410 Gone status, but was not an Assertion, so not counted as revoked."
+        )
+
+    validation_subject = state['report'].get('validationSubject')
+    if validation_subject != node_id:
+        return task_result(True, "Fetched resource returned 410 Gone status, but was not the validation subject.")
+
+    return task_result(
+        False,
+        "Hosted Assertion is revoked, as hosted response status was 410 Gone. {}".format(abv_node(node_id=node_id))
+    )
+