diff --git a/openbadges/verifier/tasks/graph.py b/openbadges/verifier/tasks/graph.py index b185c31..ecef013 100644 --- a/openbadges/verifier/tasks/graph.py +++ b/openbadges/verifier/tasks/graph.py @@ -14,7 +14,7 @@ from ..openbadges_context import OPENBADGES_CONTEXT_V2_URI from ..reducers.graph import get_next_blank_node_id from ..state import get_node_by_id, node_match_exists -from ..utils import list_of, jsonld_use_cache,make_string_from_bytes, MESSAGE_LEVEL_WARNING +from ..utils import list_of, jsonld_use_cache,make_string_from_bytes, MESSAGE_LEVEL_WARNING,make_utf8 from .task_types import (DETECT_AND_VALIDATE_NODE_CLASS, FETCH_HTTP_NODE, INTAKE_JSON, JSONLD_COMPACT_DATA, UPGRADE_1_0_NODE, UPGRADE_1_1_NODE, VALIDATE_EXPECTED_NODE_CLASS, @@ -26,6 +26,8 @@ def fetch_http_node(state, task_meta, **options): url = task_meta['url'] + url = make_utf8(url) + if options.get('cache_backend'): session = requests_cache.CachedSession( backend=options['cache_backend'], expire_after=options.get('cache_expire_after', 300)) diff --git a/openbadges/verifier/utils.py b/openbadges/verifier/utils.py index a0f6ec8..83a4087 100644 --- a/openbadges/verifier/utils.py +++ b/openbadges/verifier/utils.py @@ -104,3 +104,12 @@ def make_string_from_bytes(input_value): if isinstance(input_value,bytes): return input_value.decode() return input_value + +def make_utf8(input_value): + """ + If input_value is unicode, it is returned as is. + If it's str, convert it to Unicode using UTF-8 encoding + """ + if sys.version[:3] < '3': + return input_value if isinstance(input_value, unicode) else input_value.decode('utf8') + return input_value \ No newline at end of file diff --git a/tests/test_graph_utf8.py b/tests/test_graph_utf8.py new file mode 100644 index 0000000..f376535 --- /dev/null +++ b/tests/test_graph_utf8.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- + +import json +import responses +import unittest + +from openbadges.verifier.actions.action_types import ADD_NODE, STORE_ORIGINAL_RESOURCE +from openbadges.verifier.actions.graph import add_node, patch_node, patch_node_reference +from openbadges.verifier.actions.tasks import add_task +from openbadges.verifier.reducers.graph import graph_reducer +from openbadges.verifier.state import get_node_by_id +from openbadges.verifier.tasks.graph import fetch_http_node, jsonld_compact_data +from openbadges.verifier.tasks import run_task +from openbadges.verifier.tasks.task_types import (DETECT_AND_VALIDATE_NODE_CLASS, FETCH_HTTP_NODE, INTAKE_JSON, + JSONLD_COMPACT_DATA) +from openbadges.verifier.openbadges_context import OPENBADGES_CONTEXT_V2_URI +from openbadges.verifier.utils import MESSAGE_LEVEL_WARNING,make_utf8 +from openbadges.verifier.verifier import verify + +from .utils import set_up_context_mock, set_up_image_mock + + + + +try: + from .testfiles.test_utf8_components import test_utf8_components +except (ImportError, SystemError): + from .testfiles.test_utf8_components import test_utf8_components + + + +class HttpFetchingUTF8Tests(unittest.TestCase): + + @responses.activate + def test_basic_http_fetch_task(self): + url = 'http://example.org/Краљ_Петар' + responses.add( + responses.GET, url, + body=test_utf8_components['2_0_basic_utf8_assertion'], + status=200, content_type='application/ld+json' + ) + task = add_task(FETCH_HTTP_NODE, url=url) + + import requests + resp = requests.get(url) + self.assertEqual(resp.status_code,200) + + + success, message, actions = fetch_http_node({}, task) + + self.assertTrue(success) + self.assertEqual(len(actions), 2) + self.assertEqual(actions[0]['type'], STORE_ORIGINAL_RESOURCE) + self.assertEqual(actions[1]['name'], INTAKE_JSON) + + diff --git a/tests/testfiles/test_utf8_components.py b/tests/testfiles/test_utf8_components.py new file mode 100644 index 0000000..7c8cfae --- /dev/null +++ b/tests/testfiles/test_utf8_components.py @@ -0,0 +1,329 @@ +# -*- coding: utf-8 -*- + +test_utf8_components = { +'1_0_basic_assertion': """{ + "uid":"123abc", + "recipient": {"identity": "test@example.com","hashed": false, "type": "email"}, + "badge": "http://a.com/badgeclass", + "issuedOn": "2015-04-30", + "verify": {"type": "hosted", "url": "http://a.com/instance"} +}""", +'1_0_basic_badgeclass': """{ + "name": "Basic Badge", + "description": "Basic as it gets. v1.0", + "image": "http://a.com/badgeclass_image", + "criteria": "http://a.com/badgeclass_criteria", + "issuer": "http://a.com/issuer" +}""", +'1_0_basic_issuer': """{ + "name": "Basic Issuer", + "url": "http://a.com/issuer" +}""", +'1_0_assertion_with_errors': """{ + "uid":"123abc", + "recipient": "test@example.com", + "badge": "http://a.com/badgeclass", + "issuedOn": "2015-04-30", + "verify": {"type": "hosted", "url": "http://a.com/instance"} +}""", +'0_5_assertion': """{ + "recipient": "test@example.com", + "badge": { + "version": "0.5.0", + "name": "Basic McBadge", + "image": "http://oldstyle.com/images/1", + "description": "A basic badge.", + "criteria": "http://oldsyle.com/criteria/1", + "issuer": { + "origin": "http://oldstyle.com", + "name": "Basic Issuer" + } + } +}""", +'0_5_1_assertion': """{ + "recipient": "sha256$85c4196c5516561cef673642157499b70066cb1070852b2a37fdbf3cc599b087", + "salt": "sel gris", + "issued_on": "2011-06-01", + "badge": { + "version": "0.5.0", + "name": "Basic McBadge", + "image": "http://oldstyle.com/images/2", + "description": "A basic badge.", + "criteria": "http://oldsyle.com/criteria/2", + "issuer": { + "origin": "http://oldstyle.com", + "name": "Basic Issuer" + } + } +}""", +'1_0_basic_assertion_with_extra_properties': """{ + "uid":"123abc", + "recipient": {"identity": "test@example.com","hashed": false, "type": "email"}, + "badge": "http://a.com/badgeclass", + "issuedOn": "2015-04-30", + "verify": {"type": "hosted", "url": "http://a.com/instance3"}, + "snood":"a very fun video game" +}""", +# Assertion awarded to nobody@example.org +'1_1_basic_assertion': """{ + "@context": "https://w3id.org/openbadges/v1", + "type": "Assertion", + "id": "https://example.org/beths-robotics-badge.json", + "recipient": { + "type": "email", + "hashed": true, + "salt": "deadsea", + "identity": "sha256$ecf5409f3f4b91ab60cc5ef4c02aef7032354375e70cf4d8e43f6a1d29891942" + }, + "image": "https://example.org/beths-robot-badge.png", + "evidence": "https://example.org/beths-robot-work.html", + "issuedOn": "2016-12-31T23:59:59Z", + "badge": "https://example.org/robotics-badge.json", + "verification": { + "type": "hosted" + } +}""", +'1_1_basic_badgeclass': """{ + "@context": "https://w3id.org/openbadges/v1", + "type": "BadgeClass", + "id": "https://example.org/robotics-badge.json", + "name": "Awesome Robotics Badge", + "description": "For doing awesome things with robots that people think is pretty great.", + "image": "https://example.org/robotics-badge.png", + "criteria": "https://example.org/robotics-badge.html", + "issuer": "https://example.org/organization.json" +}""", +'1_1_basic_issuer': """{ + "@context": "https://w3id.org/openbadges/v1", + "type": "Issuer", + "id": "https://example.org/organization.json", + "name": "An Example Badge Issuer", + "url": "https://example.org", + "email": "contact@example.org" +}""", +# Assertion awarded to nobody@example.org +'2_0_basic_assertion': """{ + "@context": "https://w3id.org/openbadges/v2", + "type": "Assertion", + "id": "https://example.org/beths-robotics-badge.json", + "recipient": { + "type": "email", + "hashed": true, + "salt": "deadsea", + "identity": "sha256$ecf5409f3f4b91ab60cc5ef4c02aef7032354375e70cf4d8e43f6a1d29891942" + }, + "image": "https://example.org/beths-robot-badge.png", + "evidence": "https://example.org/beths-robot-work.html", + "issuedOn": "2016-12-31T23:59:59Z", + "badge": "https://example.org/robotics-badge.json", + "verification": { + "type": "hosted" + } +}""", +'2_0_basic_badgeclass': """{ + "@context": "https://w3id.org/openbadges/v2", + "type": "BadgeClass", + "id": "https://example.org/robotics-badge.json", + "name": "Awesome Robotics Badge", + "description": "For doing awesome things with robots that people think is pretty great.", + "image": "https://example.org/robotics-badge.png", + "criteria": "https://example.org/robotics-badge.html", + "issuer": "https://example.org/organization.json" +}""", +# Assertion awarded to nobody@example.org +'2_0_basic_utf8_assertion': """{ + "@context": "https://w3id.org/openbadges/v2", + "type": "Assertion", + "id": "https://example.org/Краљ_Петар.json", + "recipient": { + "type": "email", + "hashed": true, + "salt": "deadsea", + "identity": "sha256$ecf5409f3f4b91ab60cc5ef4c02aef7032354375e70cf4d8e43f6a1d29891942" + }, + "image": "https://example.org/beths-robot-badge.png", + "evidence": "https://example.org/beths-robot-work.html", + "issuedOn": "2016-12-31T23:59:59Z", + "badge": "https://example.org/robotics-badge.json", + "verification": { + "type": "hosted" + } +}""", +'2_0_basic_utf8_badgeclass': """{ + "@context": "https://w3id.org/openbadges/v2", + "type": "BadgeClass", + "id": "https://example.org/robotics-badge.json", + "name": "Awesome Robotics Badge", + "description": "For doing awesome things with robots that people think is pretty great.", + "image": "https://example.org/robotics-badge.png", + "criteria": "https://example.org/robotics-badge.html", + "issuer": "https://example.org/organization.json" +}""", +'2_0_basic_issuer': """{ + "@context": "https://w3id.org/openbadges/v2", + "type": "Issuer", + "id": "https://example.org/organization.json", + "name": "An Example Badge Issuer", + "url": "https://example.org", + "email": "contact@example.org" +}""", +'openbadges_context': """ +{"@context": {"issuedOn": {"@id": "obi:issueDate", "@type": "xsd:dateTime"}, "AlignmentObject": "schema:AlignmentObject", "uid": {"@id": "obi:uid"}, "claim": {"@id": "cred:claim", "@type": "@id"}, "targetCode": {"@id": "obi:targetCode"}, "image": {"@id": "schema:image", "@type": "@id"}, "Endorsement": "cred:Credential", "Assertion": "obi:Assertion", "related": {"@id": "dc:relation", "@type": "@id"}, "evidence": {"@id": "obi:evidence", "@type": "@id"}, "sec": "https://w3id.org/security#", "Criteria": "obi:Criteria", "owner": {"@id": "sec:owner", "@type": "@id"}, "revocationList": {"@id": "obi:revocationList", "@type": "@id"}, "targetName": {"@id": "schema:targetName"}, "id": "@id", "alignment": {"@id": "obi:alignment", "@type": "@id"}, "allowedOrigins": {"@id": "obi:allowedOrigins"}, "Profile": "obi:Profile", "startsWith": {"@id": "http://purl.org/dqm-vocabulary/v1/dqm#startsWith"}, "author": {"@id": "schema:author", "@type": "@id"}, "FrameValidation": "obi:FrameValidation", "validationFrame": "obi:validationFrame", "creator": {"@id": "dc:creator", "@type": "@id"}, "validationSchema": "obi:validationSchema", "validatesType": "obi:validatesType", "version": {"@id": "schema:version"}, "BadgeClass": "obi:BadgeClass", "endorsement": {"@id": "cred:credential", "@type": "@id"}, "revocationReason": {"@id": "obi:revocationReason"}, "RevocationList": "obi:RevocationList", "issuer": {"@id": "obi:issuer", "@type": "@id"}, "type": "@type", "email": {"@id": "schema:email"}, "targetDescription": {"@id": "schema:targetDescription"}, "schema": "http://schema.org/", "targetUrl": {"@id": "schema:targetUrl"}, "criteria": {"@id": "obi:criteria", "@type": "@id"}, "verificationProperty": {"@id": "obi:verificationProperty"}, "description": {"@id": "schema:description"}, "Extension": "obi:Extension", "tags": {"@id": "schema:keywords"}, "CryptographicKey": "sec:Key", "expires": {"@id": "sec:expiration", "@type": "xsd:dateTime"}, "hosted": "obi:HostedBadge", "dc": "http://purl.org/dc/terms/", "telephone": {"@id": "schema:telephone"}, "publicKey": {"@id": "sec:publicKey", "@type": "@id"}, "badge": {"@id": "obi:badge", "@type": "@id"}, "endorsementComment": {"@id": "obi:endorsementComment"}, "genre": {"@id": "schema:genre"}, "hashed": {"@id": "obi:hashed", "@type": "xsd:boolean"}, "recipient": {"@id": "obi:recipient", "@type": "@id"}, "HostedBadge": "obi:HostedBadge", "identity": {"@id": "obi:identityHash"}, "revoked": {"@id": "obi:revoked", "@type": "xsd:boolean"}, "verify": "verification", "VerificationObject": "obi:VerificationObject", "name": {"@id": "schema:name"}, "publicKeyPem": {"@id": "sec:publicKeyPem"}, "obi": "https://w3id.org/openbadges#", "url": {"@id": "schema:url", "@type": "@id"}, "cred": "https://w3id.org/credentials#", "Image": "obi:Image", "created": {"@id": "dc:created", "@type": "xsd:dateTime"}, "IdentityObject": "obi:IdentityObject", "signed": "obi:SignedBadge", "Evidence": "obi:Evidence", "narrative": {"@id": "obi:narrative"}, "caption": {"@id": "schema:caption"}, "audience": {"@id": "obi:audience"}, "extensions": "https://w3id.org/openbadges/extensions#", "verification": {"@id": "obi:verify", "@type": "@id"}, "xsd": "http://www.w3.org/2001/XMLSchema#", "TypeValidation": "obi:TypeValidation", "revokedAssertions": {"@id": "obi:revoked"}, "SignedBadge": "obi:SignedBadge", "validation": "obi:validation", "salt": {"@id": "obi:salt"}, "targetFramework": {"@id": "schema:targetFramework"}, "Issuer": "obi:Issuer"}} +""", +'openbadges_context_v1': """{ + "@context": [ + { + "id": "@id", + "type": "@type", + + "ob": "https://w3id.org/openbadges#", + "dc": "http://purl.org/dc/terms/", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "sec": "https://w3id.org/security#", + "schema": "http://schema.org/", + "xsd": "http://www.w3.org/2001/XMLSchema#", + + "about": {"@id": "schema:about", "@type": "@id"}, + "alignment": {"@id": "ob:alignment", "@type": "@id"}, + "badge": {"@id": "ob:badge", "@type": "@id"}, + "badgeOffer": {"@id": "ob:badgeOffer", "@type": "@id"}, + "badgeTemplate": {"@id": "ob:badgeTemplate", "@type": "@id"}, + "criteria": {"@id": "ob:criteria", "@type": "@id"}, + "evidence": {"@id": "ob:evidence", "@type": "@id"}, + "issued": {"@id": "ob:issued", "@type": "xsd:dateTime"}, + "issuer": {"@id": "ob:issuer", "@type": "@id"}, + "recipient": {"@id": "ob:recipient", "@type": "@id"}, + "recipientEmail": "ob:recipientEmail", + "recipientPassword": "ob:recipientPassword", + "tag": "ob:tag", + "Identity": "ob:Identity", + "Badge": "ob:Badge", + "BadgeOffer": "ob:BadgeOffer", + "BadgeTemplate": "ob:BadgeTemplate", + + "address": {"@id": "schema:address", "@type": "@id"}, + "addressCountry": "schema:addressCountry", + "addressLocality": "schema:addressLocality", + "addressRegion": "schema:addressRegion", + "comment": "rdfs:comment", + "created": {"@id": "dc:created", "@type": "xsd:dateTime"}, + "creator": {"@id": "dc:creator", "@type": "@id"}, + "description": "schema:description", + "email": "schema:email", + "familyName": "schema:familyName", + "givenName": "schema:givenName", + "image": {"@id": "schema:image", "@type": "@id"}, + "label": "rdfs:label", + "name": "schema:name", + "postalCode": "schema:postalCode", + "streetAddress": "schema:streetAddress", + "title": "dc:title", + "url": {"@id": "schema:url", "@type": "@id"}, + "PostalAddress": "schema:PostalAddress", + + "identityService": {"@id": "https://w3id.org/identity#identityService", "@type": "@id"}, + + "credential": {"@id": "sec:credential", "@type": "@id"}, + "cipherAlgorithm": "sec:cipherAlgorithm", + "cipherData": "sec:cipherData", + "cipherKey": "sec:cipherKey", + "claim": {"@id": "sec:claim", "@type": "@id"}, + "digestAlgorithm": "sec:digestAlgorithm", + "digestValue": "sec:digestValue", + "domain": "sec:domain", + "expires": {"@id": "sec:expiration", "@type": "xsd:dateTime"}, + "initializationVector": "sec:initializationVector", + "nonce": "sec:nonce", + "normalizationAlgorithm": "sec:normalizationAlgorithm", + "owner": {"@id": "sec:owner", "@type": "@id"}, + "password": "sec:password", + "privateKey": {"@id": "sec:privateKey", "@type": "@id"}, + "privateKeyPem": "sec:privateKeyPem", + "publicKey": {"@id": "sec:publicKey", "@type": "@id"}, + "publicKeyPem": "sec:publicKeyPem", + "publicKeyService": {"@id": "sec:publicKeyService", "@type": "@id"}, + "revoked": {"@id": "sec:revoked", "@type": "xsd:dateTime"}, + "signature": "sec:signature", + "signatureAlgorithm": "sec:signatureAlgorithm", + "signatureValue": "sec:signatureValue", + "EncryptedMessage": "sec:EncryptedMessage", + "CryptographicKey": "sec:Key", + "GraphSignature2012": "sec:GraphSignature2012" + }, + { + "id": "@id", + "type": "@type", + + "obi": "https://w3id.org/openbadges#", + "extensions": "https://w3id.org/openbadges/extensions#", + "validation": "obi:validation", + + "xsd": "http://www.w3.org/2001/XMLSchema#", + "schema": "http://schema.org/", + "sec": "https://w3id.org/security#", + + "Assertion": "obi:Assertion", + "BadgeClass": "obi:BadgeClass", + "Issuer": "obi:Issuer", + "IssuerOrg": "obi:Issuer", + "Extension": "obi:Extension", + "hosted": "obi:HostedBadge", + "signed": "obi:SignedBadge", + "TypeValidation": "obi:TypeValidation", + "FrameValidation": "obi:FrameValidation", + + "name": { "@id": "schema:name" }, + "description": { "@id": "schema:description" }, + "url": { "@id": "schema:url", "@type": "@id" }, + "image": { "@id": "schema:image", "@type": "@id" }, + + "uid": { "@id": "obi:uid" }, + "recipient": { "@id": "obi:recipient", "@type": "@id" }, + "hashed": { "@id": "obi:hashed", "@type": "xsd:boolean" }, + "salt": { "@id": "obi:salt" }, + "identity": { "@id": "obi:identityHash" }, + "issuedOn": { "@id": "obi:issueDate", "@type": "xsd:dateTime" }, + "expires": { "@id": "sec:expiration", "@type": "xsd:dateTime" }, + "evidence": { "@id": "obi:evidence", "@type": "@id" }, + "verify": { "@id": "obi:verify", "@type": "@id" }, + + "badge": { "@id": "obi:badge", "@type": "@id" }, + "criteria": { "@id": "obi:criteria", "@type": "@id" }, + "tags": { "@id": "schema:keywords" }, + "alignment": { "@id": "obi:alignment", "@type": "@id" }, + + "issuer": { "@id": "obi:issuer", "@type": "@id" }, + "email": "schema:email", + "revocationList": { "@id": "obi:revocationList", "@type": "@id" }, + + "validatesType": "obi:validatesType", + "validationSchema": "obi:validationSchema", + "validationFrame": "obi:validationFrame" + }], + +"validation": [ + { + "type": "TypeValidation", + "validatesType": "Assertion", + "validationSchema": "https://openbadgespec.org/v1/schema/assertion.json" + }, + { + "type": "TypeValidation", + "validatesType": "BadgeClass", + "validationSchema": "https://openbadgespec.org/v1/schema/badgeclass.json" + }, + { + "type": "TypeValidation", + "validatesType": "Issuer", + "validationSchema": "https://openbadgespec.org/v1/schema/issuer.json" + }, + { + "type": "TypeValidation", + "validatesType": "Extension", + "validationSchema": "https://openbadgespec.org/v1/schema/extension.json" + } + ] +} +""" +}