aligntment: convert badges to css, use webassets only, user iUploader…

… interface

ckanext/validation/helpers.py

@@ -1,61 +1,96 @@
 # encoding: utf-8
 import json
 
-from ckan.lib.helpers import url_for_static
+from six.moves.urllib.parse import urlparse
+from six import string_types
 from ckantoolkit import url_for, _, config, asbool, literal, h
 
+from ckanext.validation.utils import get_default_schema
+
 def get_helpers():
     validators = (
         get_validation_badge,
         validation_extract_report_from_errors,
         dump_json_value,
         bootstrap_version,
-        validation_dict,
-        use_webassets
+        validation_hide_source,
+        is_url_valid
     )
 
     return {"{}".format(func.__name__): func for func in validators}
 
 
 def get_validation_badge(resource, in_listing=False):
 
+    afterDate = config.get('ckanext.validation.show_badges_after_last_modified_date', "")
+    if afterDate and (not resource.get('last_modified')
+                      or h.date_str_to_datetime(afterDate)
+                      >= h.date_str_to_datetime(resource['last_modified'])):
+        return ''
+
     if in_listing and not asbool(
             config.get('ckanext.validation.show_badges_in_listings', True)):
         return ''
 
     if not resource.get('validation_status'):
         return ''
 
+    if not _get_schema_or_default_schema(resource):
+        return ''
+
+    statuses = {
+        'success': _('valid'),
+        'failure': _('invalid'),
+        'invalid': _('invalid'),
+        'error': _('error'),
+        'unknown': _('unknown'),
+    }
+
     messages = {
         'success': _('Valid data'),
         'failure': _('Invalid data'),
+        'invalid': _('invalid data'),
         'error': _('Error during validation'),
         'unknown': _('Data validation unknown'),
     }
 
-    if resource['validation_status'] in ['success', 'failure', 'error']:
+    if resource['validation_status'] in ['success', 'failure', 'invalid', 'error']:
         status = resource['validation_status']
     else:
         status = 'unknown'
 
+    action = 'validation.read'
+
     validation_url = url_for(
-        'validation_read',
+        action,
         id=resource['package_id'],
         resource_id=resource['id'])
 
-    badge_url = url_for_static(
-        '/images/badges/data-{}-flat.svg'.format(status))
-
-    return '''
-<a href="{validation_url}" class="validation-badge">
-    <img src="{badge_url}" alt="{alt}" title="{title}"/>
+    return u'''
+<a href="{validation_url}" class="validation-badge" title="{alt} {title}">
+    <span class="prefix">{prefix}</span><span class="status {status}">{status_title}</span>
 </a>'''.format(
         validation_url=validation_url,
-        badge_url=badge_url,
+        prefix=_('data'),
+        status=status,
+        status_title=statuses[status],
         alt=messages[status],
         title=resource.get('validation_timestamp', ''))
 
 
+def _get_schema_or_default_schema(resource):
+
+    if asbool(resource.get('align_default_schema')):
+        schema = get_default_schema(resource['package_id'])
+    else:
+        schema = resource.get('schema')
+
+    if schema and isinstance(schema, string_types):
+        schema = schema if is_url_valid(schema) else json.loads(schema)
+
+    return schema
+
+
 def validation_extract_report_from_errors(errors):
 
     report = None
@@ -84,8 +119,6 @@ def validation_extract_report_from_errors(errors):
 
     return report, errors
 
-def validation_dict(validation_json):
-    return json.loads(validation_json)
 
 def dump_json_value(value, indent=None):
     """
@@ -109,5 +142,25 @@ def bootstrap_version():
         return '2'
 
 
-def use_webassets():
-    return int(h.ckan_version().split('.')[1]) >= 9
+def validation_hide_source(type):
+    """
+    Returns True if the given source type must be hidden on form.
+    Type is one of: upload, url or json.
+    For any unexpected type returns False
+    """
+    return asbool(config.get(
+        "ckanext.validation.form.hide_{}_source".format(type),
+    ))
+
+
+def is_url_valid(url):
+    """Basic checks for url validity"""
+    if not isinstance(url, string_types):
+        return False
+
+    try:
+        tokens = urlparse(url)
+    except ValueError:
+        return False
+
+    return all([getattr(tokens, attr) for attr in ('scheme', 'netloc')])

ckanext/validation/jobs.py

@@ -101,7 +101,7 @@ def run_validation_job(resource):
     report = _validate_table(source, _format=_format, schema=schema, **options)
 
     # Hide uploaded files
-    if type(report) == Report:
+    if isinstance(report, Report):
         report = report.to_dict()
 
     if 'tasks' in report:

ckanext/validation/logic/action.py

@@ -77,11 +77,11 @@ def resource_validation_run(context, data_dict):
     async_job = data_dict.get(u'async', True)
 
     # Ensure format is supported
-    if not resource.get(u'format', u'').lower() in settings.SUPPORTED_FORMATS:
+    if not resource.get(u'format', u'').lower() in settings.get_supported_formats():
         raise t.ValidationError(
             {u'format': u'Unsupported resource format.' +
              u'Must be one of {}'.format(
-                 u','.join(settings.SUPPORTED_FORMATS))})
+                 u','.join(settings.get_supported_formats()))})
 
     # Ensure there is a URL or file upload
     if not resource.get(u'url') and not resource.get(u'url_type') == u'upload':
@@ -281,7 +281,7 @@ def resource_validation_run_batch(context, data_dict):
                 for resource in dataset['resources']:
 
                     if (not resource.get(u'format', u'').lower()
-                            in settings.SUPPORTED_FORMATS):
+                            in settings.get_supported_formats()):
                         continue
 
                     try:
@@ -381,7 +381,7 @@ def _add_default_formats(search_data_dict):
 
     filter_formats = []
 
-    for _format in settings.DEFAULT_SUPPORTED_FORMATS:
+    for _format in settings.get_supported_formats():
         filter_formats.extend([_format, _format.upper()])
 
     filter_formats_query = ['+res_format:"{0}"'.format(_format)

ckanext/validation/plugin.py

@@ -59,7 +59,6 @@ def update_config(self, config_):
             log.debug(u'Validation tables exist')
 
         tk.add_template_directory(config_, u'templates')
-        tk.add_public_directory(config_, u'public')
         tk.add_resource(u'webassets', 'ckanext-validation')
 
     # IActions
@@ -141,7 +140,7 @@ def _handle_validation_for_resource(self, context, resource):
             ) and (
             # Make sure format is supported
             resource.get(u'format', u'').lower() in
-                s.SUPPORTED_FORMATS
+                s.get_supported_formats()
                 )):
             needs_validation = True
 
@@ -190,7 +189,7 @@ def before_resource_update(self, context, current_resource, updated_resource):
             ) and (
             # Make sure format is supported
             updated_resource.get(u'format', u'').lower() in
-                s.SUPPORTED_FORMATS
+                s.get_supported_formats()
                 )):
             needs_validation = True
 

ckanext/validation/settings.py

@@ -1,10 +1,65 @@
 # encoding: utf-8
+import json
+
+import ckantoolkit as tk
 
 from ckantoolkit import config
 
-# TODO: configurable
+
+try:
+    from tabulator.config import PARSERS
+except NameError:
+    # Point in time list of parsers from v1.53.5 if library Tabulator not loaded
+    PARSERS = {
+        'csv': 'tabulator.parsers.csv.CSVParser',
+        'datapackage': 'tabulator.parsers.datapackage.DataPackageParser',
+        'gsheet': 'tabulator.parsers.gsheet.GsheetParser',
+        'html': 'tabulator.parsers.html.HTMLTableParser',
+        'inline': 'tabulator.parsers.inline.InlineParser',
+        'json': 'tabulator.parsers.json.JSONParser',
+        'jsonl': 'tabulator.parsers.ndjson.NDJSONParser',
+        'ndjson': 'tabulator.parsers.ndjson.NDJSONParser',
+        'ods': 'tabulator.parsers.ods.ODSParser',
+        'sql': 'tabulator.parsers.sql.SQLParser',
+        'tsv': 'tabulator.parsers.tsv.TSVParser',
+        'xls': 'tabulator.parsers.xls.XLSParser',
+        'xlsx': 'tabulator.parsers.xlsx.XLSXParser',
+    }
+
+SUPPORTED_FORMATS_KEY = u"ckanext.validation.formats"
 DEFAULT_SUPPORTED_FORMATS = [u'csv', u'xls', u'xlsx']
+DEFAULT_VALIDATION_OPTIONS_KEY = "ckanext.validation.default_validation_options"
+
+PASS_AUTH_HEADER = u"ckanext.validation.pass_auth_header"
+PASS_AUTH_HEADER_DEFAULT = True
+
+PASS_AUTH_HEADER_VALUE = u"ckanext.validation.pass_auth_header_value"
+
+
+def get_default_validation_options():
+    """Return a default validation options
+
+    Returns:
+        dict[str, Any]: validation options dictionary
+    """
+    default_options = tk.config.get(DEFAULT_VALIDATION_OPTIONS_KEY)
+    return json.loads(default_options) if default_options else {}
+
+
+def get_supported_formats():
+    """Returns a list of supported formats to validate.
+    We use a tabulator to parse the file contents, so only those formats for
+    which a parser exists are supported
+
+    Returns:
+        list[str]: supported format list
+    """
+    supported_formats = [
+        _format.lower()
+        for _format in tk.aslist(tk.config.get(SUPPORTED_FORMATS_KEY))
+    ]
 
+    for _format in supported_formats:
+        assert _format in PARSERS, "Format {} is not supported".format(_format)
 
-SUPPORTED_FORMATS = config.get(
-    u'ckanext.validation.formats', DEFAULT_SUPPORTED_FORMATS)
+    return supported_formats or DEFAULT_SUPPORTED_FORMATS

ckanext/validation/templates/package/resource_read.html

@@ -6,11 +6,7 @@ <h1 class="page-heading">{{ h.resource_display_name(res) | truncate(50) }}
   {{ h.get_validation_badge(res)|safe }}
   </h1>
 
-  {% if h.use_webassets() %}
-      {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/validation-css' %}
-  {% else %}
-      {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/main' %}
-  {% endif %}
+  {% include 'validation/snippets/validation_report_asset.html' %}
 
 {% endblock %}
 

ckanext/validation/templates/package/snippets/resource_item.html

@@ -4,12 +4,5 @@
 {{ super() }}
 {{ h.get_validation_badge(res, in_listing=True)|safe }}
 
-  {% if h.use_webassets() %}
-      {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/validation-css' %}
-  {% else %}
-      {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/main' %}
-  {% endif %}
-
+{% include 'validation/snippets/validation_report_asset.html' %}
 {% endblock %}
-
-

ckanext/validation/templates/scheming/form_snippets/resource_schema.html

@@ -1,7 +1,7 @@
 {% import 'macros/form.html' as form %}
 
   {% set value = data[field.field_name] %}
-  {% set is_url = value and value[4:]|lower == 'http' %}
+  {% set is_url = h.is_url_valid(value) %}
   {% set is_json = not is_url and value %}
 
   <div class="image-upload"
@@ -11,6 +11,10 @@
        data-module-is_json="{{ 'true' if is_json else 'false' }}"
        data-module-field_url="schema_url"
        data-module-field_json="schema_json"
+       data-module-hide_upload="{{ h.dump_json(h.validation_hide_source("upload")) }}"
+       data-module-hide_url="{{ h.dump_json(h.validation_hide_source("url")) }}"
+       data-module-hide_json="{{ h.dump_json(h.validation_hide_source("json")) }}"
+       data-module-align_id="field-align_default_schema"
        >
     <div id="resource-schema-buttons">
       <label class="control-label">{{ _('Data Schema') }}</label>
@@ -60,10 +64,6 @@
     {% set existing_value = h.scheming_display_json_value(value, indent=None) if is_json else value %}
     <input type="hidden" id="field-schema" name="schema" value="{{ existing_value }}" />
 
-    {% if h.use_webassets() %}
-        {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/resource-schema-form' %}
-    {% else %}
-        {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/resource-schema-form' %}
-    {% endif %}
+    {% asset 'ckanext-validation/resource-schema-form' %}
 
   </div>

ckanext/validation/templates/scheming/snippets/errors.html

@@ -6,9 +6,9 @@
 
   {% if validation_report %}
     {% if h.bootstrap_version() == '3' %}
-      {% snippet 'validation/snippets/validation_report_dialog.html', validation_report=h.dump_json_value(validation_report) %}
+      {% snippet 'validation/snippets/validation_report_dialog.html', validation_report=validation_report %}
     {% else %}
-      {% snippet 'validation/snippets/validation_report_dialog_bs2.html', validation_report=h.dump_json_value(validation_report) %}
+      {% snippet 'validation/snippets/validation_report_dialog_bs2.html', validation_report=validation_report %}
     {% endif %}
   {% endif %}
 

ckanext/validation/templates/validation/snippets/validation_report_asset.html

@@ -0,0 +1,2 @@
+{% asset 'ckanext-validation/report-css' %}
+{% asset 'ckanext-validation/report-js' %}

ckanext/validation/templates/validation/snippets/validation_report_dialog.html

@@ -8,15 +8,10 @@ <h3>
           </h3>
       </div>
       <div class="modal-body">
-        <div id="report" data-module="validation-report" data-module-report="{{ validation_report }}"></div>
+        <div id="report" data-module="validation-report" data-module-report="{{ h.dump_json_value(validation_report) }}"></div>
       </div>
     </div>
   </div>
 </div>
 
-{% if h.use_webassets() %}
-    {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-form-css' %}
-    {% snippet 'validation/snippets/validation_asset.html', name='ckanext-validation/report-form-js' %}
-{% else %}
-    {% snippet 'validation/snippets/validation_resource.html', name='ckanext-validation/report-form' %}
-{% endif %}
+{% include 'validation/snippets/validation_report_asset.html' %}