Merge pull request #220 from the-deep/hotfix-xlsx-escape-1.2.4

[Hotfix] Don't escape character for xlsx

Author: Bibek Pandey

deep/settings.py

@@ -39,32 +39,8 @@
 
 # Application definition
 
-INSTALLED_APPS = [
-    # DJANGO APPS
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.messages',
-    'django.contrib.sessions',
-    'django.contrib.staticfiles',
-    'django.contrib.gis',
-    'django.contrib.postgres',
-
-    # LIBRARIES
-    'autofixture',
-    'channels',
-    'corsheaders',
-    'crispy_forms',
-    'django_filters',
-    'djangorestframework_camel_case',
-    'drf_dynamic_fields',
-    'rest_framework',
-    'reversion',
-    'storages',
-    'django_premailer',
-    'raven.contrib.django.raven_compat',
-
-    # DEEP APPS
+DEEP_APPS = [
+    # DEEP CORE APPS
     'analysis_framework',
     'ary',
     'category_editor',
@@ -93,6 +69,32 @@
     'redis_store',
 ]
 
+INSTALLED_APPS = [
+    # DJANGO APPS
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.messages',
+    'django.contrib.sessions',
+    'django.contrib.staticfiles',
+    'django.contrib.gis',
+    'django.contrib.postgres',
+
+    # LIBRARIES
+    'autofixture',
+    'channels',
+    'corsheaders',
+    'crispy_forms',
+    'django_filters',
+    'djangorestframework_camel_case',
+    'drf_dynamic_fields',
+    'rest_framework',
+    'reversion',
+    'storages',
+    'django_premailer',
+    'raven.contrib.django.raven_compat',
+] + DEEP_APPS
+
 MIDDLEWARE = [
     'django.middleware.security.SecurityMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
@@ -294,6 +296,7 @@ def add_username_attribute(record):
     """
     Append username(email) to logs
     """
+    record.username = ''
     if hasattr(record, 'request'):
         if hasattr(record.request, 'user') and\
                 not record.request.user.is_anonymous():
@@ -315,9 +318,11 @@ def add_username_attribute(record):
         },
         'formatters': {
             'simple': {
-                'format': '%(asctime)s ' + os.environ.get('EBS_HOSTNAME', '') +
-                          ' DJANGO-' + os.environ.get('EBS_ENV_TYPE', '') +
-                          ': %(username)s %(message)s',
+                'format':
+                '%(asctime)s {} DJANGO-{} : %(username)s %(message)s'.format(
+                    os.environ.get('EBS_HOSTNAME', ''),
+                    os.environ.get('EBS_ENV_TYPE', '')
+                ),
                 'datefmt': '%Y-%m-%dT%H:%M:%S',
             },
         },
@@ -327,8 +332,10 @@ def add_username_attribute(record):
                 'class': 'logging.handlers.SysLogHandler',
                 'filters': ['add_username_attribute'],
                 'formatter': 'simple',
-                'address': (os.environ.get('PAPERTRAIL_HOST'),
-                            int(os.environ.get('PAPERTRAIL_PORT')))
+                'address': (
+                    os.environ.get('PAPERTRAIL_HOST'),
+                    int(os.environ.get('PAPERTRAIL_PORT')),
+                )
             },
         },
         'loggers': {
@@ -344,6 +351,16 @@ def add_username_attribute(record):
                 'handlers': ['SysLog'],
                 'propagate': True,
             },
+            **{
+                # TODO: move all apps to sub directory deep
+                # and use deep as logger
+                app: {
+                    'level': 'DEBUG',
+                    'handlers': ['SysLog'],
+                    'propagate': True,
+                }
+                for app in DEEP_APPS
+            }
         },
     }
 

export/formats/xlsx.py

@@ -4,13 +4,17 @@
 from openpyxl.writer.excel import save_virtual_workbook
 
 from utils.common import (
-    get_valid_xml_string as xstr,
+    get_valid_xml_string,
     parse_date,
     parse_time,
     parse_number,
 )
 
 
+def xstr(string):
+    return get_valid_xml_string(string, escape=False)
+
+
 class WorkBook:
     """
     An xlsx workbook

utils/common.py

@@ -1,4 +1,4 @@
-from xml.sax.saxutils import escape
+from xml.sax.saxutils import escape as xml_escape
 from datetime import timedelta, datetime
 from django.conf import settings
 
@@ -63,9 +63,9 @@ def is_valid_xml_char_ordinal(c):
     )
 
 
-def get_valid_xml_string(string):
+def get_valid_xml_string(string, escape=True):
     if string:
-        s = escape(string)
+        s = xml_escape(string) if escape else string
         return ''.join(c for c in s
                        if is_valid_xml_char_ordinal(c))
     return ''

utils/extractor/formats/html.py

@@ -2,6 +2,7 @@
 from urllib.parse import urljoin
 
 import logging
+import base64
 import traceback
 import re
 import requests
@@ -44,9 +45,15 @@ def process(doc, url):
         try:
             fp = tempfile.NamedTemporaryFile(dir=settings.BASE_DIR)
             img_src = urljoin(url, img.get('src'))
-            r = requests.get(img_src, stream=True)
-            write_file(r, fp)
+            if re.search(r'http[s]?://', img_src):
+                r = requests.get(img_src, verify=False, stream=True)
+                write_file(r, fp)
+            else:
+                image = base64.b64decode(img_src.split(',')[1])
+                fp.write(image)
             images.append(fp)
+        except requests.exceptions.ConnectionError:
+            logger.warn(traceback.format_exc())
         except Exception:
             logger.error(traceback.format_exc())
 

utils/extractor/web_document.py

@@ -31,7 +31,7 @@ def __init__(self, url):
             r = requests.head(url, headers=DEFAULT_HEADERS)
         except requests.exceptions.RequestException:
             # If we can't get header, assume html and try to continue.
-            r = requests.get(url, headers=DEFAULT_HEADERS)
+            r = requests.get(url, verify=False, headers=DEFAULT_HEADERS)
             doc = r.content
             super().__init__(doc, type, params=params)
             return