Merge pull request #261 from the-deep/hotfix-generate-preview-thumbnail

Hotfix generate preview thumbnail

Author: AdityaKhatri

apps/bulk_data_migration/management/commands/generate_preview.py

@@ -0,0 +1,21 @@
+from django.core.management.base import BaseCommand
+
+from lead.tasks import generate_previews
+
+
+class Command(BaseCommand):
+    help = 'Extract preview/images from leads'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--lead_id',
+            nargs='+',
+            type=int,
+            help='List of lead ids'
+        )
+
+    def handle(self, *args, **options):
+        if options['lead_id']:
+            generate_previews.delay(options['lead_id'])
+        else:
+            generate_previews.delay()

apps/connector/sources/base.py

@@ -2,6 +2,8 @@
 
 
 class Source(ABC):
+    DEFAULT_PER_PAGE = 25
+
     def __init__(self):
         if not hasattr(self, 'title') \
                 or not hasattr(self, 'key') \
@@ -12,9 +14,16 @@ def __init__(self):
     def fetch(params, page=None, limit=None):
         pass
 
-    def query_leads(self, params):
+    def query_leads(self, params, limit=None, offset=None):
         from connector.serializers import SourceDataSerializer
+
+        if offset is None or offset < 0:
+            offset = 0
+        if not limit or limit < 0:
+            limit = Source.DEFAULT_PER_PAGE
+
+        data = self.fetch(params)[0]
         return SourceDataSerializer(
-            self.fetch(params)[0],
+            data[offset:offset + limit],
             many=True,
         ).data

apps/connector/sources/rss_feed.py

@@ -66,11 +66,7 @@ def fetch(self, params, offset=None, limit=None):
         url_field = params.get('url-field')
         website_field = params.get('website-field')
 
-        for item in (
-                items[offset:offset + limit] if (
-                    offset is not None and limit is not None
-                ) else items
-        ):
+        for item in items:
             def get_field(field):
                 if not field:
                     return ''
@@ -103,7 +99,7 @@ def query_options(self, params):
             options[field]['options'] = fields
         return options
 
-    def query_fields(self, params):
+    def query_fields(self, params, limit=None, offset=None):
         if not params or not params.get('feed-url'):
             return []
 
@@ -145,4 +141,8 @@ def replace_ns(tag):
             if fields.count(field) == 1:
                 real_fields.append(field)
 
-        return real_fields
+        if offset is None or offset < 0:
+            offset = 1
+        if not limit or limit < 0:
+            limit = Source.DEFAULT_PER_PAGE
+        return real_fields[offset:offset + limit]

apps/connector/views.py

@@ -10,6 +10,8 @@
 from rest_framework.decorators import action
 from deep.permissions import ModifyPermission
 from project.models import Project
+from utils.common import parse_number
+
 from .serializers import (
     SourceSerializer,
     SourceDataSerializer,
@@ -24,6 +26,7 @@
     ConnectorProject,
 )
 from .sources.store import source_store
+from .sources.base import Source
 
 
 class SourceViewSet(viewsets.ViewSet):
@@ -45,7 +48,13 @@ class SourceQueryView(views.APIView):
     def query(self, source_type, query, params):
         source = source_store[source_type]()
         method = getattr(source, 'query_{}'.format(query))
-        results = method(params)
+
+        query_params = self.request.query_params
+
+        limit = parse_number(query_params.get('limit'))
+        offset = parse_number(query_params.get('offset'))
+
+        results = method(params, limit, offset)
 
         if isinstance(results, list):
             return response.Response({
@@ -112,8 +121,8 @@ def get_leads(self, request, pk=None, version=None):
         project_id = request.data.pop('project', None)
         project = project_id and Project.objects.get(id=project_id)
 
-        offset = request.data.pop('offset', None)
-        limit = request.data.pop('limit', None)
+        offset = request.data.pop('offset', None) or 0
+        limit = request.data.pop('limit', None) or Source.DEFAULT_PER_PAGE
 
         params = {
             **(connector.params or {}),
@@ -122,6 +131,12 @@ def get_leads(self, request, pk=None, version=None):
 
         source = source_store[connector.source]()
         data, count = source.fetch(params, offset, limit)
+
+        # Paginate manually
+        # FIXME: Make this better: probably cache, and also optimize
+        # Because, right now, every data is pulled and then only paginated
+        data = data[offset:offset + limit]
+
         serializer = SourceDataSerializer(
             data,
             many=True,
@@ -131,7 +146,7 @@ def get_leads(self, request, pk=None, version=None):
 
         return response.Response({
             'count': count,
-            'count_per_page': getattr(source, 'count_per_page', None),
+            'count_per_page': limit,
             'results': results
         })
 

apps/lead/tasks.py

@@ -2,6 +2,7 @@
 # from channels import Group
 from django.core.files import File
 from django.db import transaction
+from django.db.models import Q
 # from django.utils import timezone
 from django.conf import settings
 from lead.models import (
@@ -16,7 +17,7 @@
 from utils.extractor.thumbnailers import DocThumbnailer
 # from utils.websocket.subscription import SubscriptionConsumer
 
-# import json
+import time
 import reversion
 import os
 import re
@@ -192,7 +193,7 @@ def send_lead_text_to_deepl(self, lead_id):
         preview.classified_doc_id = classified_doc_id
         preview.save()
         return True
-    except Exception as e:
+    except Exception:
         # Retry with exponential decay
         logger.warning("Error while sending request to deepl. {}".format(
             traceback.format_exc()))
@@ -216,7 +217,7 @@ def extract_from_lead(lead_id):
     # and try to prevent useless parallel extraction of same lead that
     # that might happen.
     key = 'lead_extraction_{}'.format(lead_id)
-    lock = redis.get_lock(key, 60 * 60 * 4)  # Lock lifetime 4 hours
+    lock = redis.get_lock(key, 60 * 60 * 0.5)  # Lock lifetime half hours
     have_lock = lock.acquire(blocking=False)
     if not have_lock:
         return False
@@ -252,3 +253,16 @@ def extract_from_lead(lead_id):
 
     lock.release()
     return return_value
+
+
+@shared_task
+def generate_previews(lead_ids=None):
+    """Generae previews of leads which do not have preview"""
+    lead_ids = lead_ids or Lead.objects.filter(
+        Q(leadpreview__isnull=True) |
+        Q(leadpreview__text_extract=''),
+    ).values_list('id', flat=True)
+
+    for lead_id in lead_ids:
+        extract_from_lead.s(lead_id).delay()
+        time.sleep(0.5)

deep/documents_types.py

@@ -7,6 +7,9 @@
     'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
     'application/wps-office.docx',
 ]
+MSWORD_MIME_TYPES = [
+    'application/msword', 'application/wps-office.doc',
+]
 POWERPOINT_MIME_TYPES = [
     'application/vnd.openxmlformats-officedocument.presentationml.presentation', # noqa
     'application/vnd.ms-powerpoint',
@@ -22,14 +25,15 @@
 # Overall Supported Mime Types
 DEEP_SUPPORTED_MIME_TYPES = [
     'application/rtf', 'text/plain', 'font/otf', 'text/csv',
-    'application/json', 'application/xml', 'application/msword',
+    'application/json', 'application/xml',
 ] + (
-    DOCX_MIME_TYPES + PDF_MIME_TYPES + POWERPOINT_MIME_TYPES +
-    SHEET_MIME_TYPES + ODS_MIME_TYPES + IMAGE_MIME_TYPES
+    DOCX_MIME_TYPES + MSWORD_MIME_TYPES + PDF_MIME_TYPES +
+    POWERPOINT_MIME_TYPES + SHEET_MIME_TYPES + ODS_MIME_TYPES +
+    IMAGE_MIME_TYPES
 )
 
 DEEP_SUPPORTED_EXTENSIONS = [
     'docx', 'xlsx', 'pdf', 'pptx',
     'json', 'png', 'jpg', 'jpeg', 'csv', 'txt',
-    'geojson', 'zip', 'ods',
+    'geojson', 'zip', 'ods', 'doc',
 ]

utils/common.py

@@ -95,7 +95,9 @@ def parse_time(time_str):
 
 
 def parse_number(num_str):
-    if not num_str:
+    try:
+        num = float(num_str)
+    except (ValueError, TypeError):
         return None
     num = float(num_str)
     if num == round(num):

utils/extractor/document.py

@@ -5,19 +5,22 @@
 PDF = 'pdf'
 DOCX = 'docx'
 PPTX = 'pptx'
+MSWORD = 'doc'
 
 EXTRACTORS = {
     HTML: extractors.HtmlExtractor,
     PDF: extractors.PdfExtractor,
     DOCX: extractors.DocxExtractor,
     PPTX: extractors.PptxExtractor,
+    MSWORD: extractors.MswordExtractor,
 }
 
 THUMBNAILERS = {
     HTML: thumbnailers.WebThumbnailer,
     PDF: thumbnailers.DocThumbnailer,
     DOCX: thumbnailers.DocThumbnailer,
     PPTX: thumbnailers.DocThumbnailer,
+    MSWORD: thumbnailers.DocThumbnailer,
 }
 
 

utils/extractor/extractors.py

@@ -3,7 +3,8 @@
 from .formats.pdf import process as pdf_extract
 from .formats.docx import (
     process as docx_extract,
-    pptx_process as pptx_extract
+    pptx_process as pptx_extract,
+    msword_process as msword_extract
 )
 
 
@@ -69,3 +70,11 @@ class PptxExtractor(BaseExtractor):
     """
     ERROR_MSG = "Not a pptx document"
     EXTRACT_METHOD = pptx_extract
+
+
+class MswordExtractor(BaseExtractor):
+    """
+    Extractor class to extract msword documents.
+    """
+    ERROR_MSG = "Not a msword (.doc) document"
+    EXTRACT_METHOD = msword_extract

utils/extractor/file_document.py

@@ -1,7 +1,7 @@
 import os
 from .document import (
     Document,
-    HTML, PDF, DOCX, PPTX,
+    HTML, PDF, DOCX, PPTX, MSWORD,
 )
 
 
@@ -14,6 +14,7 @@ class FileDocument(Document):
     HTML_TYPES = ['.html', '.htm', '.txt']
     PDF_TYPES = ['.pdf', ]
     DOCX_TYPES = ['.docx', ]
+    MSWORD_TYPES = ['.doc', ]
     PPTX_TYPES = ['.pptx', ]
 
     def __init__(self, file, name):
@@ -28,6 +29,8 @@ def __init__(self, file, name):
             type = HTML
         elif extension in self.DOCX_TYPES:
             type = DOCX
+        elif extension in self.MSWORD_TYPES:
+            type = MSWORD
         elif extension in self.PPTX_TYPES:
             type = PPTX