the-deep · ics-yamen · Nov 16, 2024
diff --git a/.gitignore b/.gitignore
@@ -2,11 +2,12 @@
 
 .venv
 .vscode/
+.idea/
 # dependencies
 /node_modules
 vendor/
 src/public/
-
+/media/
 celery-worker.state
 
 # testing

diff --git a/.vscode/settings.json b/.vscode/settings.json
diff --git a/Dockerfile b/Dockerfile
@@ -1,49 +1,50 @@
-FROM python:3.8-slim-buster as base
+FROM python:3.8-bullseye AS base
 
 LABEL maintainer="Deep Dev [email protected]"
-
 ENV PYTHONUNBUFFERED 1
 
 WORKDIR /code
 
+# Copy dependency files
 COPY pyproject.toml poetry.lock /code/
 
+# Install required system dependencies
 RUN apt-get update -y \
     && apt-get install -y --no-install-recommends \
-        # Basic Packages
         iproute2 git vim \
-        # Build required packages
         gcc libc-dev libproj-dev \
-        # NOTE: procps: For pkill command
-        procps \
-        # Deep Required Packages
         wait-for-it binutils gdal-bin \
-    # Upgrade pip and install python packages for code
+        libcairo2 \
+        libpango1.0-dev \
+        libpangocairo-1.0-0 \
+        fonts-dejavu-core \
+        fonts-liberation \
     && pip install --upgrade --no-cache-dir pip poetry \
-    && poetry --version \
-    # Configure to use system instead of virtualenvs
     && poetry config virtualenvs.create false \
     && poetry install --no-root \
-    # Clean-up
-    && pip uninstall -y poetry virtualenv-clone virtualenv \
     && apt-get remove -y gcc libc-dev libproj-dev \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
+# Verify installation
+RUN pip install weasyprint==53.0
 
 # -------------------------- WEB ---------------------------------------
 FROM base AS web
 
+# Copy all project files
 COPY . /code/
 
 
 # -------------------------- WORKER ---------------------------------------
 FROM base AS worker
 
+# Additional worker-specific tools
 RUN apt-get update -y \
     && apt-get install -y --no-install-recommends \
         libreoffice \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*
 
+# Copy all project files
 COPY . /code/
diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py
@@ -581,6 +581,7 @@ def send_trigger_request_to_extractor(
             'request_type': NlpRequestType.USER if high_priority else NlpRequestType.SYSTEM,
         }
         response_content = None
+        print('the extraction endpoint is ', DeeplServiceEndpoint.DOCS_EXTRACTOR_ENDPOINT)
         try:
             response = requests.post(
                 DeeplServiceEndpoint.DOCS_EXTRACTOR_ENDPOINT,
@@ -610,6 +611,7 @@ def trigger_lead_extract(cls, lead, task_instance=None):
             return True
         # Get the lead to be extracted
         url_to_extract = None
+        print('is lead url', lead.url, "is lead attachment", lead.attachment)
         if lead.attachment:
             url_to_extract = generate_file_url_for_legacy_deepl_server(lead.attachment)
         elif lead.url:

diff --git a/apps/organization/views.py b/apps/organization/views.py
@@ -44,3 +44,5 @@ def get_queryset(self):
         if self.kwargs.get('pk'):
             return Organization.objects.prefetch_related('parent')
         return Organization.objects.filter(parent=None)
+
+
diff --git a/apps/static/image/graphQL-logo.svg b/apps/static/image/graphQL-logo.svg
diff --git a/apps/templates/connector/pdf.html b/apps/templates/connector/pdf.html
@@ -0,0 +1,79 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Data PDF</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            margin: 20px;
+            background-color: #f4f4f4; /* Light background for better contrast */
+        }
+        h1 {
+            text-align: center; /* Center the main heading */
+            color: #333;
+        }
+        .data-report {
+            max-width: 800px; /* Limit the width for better readability */
+            margin: 0 auto; /* Center the report */
+            background-color: #ffffff; /* White background for the report */
+            border-radius: 8px; /* Rounded corners for the report container */
+            box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); /* Subtle shadow for depth */
+            padding: 20px;
+        }
+        .entry {
+            margin-bottom: 20px;
+            padding: 15px;
+            border-radius: 8px;
+            background-color: #f9f9f9; /* Slightly different background for each entry */
+        }
+        .entry h3 {
+            margin-top: 0;
+            font-size: 20px; /* Increased font size for better visibility */
+            color: #555; /* Slightly darker color for the entry title */
+        }
+        .entry ul {
+            list-style-type: none;
+            padding: 0;
+        }
+        .entry li {
+            padding: 10px 0; /* Increased padding for each item */
+            display: flex;
+            justify-content: space-between;
+            border-bottom: 2px solid #ddd; /* Bold line between entries */
+        }
+        .entry li:last-child {
+            border-bottom: none; /* Remove border from the last item */
+        }
+        .key {
+            font-weight: bold;
+            color: #333;
+            flex-basis: 30%;
+        }
+        .value {
+            text-align: right;
+            flex-basis: 65%;
+            color: #666; /* Lighter color for the value text */
+        }
+    </style>
+</head>
+<body>
+    <h1>Data Report</h1>
+    <div class="data-report">
+        {% load filter %}  <!-- Ensure you have loaded your custom filters -->
+        {% for row in rows %}
+            <div class="entry">
+                <h3>Entry {{ forloop.counter }}</h3>
+                <ul>
+                    {% for key, value in columns|zipl:row %}
+                        <li>
+                            <div class="key">{{ key }}</div>
+                            <div class="value">{{ value }}</div>  <!-- Accessing the value directly -->
+                        </li>
+                    {% endfor %}
+                </ul>
+            </div>
+        {% endfor %}
+    </div>
+</body>
+</html>
diff --git a/apps/unified_connector/migrations/0011_alter_connectorsource_source.py b/apps/unified_connector/migrations/0011_alter_connectorsource_source.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2.25 on 2024-10-16 12:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('unified_connector', '0010_auto_20240625_0806'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='connectorsource',
+            name='source',
+            field=models.CharField(choices=[('atom-feed', 'Atom Feed'), ('relief-web', 'Relifweb'), ('rss-feed', 'RSS Feed'), ('unhcr-portal', 'UNHCR Portal'), ('humanitarian-resp', 'Humanitarian Response'), ('pdna', 'Post Disaster Needs Assessments'), ('emm', 'European Media Monitor'), ('kobo', 'KoboToolbox')], max_length=20),
+        ),
+    ]
diff --git a/apps/unified_connector/models.py b/apps/unified_connector/models.py
@@ -15,6 +15,7 @@
     humanitarian_response,
     pdna,
     emm,
+    kobo,
 )
 
 
@@ -120,6 +121,7 @@ class Source(models.TextChoices):
         HUMANITARIAN_RESP = 'humanitarian-resp', 'Humanitarian Response'
         PDNA = 'pdna', 'Post Disaster Needs Assessments'
         EMM = 'emm', 'European Media Monitor'
+        KOBO = 'kobo', 'KoboToolbox'
 
     class Status(models.IntegerChoices):
         PENDING = 0, 'Pending'
@@ -135,6 +137,7 @@ class Status(models.IntegerChoices):
         Source.HUMANITARIAN_RESP: humanitarian_response.HumanitarianResponse,
         Source.PDNA: pdna.PDNA,
         Source.EMM: emm.EMM,
+        Source.KOBO: kobo.Kobo,
     }
 
     title = models.CharField(max_length=255)
@@ -228,3 +231,4 @@ def update_aleady_added_using_lead(cls, lead, added=True):
             connector_lead=lead.connector_lead,
             source__unified_connector__project=lead.project,
         ).update(already_added=added)
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -44,3 +44,5 @@ def get_queryset(self):
		if self.kwargs.get('pk'):
		return Organization.objects.prefetch_related('parent')
		return Organization.objects.filter(parent=None)