[wip] add domain upload and download features

Author: stitch

dashboard/internet_nl_dashboard/logic/domains.py

@@ -7,6 +7,7 @@
 from actstream import action
 from constance import config
 from django.db.models import Count, Prefetch
+from django.http import JsonResponse
 from django.utils import timezone
 from websecmap.organizations.models import Url
 from websecmap.scanners.models import Endpoint
@@ -18,6 +19,9 @@
                                                     determine_next_scan_moment)
 from dashboard.internet_nl_dashboard.scanners.scan_internet_nl_per_account import (initialize_scan,
                                                                                    update_state)
+import pyexcel as p
+
+from dashboard.internet_nl_dashboard.views.download_spreadsheet import create_spreadsheet_download
 
 log = logging.getLogger(__package__)
 
@@ -796,3 +800,25 @@ def delete_url_from_urllist(account: Account, urllist_id: int, url_id: int) -> b
     urllist.urls.remove(url_is_in_list)
 
     return True
+
+
+def download_as_spreadsheet(account: Account, urllist_id: int, file_type: str = "xlsx") -> Any:
+
+    urls = TaggedUrlInUrllist.objects.all().filter(
+        urllist__account=account,
+        urllist__pk=urllist_id
+    )
+
+    if not urls:
+        return JsonResponse({})
+
+    # results is a matrix / 2-d array / array with arrays.
+    data: List[List[Any]] = []
+    data += [["List(s)", "Domain(s)", "Tags"]]
+
+    for url in urls.all():
+        data += [[url.urllist.name, url.url.url, ", ".join(url.tags.values_list('name', flat=True))]]
+
+    book = p.get_book(bookdict={"Domains": data})
+
+    return create_spreadsheet_download("internet dashboard list", book, file_type)

dashboard/internet_nl_dashboard/migrations/0013_auto_20230302_1418.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.16 on 2023-03-02 14:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('internet_nl_dashboard', '0012_urllistreport_is_shared_on_homepage'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='urllist',
+            name='automatically_share_new_reports',
+            field=models.BooleanField(default=False, help_text='Sharing can be disabled and re-enabled where the report code and the share code (password) stay the same. Sharing means that all new reports will be made public under a set of standard urls.'),
+        ),
+        migrations.AddField(
+            model_name='urllist',
+            name='default_public_share_code_for_new_reports',
+            field=models.CharField(blank=True, default='', help_text='An unencrypted share code that can be seen by all users in an account. Can be modified by all. New reports get this code set automatically. You can change this per report. An empty field means no share code and the report is accessible publicly.', max_length=64),
+        ),
+        migrations.AddField(
+            model_name='urllist',
+            name='enable_report_sharing_page',
+            field=models.BooleanField(default=False, help_text='When true there will be page under the list-id that shows all reports that are shared publicly.'),
+        ),
+    ]

dashboard/internet_nl_dashboard/models.py

@@ -226,6 +226,32 @@ class UrlList(models.Model):
         blank=True
     )
 
+    enable_report_sharing_page = models.BooleanField(
+        default=False,
+        help_text="When true there will be page under the list-id that shows all reports that are shared publicly."
+    )
+
+    # will be available under: /public/account-id/list-id/latest
+    # will be available under: /public/account-id/list-id/list (for a list of public reports for this list)
+    #   and                    /public/account-id/list-id/report-id
+    #   and                    /public/account-id/list-name-slug/latest
+    #   and                    /public/account-id/list-name-slug/report-id
+    automatically_share_new_reports = models.BooleanField(
+        help_text="Sharing can be disabled and re-enabled where the report code and the share code (password) "
+                  "stay the same. Sharing means that all new reports will be made public under a set of standard urls.",
+        default=False
+    )
+
+    default_public_share_code_for_new_reports = models.CharField(
+        max_length=64,
+        help_text="An unencrypted share code that can be seen by all users in an account. Can be modified by all. "
+                  "New reports get this code set automatically. You can change this per report. An empty field "
+                  "means no share code and the report is accessible publicly.",
+        blank=True,
+        default=""
+    )
+
+
     def __str__(self):
         return "%s/%s" % (self.account, self.name)
 

dashboard/internet_nl_dashboard/urls.py

@@ -50,6 +50,7 @@ def to_url(value):
     path('data/urllist/url/save/', domains.alter_url_in_urllist_),
     path('data/urllist/url/add/', domains.add_urls_to_urllist),
     path('data/urllist/url/delete/', domains.delete_url_from_urllist_),
+    path('data/urllist/download/', domains.download_list_),
 
     path('data/urllist/tag/add/', tags.add_tag_),
     path('data/urllist/tag/remove/', tags.remove_tag_),

dashboard/internet_nl_dashboard/views/domains.py

@@ -3,6 +3,7 @@
 
 from django.contrib.auth.decorators import login_required
 from django.http import JsonResponse
+from django.views.decorators.http import require_http_methods
 from websecmap.app.common import JSEncoder
 
 from dashboard.internet_nl_dashboard.logic.domains import (alter_url_in_urllist, cancel_scan,
@@ -13,7 +14,7 @@
                                                            get_urllists_from_account,
                                                            save_urllist_content,
                                                            save_urllist_content_by_name, scan_now,
-                                                           update_list_settings)
+                                                           update_list_settings, download_as_spreadsheet)
 from dashboard.internet_nl_dashboard.views import LOGIN_URL, get_account, get_json_body
 
 
@@ -81,3 +82,11 @@ def cancel_scan_(request):
     request = get_json_body(request)
     response = cancel_scan(account, request.get('id'))
     return JsonResponse(response)
+
+
+@login_required(login_url=LOGIN_URL)
+@require_http_methods(["POST"])
+def download_list_(request):
+    params = get_json_body(request)
+    return download_as_spreadsheet(get_account(request), params.get('list-id', None), params.get('file-type', None))
+

dashboard/internet_nl_dashboard/views/download_spreadsheet.py

@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 import logging
+from typing import Any
 
 import django_excel as excel
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse, JsonResponse
 from django.utils.text import slugify
-from websecmap.app.common import JSEncoder
 
 from dashboard.internet_nl_dashboard.logic.report_to_spreadsheet import (create_spreadsheet,
                                                                          upgrade_excel_spreadsheet)
@@ -20,30 +20,39 @@ def download_spreadsheet(request, report_id, file_type) -> HttpResponse:
 
     filename, spreadsheet = create_spreadsheet(account=account, report_id=report_id)
 
-    if not spreadsheet:
-        return JsonResponse({}, encoder=JSEncoder)
-
     if file_type == "xlsx":
         # todo: requesting infinite files will flood the system as temp files are saved. Probably load file into
         #   memory and then remove the original file. With the current group of users the risk is minimal, so no bother
-        tmp_file_handle = upgrade_excel_spreadsheet(spreadsheet)
-        with open(tmp_file_handle.name, 'rb') as file_handle:
-            response = HttpResponse(file_handle.read(),
-                                    content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
-            response["Content-Disposition"] = f"attachment; filename={slugify(filename)}.xlsx"
-        return response
-
-    if file_type == "ods":
-        output: HttpResponse = excel.make_response(spreadsheet, file_type)
-        output["Content-Disposition"] = f"attachment; filename={slugify(filename)}.ods"
-        output["Content-type"] = "application/vnd.oasis.opendocument.spreadsheet"
-        return output
-
-    if file_type == "csv":
-        output = excel.make_response(spreadsheet, file_type)
-        output["Content-Disposition"] = f"attachment; filename={slugify(filename)}.csv"
-        output["Content-type"] = "text/csv"
-        return output
-
-    # anything that is not valid at all.
-    return JsonResponse({}, encoder=JSEncoder)
+
+        # Upgrading happens with openpyxl which supports formulas. You cannot open those files with django_excel as
+        # that does _not_ understand formulas and will simply delete them.
+        file_type = "xlsx-openpyxl"
+        spreadsheet = upgrade_excel_spreadsheet(spreadsheet)
+
+    return create_spreadsheet_download(filename, spreadsheet, file_type)
+
+
+def create_spreadsheet_download(file_name: str, spreadsheet_data: Any, file_type: str = "xlsx") -> HttpResponse:
+
+    if file_type not in ["xlsx", "ods", "csv", "xlsx-openpyxl"] or not spreadsheet_data or not file_name:
+        return JsonResponse({})
+
+    content_types = {
+        "csv": "text/csv",
+        "ods": "application/vnd.oasis.opendocument.spreadsheet",
+        "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "xlsx-openpyxl": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    }
+
+    if file_type == "xlsx-openpyxl":
+        with open(spreadsheet_data.name, 'rb') as file_handle:
+            output = HttpResponse(file_handle.read())
+        file_type = "xlsx"
+    else:
+        # Simple xls files and such
+        output: HttpResponse = excel.make_response(spreadsheet_data, file_type)
+
+    output["Content-Disposition"] = f"attachment; filename={slugify(file_name)}.{file_type}"
+    output["Content-type"] = content_types[file_type]
+
+    return output