Add possibility to load and dump data with superuser access

CHANGELOG.md

@@ -4,7 +4,7 @@
 
 ### Added
 
-- nothing added
+- Add possibility to load and dump data with superuser access
 
 ### Changed
 

concrete_datastore/admin/admin_site.py

@@ -157,6 +157,12 @@ def get_app_list(self, request):
         def index(self, request, extra_context=None, *args, **kwargs):
             if extra_context is None:
                 extra_context = {}
+            extra_context.update(
+                {
+                    'enable_db_dump': settings.ENABLE_DATABASE_DUMP,
+                    'enable_db_load': settings.ENABLE_DATABASE_LOAD,
+                }
+            )
             extra_context[
                 'display_datamodel'
             ] = settings.ENABLE_SERVE_DATAMODEL

concrete_datastore/concrete/templates/admin/index.html

@@ -1,17 +1,104 @@
 {% extends "admin/index.html" %}
 {% load i18n admin_urls static admin_list %}
+{% block extrahead %}
+<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
+<style type="text/css">
+    .custom-btn{
+        display: block;
+        text-decoration: none;
+        cursor: pointer;
+        background-color: #79aec8;
+        border: none;
+        color: #fff;
+        padding: 7px;
+        cursor: pointer;
+        border-radius: 4px;
+    }
+    .custom-btn:hover{
+        background-color: #79aec8;
+        text-decoration: none;
+        color: #fff;
+    }
+    .custom-btn:visited,.custom-btn:link{
+        background-color: #79aec8;
+        color: #fff;
+    }
+    .loader {
+        border: 16px solid #f3f3f3;
+        border-radius: 50%;
+        border-top: 16px solid #132452;
+        width: 120px;
+        height: 120px;
+        animation: spin 2s linear infinite;
+    }
+
+    @keyframes spin {
+        0% { transform: rotate(0deg); }
+        100% { transform: rotate(360deg); }
+    }
+</style>
+
+{% endblock %}
 {% block content %}
+<div id="loading-component" style="display: none;text-align: center;">
+    <h1>Loading data</h1>
+    <div class="loader" style="margin: 0 auto;"></div>
+</div>
 <div id="content-main"class="module">
-{% if display_datamodel %}
-<a href="{% url 'datamodel' action='view' %}">> Show datamodel</a>
-<br>
-<br>
-{% endif %}
-{% if use_core_automation %}
-<button class="btn btn-primary" style="background: #79aec8;padding: 10px 15px;border: none;border-radius: 4px;color: #fff;cursor: pointer;" onclick="window.location.href='{{target_admin_view}}';">Go to {{target_admin_view_name}}</button>
-<br>
-<br>
-{% endif %}
+    {% if use_core_automation %}
+        <a class="custom-btn" href="{{target_admin_view}}">Go to {{target_admin_view_name}}</a>
+    {% endif %}
+    {% if display_datamodel %}
+        <p><a href="{% url 'datamodel' action='view' %}"><u>Show datamodel</u></a></p>
+    {% endif %}
+    {% if enable_db_dump %}
+        <p><a href="{% url 'concrete:dump_data' %}" onclick="showWaitingDiv()"><u>Show JSON database dump</u></a><p>
+        <br>
+        <a onclick="donwloadDump()" href="{% url 'concrete:dump_data' %}?download=true" class="custom-btn"><i class="fa fa-download" style="padding: 0px 15px"></i>Download a database dump file</a>
+    {% endif %}
+    {% if enable_db_load %}
+
+        <div class="inputfile-box" style="position: relative; height: 3rem">
+            <form action="{% url 'concrete:load_data' %}" method="POST" enctype="multipart/form-data">
+                {% csrf_token %}
+              <input type="file" name="file" id="file" class="inputfile" onchange="submitAndWait(this.form)" style="display: none;">
+              <label for="file">
+                <p><a class="custom-btn"><i class="fa fa-upload" style="padding: 0px 15px"></i>Select a dump file to load into the database
+                </a></p>
+              </label>
+            </form>
+        </div>
+
+        <script type="text/javascript">
+            function showWaitingDiv() {
+                // Show waiting component
+                document.getElementById('loading-component').style.display = "block";
+                // Hide other contents
+                document.getElementById('content-main').style.display = "none";
+                document.getElementById('content-related').style.display = "none";
+            };
+            function hideWaitingDiv() {
+                // Show waiting component
+                document.getElementById('loading-component').style.display = "none";
+                // Hide other contents
+                document.getElementById('content-main').style.display = "block";
+                document.getElementById('content-related').style.display = "block";
+            };
+            function submitAndWait(form) {
+                form.submit();
+                showWaitingDiv();
+            };
+            async function donwloadDump() {
+                const test = await Navigator.StorageManager.getDirectory();
+                showWaitingDiv();
+                document.location.href="{% url 'concrete:dump_data' %}?download=true";
+                setInterval(()=>{console.log(test)}, 1000)
+                // hideWaitingDiv();
+            }
+        </script>
+        <br>
+    {% endif %}
+
 {% if app_list %}
     {% for models_group in app_list %}
         <div class="app-{{ models_group.app_label }} module">

concrete_datastore/concrete/templates/mainApp/datamodel.html

@@ -20,7 +20,7 @@ <h1>Datamodel</h1>
         </div>
         <div class="json-datamodel tabcontent" id="json" style="display: none">
             <div class="datamodel-buttons">
-                <button class="single-button" onclick="copyToClipboard('json')"><i class="fas fa-camera"></i>COPY</button>
+                <button class="single-button" onclick="copyToClipboard('json')"><i class="fa fa-camera"></i>COPY</button>
                 <a class="single-button" href="{% url 'datamodel' action='download' %}?data-format=json">DOWNLOAD</a>
             </div>
             <div class="meta-definition">

concrete_datastore/concrete/templates/mainApp/index.html

@@ -4,6 +4,7 @@
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <title>CONCRETE</title>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css">
     <style type="text/css" media="screen">
 * {
   margin: 0;

concrete_datastore/concrete/urls.py

@@ -1,10 +1,13 @@
 # coding: utf-8
 from django.urls import re_path
+from django.conf import settings
 
 from concrete_datastore.concrete.views import email_confirmation_view
 from concrete_datastore.concrete.views import unsubscribe_notifications_view
 from concrete_datastore.concrete.views import (
     unsubscribe_notifications_result_view,
+    dump_data,
+    load_data,
 )
 
 app_name = 'concrete_datastore.concrete'
@@ -26,3 +29,8 @@
         name='unsubscribe_notifications_result',
     ),
 ]
+
+if settings.ENABLE_DATABASE_DUMP:
+    urlpatterns.append(re_path(r'dump-data', dump_data, name="dump_data"))
+if settings.ENABLE_DATABASE_LOAD:
+    urlpatterns.append(re_path(r'load-data', load_data, name="load_data"))

concrete_datastore/concrete/views.py

@@ -1,9 +1,19 @@
 # coding: utf-8
 import json
 import logging
-
+import sys
+import os
+from io import StringIO
+from tempfile import NamedTemporaryFile
+from django.utils import timezone
+from django.core.management import call_command
 from django.conf import settings
-from django.http import HttpResponse
+from django.http import (
+    HttpResponse,
+    JsonResponse,
+    HttpResponseForbidden,
+    StreamingHttpResponse,
+)
 from django.shortcuts import get_object_or_404, render, redirect
 from django.contrib.auth import get_user_model
 from concrete_datastore.concrete.models import UserConfirmation, DIVIDER_MODEL
@@ -105,3 +115,83 @@ def unsubscribe_notifications_result_view(request, token):
             'platform_name': settings.PLATFORM_NAME,
         },
     )
+
+
+class RedirectStdStreams:
+    def __init__(self, stdout=sys.stdout, stderr=sys.stderr):
+        self._stdout = stdout
+        self._stderr = stderr
+
+    def __enter__(self):
+        self.old_stdout, self.old_stderr = sys.stdout, sys.stderr
+        self.old_stdout.flush()
+        self.old_stderr.flush()
+        sys.stdout, sys.stderr = self._stdout, self._stderr
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._stdout.flush()
+        self._stderr.flush()
+        sys.stdout = self.old_stdout
+        sys.stderr = self.old_stderr
+
+
+def dump_data(request):
+    if request.user.is_anonymous is True or request.user.is_superuser is False:
+        return HttpResponseForbidden()
+    resp = StringIO()
+    errors = StringIO()
+    try:
+        with RedirectStdStreams(stdout=resp, stderr=errors):
+            call_command('dumpdata', 'concrete')
+        resp_value = resp.getvalue()
+        errors_value = errors.getvalue()
+        if errors.getvalue():
+            return JsonResponse(data={'error': errors_value}, status=400)
+        if request.GET.get('download', '').lower() == 'true':
+            #: Download the json file
+            now = timezone.now()
+            filename = 'dump_{}.json'.format(now.strftime("%Y-%m-%d_%H-%M"))
+            response = StreamingHttpResponse(resp_value, content_type="json")
+            response[
+                'Content-Disposition'
+            ] = 'attachment; filename="{}"'.format(filename)
+            return response
+
+        json_resp = json.loads(resp_value)
+        return JsonResponse(
+            data={'result': json_resp, 'objects_count': len(json_resp)},
+            status=200,
+        )
+    except Exception as e:
+        return JsonResponse(data={'error': str(e)}, status=400)
+
+
+def load_data(request):
+    if request.user.is_anonymous is True or request.user.is_superuser is False:
+        return HttpResponseForbidden()
+
+    file = request.FILES.get('file')
+    if file is None:
+        return JsonResponse(data={'error': 'No file was given'}, status=400)
+    full_path = ""
+    fd = NamedTemporaryFile(suffix='.json', mode='w')
+    try:
+        full_path = os.path.join(os.getcwd(), fd.name)
+        json.dump(json.loads(file.read().decode('utf-8')), fd)
+        resp = StringIO()
+        errors = StringIO()
+        with RedirectStdStreams(stdout=resp, stderr=errors):
+            call_command('loaddata', full_path)
+        resp_value = resp.getvalue()
+        errors_value = errors.getvalue()
+        if errors_value:
+            response = JsonResponse(data={'error': errors_value}, status=400)
+        else:
+            response = JsonResponse(data={'message': resp_value}, status=200)
+    except Exception as e:
+        response = JsonResponse(
+            data={'error': f'An error has occured: {e}'}, status=400
+        )
+    finally:
+        fd.close()
+        return response

concrete_datastore/routes/views.py

@@ -1,7 +1,7 @@
 # coding: utf-8
-from importlib import import_module
 import yaml
 import json
+import datetime
 from django.conf import settings
 from django.http import (
     HttpResponse,
@@ -15,6 +15,7 @@
 from rest_framework.views import APIView
 from rest_framework import authentication
 from rest_framework.renderers import OpenAPIRenderer
+from concrete_datastore.api.v1.datetime import format_datetime
 from concrete_datastore.api.v1.authentication import (
     TokenExpiryAuthentication,
     URLTokenExpiryAuthentication,
@@ -70,9 +71,20 @@ class DatamodelServer(APIView, TemplateView):
     )
 
     def _get_datamodel_format(self, data_format='yaml'):
+        def default_dumper(value):
+            #: Default dumper for json because the DateTime and Date objects
+            #: are not json serializable
+            if type(value) == datetime.date:
+                return value.isoformat()
+            if type(value) == datetime.datetime:
+                return format_datetime(value)
+            return str(value)
+
         datamodel_content_json = settings.META_MODEL_DEFINITIONS
         if data_format == 'json':
-            return json.dumps(datamodel_content_json, indent=4)
+            return json.dumps(
+                datamodel_content_json, indent=4, default=default_dumper
+            )
         return yaml.dump(datamodel_content_json, allow_unicode=True)
 
     def get(self, request, *args, **kwargs):
@@ -103,12 +115,14 @@ def get(self, request, *args, **kwargs):
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
-        datamodel_content_json = settings.META_MODEL_DEFINITIONS
-        context['json_content'] = json.dumps(datamodel_content_json, indent=2)
+        datamodel_content_json = self._get_datamodel_format(data_format='json')
         datamodel_content_yml = self._get_datamodel_format()
-        content = DatamodelYamlToHtml(datamodel_content_json, indent=2)
+        content = DatamodelYamlToHtml(
+            settings.META_MODEL_DEFINITIONS, indent=2
+        )
         context['yaml_displayed_content'] = content.render_yaml()
         context['yaml_content'] = datamodel_content_yml
+        context['json_content'] = datamodel_content_json
         return context
 
 

concrete_datastore/settings/base.py

@@ -595,6 +595,10 @@
 
 ENABLE_SWAGGER_UI = True
 
+ENABLE_DATABASE_DUMP = True
+
+ENABLE_DATABASE_LOAD = True
+
 ENABLE_SERVE_DATAMODEL = True
 SWAGGER_SPEC_PATH = 'openapi-schema'
 SWAGGER_UI_PATH = 'swagger-ui'