diff --git a/api/views.py b/api/views.py index 8d2da8318..79d8c39fd 100644 --- a/api/views.py +++ b/api/views.py @@ -206,7 +206,7 @@ def post(self, request, answerpaper_id, question_id, format=None): answerpaper.save() json_data = None if question.type in ['code', 'upload']: - json_data = question.consolidate_answer_data(user_answer, user) + json_data = question.consolidate_answer_data(user_answer, user, answerpaper.id) result = answerpaper.validate_answer(user_answer, question, json_data, answer.id) diff --git a/requirements/requirements-common.txt b/requirements/requirements-common.txt index 1677ca40f..ffa06d1b5 100644 --- a/requirements/requirements-common.txt +++ b/requirements/requirements-common.txt @@ -24,3 +24,4 @@ qrcode more-itertools==8.4.0 django-storages==1.11.1 boto3==1.17.17 +aiohttp==3.7.4.post0 diff --git a/yaksh/code_server.py b/yaksh/code_server.py index 60f966f00..b79676f36 100644 --- a/yaksh/code_server.py +++ b/yaksh/code_server.py @@ -33,6 +33,7 @@ # Local imports from .settings import N_CODE_SERVERS, SERVER_POOL_PORT from .grader import Grader +from .file_utils import delete_files, Downloader MY_DIR = abspath(dirname(__file__)) @@ -91,6 +92,7 @@ def __init__(self, n, pool_port=50000): def _make_app(self): app = Application([ + (r"/files", FileHandler), (r"/.*", MainHandler, dict(server=self)), ]) app.listen(self.my_port) @@ -188,6 +190,29 @@ def post(self): self.write('OK') +class FileHandler(RequestHandler): + + def get(self): + self.write("Submit the file urls and path to download/delete") + + def post(self): + try: + files = self.get_arguments("files") + path = self.get_argument("path") + # Action is download or delete + action = self.get_argument("action") + if action == "download": + Downloader().main(files, path) + elif action == "delete": + delete_files(files, path) + else: + self.write( + "Please add action download or delete in the post data" + ) + except Exception as e: + self.write(e) + + def submit(url, uid, json_data, user_dir): '''Submit a job to the code server. diff --git a/yaksh/evaluator_tests/test_python_evaluation.py b/yaksh/evaluator_tests/test_python_evaluation.py index 343c8fbdc..9e0ddb6f5 100644 --- a/yaksh/evaluator_tests/test_python_evaluation.py +++ b/yaksh/evaluator_tests/test_python_evaluation.py @@ -33,9 +33,8 @@ def setUp(self): "test_case": 'assert(add(-1,-2)==-3)', 'weight': 0.0, 'hidden': True}, ] - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) + self.timeout_msg = ("You probably have an infinite loop in" + " your code.") self.file_paths = None def tearDown(self): @@ -650,9 +649,8 @@ def test_infinite_loop(self): "expected_output": "3", "weight": 0.0 }] - timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) + timeout_msg = ("You probably have an infinite loop in" + " your code.") user_answer = "while True:\n\tpass" kwargs = {'metadata': { @@ -731,9 +729,8 @@ def setUp(self): f.write('2'.encode('ascii')) tmp_in_dir_path = tempfile.mkdtemp() self.in_dir = tmp_in_dir_path - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) + self.timeout_msg = ("You probably have an infinite loop in" + " your code.") self.file_paths = None def tearDown(self): @@ -930,41 +927,5 @@ def check_answer(user_answer): result.get("error")[0]["message"] ) - def test_assignment_upload(self): - # Given - user_answer = "Assignment Upload" - hook_code = dedent("""\ - def check_answer(user_answer): - success = False - err = "Incorrect Answer" - mark_fraction = 0.0 - with open("test.txt") as f: - data = f.read() - if data == '2': - success, err, mark_fraction = True, "", 1.0 - return success, err, mark_fraction - """ - ) - - test_case_data = [{"test_case_type": "hooktestcase", - "hook_code": hook_code, "weight": 1.0 - }] - kwargs = {'metadata': { - 'user_answer': user_answer, - 'file_paths': self.file_paths, - 'assign_files': [(self.tmp_file, False)], - 'partial_grading': False, - 'language': 'python'}, - 'test_case_data': test_case_data, - } - - # When - grader = Grader(self.in_dir) - result = grader.evaluate(kwargs) - - # Then - self.assertTrue(result.get('success')) - - if __name__ == '__main__': unittest.main() diff --git a/yaksh/evaluator_tests/test_r_evaluation.py b/yaksh/evaluator_tests/test_r_evaluation.py index a196d91c2..7b8a51e25 100644 --- a/yaksh/evaluator_tests/test_r_evaluation.py +++ b/yaksh/evaluator_tests/test_r_evaluation.py @@ -46,9 +46,8 @@ def setUp(self): "test_case_type": "standardtestcase", "weight": 0.0, "hidden": True }] - self.timeout_msg = ("Code took more than {0} seconds to run. " - "You probably have an infinite loop in" - " your code.").format(SERVER_TIMEOUT) + self.timeout_msg = ("You probably have an infinite loop in" + " your code.") self.file_paths = None def tearDown(self): diff --git a/yaksh/file_utils.py b/yaksh/file_utils.py index 7c31c70f1..ddccbbdf9 100644 --- a/yaksh/file_utils.py +++ b/yaksh/file_utils.py @@ -3,6 +3,10 @@ import zipfile import tempfile import csv +import asyncio +import os +import aiohttp +import async_timeout def copy_files(file_paths): @@ -28,7 +32,7 @@ def delete_files(files, file_path=None): if file_path: file = os.path.join(file_path, file_name) else: - file = file_name + file = os.path.join(os.getcwd(), file_name) if os.path.exists(file): if os.path.isfile(file): os.remove(file) @@ -66,3 +70,26 @@ def is_csv(document): except (csv.Error, UnicodeDecodeError): return False, None return True, dialect + + +class Downloader: + async def get_url(self, url, path, session): + file_name = url.split("/")[-1] + if not os.path.exists(path): + os.makedirs(path) + async with async_timeout.timeout(120): + async with session.get(url) as response: + with open(os.path.join(path, file_name), 'wb') as fd: + async for data in response.content.iter_chunked(1024): + fd.write(data) + return file_name + + async def download(self, urls, path): + async with aiohttp.ClientSession() as session: + tasks = [self.get_url(url, path, session) for url in urls] + return await asyncio.gather(*tasks) + + def main(self, urls, download_path): + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.download(urls, download_path)) + diff --git a/yaksh/hook_evaluator.py b/yaksh/hook_evaluator.py index ff428c3d3..486e44557 100644 --- a/yaksh/hook_evaluator.py +++ b/yaksh/hook_evaluator.py @@ -5,7 +5,7 @@ import psutil # Local imports -from .file_utils import copy_files, delete_files +from .file_utils import copy_files, delete_files, Downloader from .base_evaluator import BaseEvaluator from .grader import TimeoutException from .error_messages import prettify_exceptions @@ -55,10 +55,10 @@ def check_code(self): Returns (False, error_msg, 0.0): If mandatory arguments are not files or if the required permissions are not given to the file(s). """ - if self.file_paths: - self.files = copy_files(self.file_paths) if self.assignment_files: - self.assign_files = copy_files(self.assignment_files) + self.assign_files = Downloader().main( + self.assignment_files, os.getcwd() + ) success = False mark_fraction = 0.0 try: diff --git a/yaksh/models.py b/yaksh/models.py index fd483189f..23fdb9b55 100644 --- a/yaksh/models.py +++ b/yaksh/models.py @@ -28,6 +28,7 @@ import pandas as pd import qrcode import hashlib +import urllib # Django Imports from django.db import models, IntegrityError @@ -47,6 +48,7 @@ from django.db.models import Count from django.db.models.signals import pre_delete from django.db.models.fields.files import FieldFile +from django.core.exceptions import SuspiciousFileOperation from django.core.files.base import ContentFile # Local Imports from yaksh.code_server import ( @@ -284,7 +286,10 @@ def file_cleanup(sender, instance, *args, **kwargs): for field_name, _ in instance.__dict__.items(): field = getattr(instance, field_name) if issubclass(field.__class__, FieldFile) and field.name: - field.delete(save=False) + try: + field.delete(save=False) + except SuspiciousFileOperation: + pass ############################################################################### class CourseManager(models.Manager): @@ -1432,7 +1437,7 @@ class Question(models.Model): ] } - def consolidate_answer_data(self, user_answer, user=None, regrade=False): + def consolidate_answer_data(self, user_answer, user, paper_id): question_data = {} metadata = {} test_case_data = [] @@ -1446,36 +1451,18 @@ def consolidate_answer_data(self, user_answer, user=None, regrade=False): metadata['user_answer'] = user_answer metadata['language'] = self.language metadata['partial_grading'] = self.partial_grading - files = FileUpload.objects.filter(question=self) - if files: - if settings.USE_AWS: - metadata['file_paths'] = [ - (file.file.url, file.extract) - for file in files - ] - else: - metadata['file_paths'] = [ - (self.get_file_url(file.file.url), file.extract) - for file in files - ] - if self.type == "upload" and regrade: - file = AssignmentUpload.objects.only( + if self.type == "upload" and self.grade_assignment_upload: + assignments = AssignmentUpload.objects.only( "assignmentFile").filter( - assignmentQuestion_id=self.id, answer_paper__user_id=user.id - ).order_by("-id").first() - if file: - if settings.USE_AWS: - metadata['assign_files'] = [file.assignmentFile.url] - else: - metadata['assign_files'] = [ - self.get_file_url(file.assignmentFile.url) - ] + assignmentQuestion_id=self.id, answer_paper_id=paper_id + ) + if assignments.exists(): + metadata['assign_files'] = [ + assignment.get_file_url() for assignment in assignments + ] question_data['metadata'] = metadata return json.dumps(question_data) - def get_file_url(self, path): - return f'{settings.DOMAIN_HOST}{path}' - def dump_questions(self, question_ids, user): questions = Question.objects.filter(id__in=question_ids, user_id=user.id, active=True @@ -1705,6 +1692,12 @@ def toggle_hide_status(self): def get_filename(self): return os.path.basename(self.file.name) + def get_file_url(self): + url = self.file.url + if not settings.USE_AWS: + url = urllib.parse.urljoin(settings.DOMAIN_HOST, url) + return url + pre_delete.connect(file_cleanup, sender=FileUpload) ############################################################################### class Answer(models.Model): @@ -2611,8 +2604,11 @@ def regrade(self, question_id, server_port=SERVER_POOL_PORT): return (False, f'{msg} {question.type} answer submission error') else: answer = user_answer.answer - json_data = question.consolidate_answer_data(answer, self.user, True) \ - if question.type == 'code' else None + if question.type in ['code', 'upload']: + json_data = question.consolidate_answer_data( + answer, self.user, self.id) + else: + json_data = None result = self.validate_answer(answer, question, json_data, user_answer.id, server_port=server_port @@ -2689,6 +2685,12 @@ class AssignmentUpload(models.Model): objects = AssignmentUploadManager() + def get_file_url(self): + url = self.assignmentFile.url + if not settings.USE_AWS: + url = urllib.parse.urljoin(settings.DOMAIN_HOST, url) + return url + def __str__(self): return f'Assignment File of the user {self.answer_paper.user}' diff --git a/yaksh/settings.py b/yaksh/settings.py index 7b42298c2..83e50d450 100644 --- a/yaksh/settings.py +++ b/yaksh/settings.py @@ -16,7 +16,7 @@ SERVER_HOST_NAME = config('SERVER_HOST_NAME', default='http://localhost') # Timeout for the code to run in seconds. This is an integer! -SERVER_TIMEOUT = config('SERVER_TIMEOUT', default=4, cast=int) +SERVER_TIMEOUT = config('SERVER_TIMEOUT', default=10, cast=int) # The root of the URL, for example you might be in the situation where you # are not hosted as host.org/exam/ but as host.org/foo/exam/ for whatever diff --git a/yaksh/static/yaksh/js/requesthandler.js b/yaksh/static/yaksh/js/requesthandler.js index 0a06c7912..0a2e98729 100644 --- a/yaksh/static/yaksh/js/requesthandler.js +++ b/yaksh/static/yaksh/js/requesthandler.js @@ -148,6 +148,7 @@ var marker; Dropzone.autoDiscover = false; var submitfiles; $(document).ready(function(){ + var submitButton = $("#check"); var filezone = $("div#dropzone_file").dropzone({ url: $("#code").attr("action"), parallelUploads: 10, @@ -156,36 +157,35 @@ $(document).ready(function(){ paramName: "assignment", autoProcessQueue: false, init: function() { - var submitButton = document.querySelector("#check"); myDropzone = this; - submitButton.addEventListener("click", function(e) { - if (myDropzone.getQueuedFiles().length === 0) { + submitButton.on("click", function(e) { + e.preventDefault(); + if(myDropzone.getQueuedFiles().length === 0) { $("#upload_alert").modal("show"); - e.preventDefault(); - return; } - if (myDropzone.getAcceptedFiles().length > 0) { - if (submitfiles === true) { - submitfiles = false; - return; - } - e.preventDefault(); - myDropzone.processQueue(); - myDropzone.on("complete", function () { - submitfiles = true; - $('#check').trigger('click'); + myDropzone.processQueue(); + return false; + }); + this.on('sending', function (file, xhr, formData) { + // Append all form inputs to the formData Dropzone will POST + var data = $("#code").serializeArray(); + $.each(data, function (key, el) { + formData.append(el.name, el.value); }); - } }); }, - success: function (file, response) { - document.open(); - document.write(response); - document.close(); + successmultiple: function (file, response) { + if(response.uid != undefined) { + lock_screen(); + myDropzone.removeAllFiles(file); + $("#file_uploads").html(response.uploads); + get_result(response.uid); + } else { + document.open(); + document.write(response); + document.close(); + } }, - headers: { - "X-CSRFToken": document.getElementById("code").elements[0].value - } }); if(is_exercise == "True" && can_skip == "False") { setTimeout(function() {show_solution();}, delay_time*1000); diff --git a/yaksh/tasks.py b/yaksh/tasks.py index 5068c6430..6fa4119d9 100644 --- a/yaksh/tasks.py +++ b/yaksh/tasks.py @@ -3,7 +3,8 @@ from textwrap import dedent import csv import json - +import requests +from urllib import parse # Django and celery imports from celery import shared_task from django.urls import reverse @@ -12,9 +13,10 @@ # Local imports from .models import ( Course, QuestionPaper, Quiz, AnswerPaper, CourseStatus, User, Question, - Answer + Answer, FileUpload ) from notifications_plugin.models import NotificationMessage, Notification +from yaksh.settings import SERVER_HOST_NAME, SERVER_POOL_PORT @shared_task @@ -194,3 +196,33 @@ def _read_marks_csv( notification = Notification.objects.add_single_notification( request_user, nm.id ) + + +@shared_task +def send_files_to_code_server(data): + ap_id = data.get("answerpaper_id", 0) + action = data.get("action", None) + path = data.get("path", None) + if path is None or action is None: + pass + else: + ap = get_object_or_404( + AnswerPaper.objects.prefetch_related("questions"), id=ap_id + ) + questions = ap.questions.values_list("id", flat=True) + uploads = FileUpload.objects.only("file").filter( + question_id__in=questions) + if uploads.exists(): + post_url = parse.urljoin( + f"{SERVER_HOST_NAME}:{SERVER_POOL_PORT}", "files" + ) + if action == "download": + files = [file.get_file_url() for file in uploads] + else: + files = [file.get_filename() for file in uploads] + data = {"files": files, "action": action, "path": path} + response = requests.post(post_url, data=data) + message = response.content + if response.status_code == 200: + message = "Successfully downloaded/deleted files" + return message diff --git a/yaksh/templates/yaksh/error_template.html b/yaksh/templates/yaksh/error_template.html index b93d2f156..4eff36289 100644 --- a/yaksh/templates/yaksh/error_template.html +++ b/yaksh/templates/yaksh/error_template.html @@ -1,7 +1,4 @@ {% load static %} -{% block css%} - -{% endblock %} {% block script %} {% endblock %} diff --git a/yaksh/templates/yaksh/question.html b/yaksh/templates/yaksh/question.html index 9fe57edde..123d831cc 100644 --- a/yaksh/templates/yaksh/question.html +++ b/yaksh/templates/yaksh/question.html @@ -95,15 +95,6 @@ } -function validate(){ - uploaded_file = document.getElementById("assignment").value; - if(uploaded_file == ""){ - $("#upload_alert").modal("show"); - return false; - } - return true; -} - function call_skip(url) { form = document.forms["code"] @@ -270,8 +261,8 @@

Solution by teacher

{% if question.type == "upload" %} +
{% if assignment_files %} -
-
{% endif %} +

@@ -292,14 +283,16 @@

Solution by teacher

-
- {% if qrcode %} - - {% else %} - Generate QR Code - {% endif %} -
+ {% if not question.grade_assignment_upload %} +
+ {% if qrcode %} + + {% else %} + Generate QR Code + {% endif %} +
+ {% endif %} {% endif %} @@ -344,7 +337,7 @@

Write your program below:

{% if question.type == "mcq" or question.type == "mcc" or question.type == "integer" or question.type == "float" or question.type == "string" %}
{% elif question.type == "upload" %} -
+
{% elif question.type == "arrange" %}
{% else %} diff --git a/yaksh/templates/yaksh/uploaded_files.html b/yaksh/templates/yaksh/uploaded_files.html new file mode 100644 index 000000000..fa73a491c --- /dev/null +++ b/yaksh/templates/yaksh/uploaded_files.html @@ -0,0 +1,9 @@ +{% load custom_filters %} + \ No newline at end of file diff --git a/yaksh/test_models.py b/yaksh/test_models.py index 019e2fc52..642dfe438 100644 --- a/yaksh/test_models.py +++ b/yaksh/test_models.py @@ -1547,7 +1547,7 @@ def add(a,b): # When json_data = self.question1.consolidate_answer_data( - user_answer, user, regrade=True + user_answer, user, self.answerpaper.id ) get_result = self.answerpaper.validate_answer(user_answer, self.question1, @@ -2193,8 +2193,9 @@ def test_stdout_based_testcase(self): def test_consolidate_answer_data(self): """ Test consolidate answer data model method """ + user_answer = "demo_answer" result = self.question1.consolidate_answer_data( - user_answer="demo_answer" + user_answer, self.user, None ) actual_data = json.loads(result) exp_data = json.loads(self.answer_data_json) diff --git a/yaksh/views.py b/yaksh/views.py index 4a6f462d2..153d87f80 100644 --- a/yaksh/views.py +++ b/yaksh/views.py @@ -60,7 +60,9 @@ from .send_emails import (send_user_mail, generate_activation_key, send_bulk_mail) from .decorators import email_verified, has_profile -from .tasks import regrade_papers, update_user_marks +from .tasks import ( + regrade_papers, update_user_marks, send_files_to_code_server +) from notifications_plugin.models import Notification import hashlib @@ -656,6 +658,10 @@ def start(request, questionpaper_id=None, attempt_num=None, course_id=None, raise Http404(msg) new_paper = quest_paper.make_answerpaper(user, ip, attempt_number, course_id) + # send celery task info to download files on code server + data = {"answerpaper_id": new_paper.id, "action": "download", + "path": user.profile.get_user_dir()} + task = send_files_to_code_server.delay(data) if new_paper.status == 'inprogress': return show_question( request, new_paper.current_question(), @@ -808,6 +814,7 @@ def check(request, q_id, attempt_num=None, questionpaper_id=None, course_id=course_id ) current_question = get_object_or_404(Question, pk=q_id) + uploaded_assignments = None def is_valid_answer(answer): status = True if ((current_question.type == "mcc" or @@ -880,18 +887,21 @@ def is_valid_answer(answer): assignmentFile=fname, answer_paper_id=paper.id )) - AssignmentUpload.objects.bulk_create(uploaded_files) - user_answer = 'ASSIGNMENT UPLOADED' - new_answer = Answer( - question=current_question, answer=user_answer, - correct=False, error=json.dumps([]) + uploaded_assignments = AssignmentUpload.objects.bulk_create( + uploaded_files ) - new_answer.save() - paper.answers.add(new_answer) - next_q = paper.add_completed_question(current_question.id) - return show_question(request, next_q, paper, - course_id=course_id, module_id=module_id, - previous_question=current_question) + user_answer = 'ASSIGNMENT UPLOADED' + if not current_question.grade_assignment_upload: + new_answer = Answer( + question=current_question, answer=user_answer, + correct=False, error=json.dumps([]) + ) + new_answer.save() + paper.answers.add(new_answer) + next_q = paper.add_completed_question(current_question.id) + return show_question(request, next_q, paper, + course_id=course_id, module_id=module_id, + previous_question=current_question) else: user_answer = request.POST.get('answer') if not is_valid_answer(user_answer): @@ -917,12 +927,15 @@ def is_valid_answer(answer): # If we were not skipped, we were asked to check. For any non-mcq # questions, we obtain the results via XML-RPC with the code executed # safely in a separate process (the code_server.py) running as nobody. - json_data = current_question.consolidate_answer_data( - user_answer, user) if current_question.type == 'code' else None + if current_question.type in ['code', 'upload']: + json_data = current_question.consolidate_answer_data( + user_answer, user, paper.id) + else: + json_data = None result = paper.validate_answer( user_answer, current_question, json_data, uid ) - if current_question.type == 'code': + if current_question.type in ['code', 'upload']: if (paper.time_left() <= 0 and not paper.question_paper.quiz.is_exercise): url = '{0}:{1}'.format(SERVER_HOST_NAME, SERVER_POOL_PORT) @@ -936,6 +949,10 @@ def is_valid_answer(answer): module_id=module_id, previous_question=current_question) else: + template = loader.get_template("yaksh/uploaded_files.html") + result['uploads'] = template.render( + {"files": uploaded_assignments} + ) return JsonResponse(result) else: next_question, error_message, paper = _update_paper( @@ -1050,6 +1067,9 @@ def complete(request, reason=None, attempt_num=None, questionpaper_id=None, attempt_number=attempt_num, course_id=course_id ) + data = {"answerpaper_id": paper.id, "action": "delete", + "path": user.profile.get_user_dir()} + send_files_to_code_server.delay(data) course = Course.objects.get(id=course_id) learning_module = course.learning_module.get(id=module_id) learning_unit = learning_module.learning_unit.get(quiz=q_paper.quiz)