Add --server option to test command

Author: daltonserey

bin/tstlib.py

@@ -99,11 +99,15 @@ def pop_argument(args, index=0):
     if index >= len(args):
         return None
 
-    return args.pop(0)
+    return args.pop(index)
 
 
-def pop_option(args, option, short=None, default=None, type=str):
-    selectors = ['--' + option]
+def pop_option(args, option, short=None, default=None, vtype=str):
+    if type(option) == list:
+        selectors = ['--' + opt for opt in option]
+    else:
+        selectors = ['--' + option]
+
     if short is not None:
         selectors.append('-' + short)
 
@@ -114,7 +118,7 @@ def pop_option(args, option, short=None, default=None, type=str):
     value = args.pop(index + 1)
     selector = args.pop(index)
 
-    return type(value)
+    return vtype(value)
 
 
 def pop_flag(args, flag, short=None):
@@ -518,7 +522,6 @@ def read_tstjson(file=TSTJSON, exit=False, quit_on_fail=False):
 
     try:
         with codecs.open(file, mode='r', encoding='utf-8') as f:
-            print(file)
             tstjson = json.loads(to_unicode(f.read()))
 
     except ValueError:
@@ -607,7 +610,6 @@ def abort(msg):
             if files[filename]['category'] == 'answer':
                 assignment['checksum'] = md5.md5(contents.encode('utf-8')).hexdigest()
 
-    assignment['checksum'] = md5.md5(contents).hexdigest()
     assignment['files'] = files
     assignment['unknown_files'] = unknown_files
 

commands/tst-checkout

@@ -159,7 +159,7 @@ def main(args):
     # parse command line
     command = tst.pop_argument(args)
     overwrite = tst.pop_flag(args, 'overwrite', short='o')
-    index = tst.pop_option(args, 'index', short='i', type=int)
+    index = tst.pop_option(args, 'index', short='i', vtype=int)
     name = tst.pop_argument(args)
     dirname = tst.pop_argument(args)
     _assert(args == [], "tst: fatal: too many arguments")

commands/tst-commit

@@ -300,21 +300,20 @@ def commit_assignment(tstjson, args):
         # take unique unknown file as answer, add to tstjson and notify user
         filename = assignment['unknown_files'].pop(0)
         tst.cprint(tst.LCYAN, "Assuming `%s` as answer file" % filename)
-    
     _assert(filename, "No answer file was identified")
 
+    # read current answer
     if not os.path.exists(filename):
         tst.cprint(tst.LRED, "Answer file not found: '%s'" % filename)
         print("---")
         print("Use `tst commit <filename>` to change and commit the answer file")
         print("Or bring back the file and run `tst commit` again")
         sys.exit()
-
-    # read current answer
     with codecs.open(filename, mode='r', encoding='utf-8') as f:
         contents = f.read()
+
     checksum = md5.md5(contents.encode('utf-8')).hexdigest()
-    if filename == current_answer_file and checksum == tstjson.get('last_commit'):
+    if filename == current_answer_file and checksum == tstjson.get('checksum'):
         tst.cprint(tst.LCYAN, 'No changes to commit')
         sys.exit()
 
@@ -341,7 +340,7 @@ def commit_assignment(tstjson, args):
         'category': 'answer',
         'data': contents
     }
-    tstjson['last_commit'] = checksum
+    tstjson['checksum'] = checksum
     tst.save_tstjson(tstjson)
 
     tst.cprint(tst.LCYAN, "Assignment successfully saved")

commands/tst-test

@@ -11,6 +11,7 @@ import re
 import sys
 import json
 import glob
+import time
 import shlex
 import signal
 import string
@@ -682,12 +683,60 @@ def debug(msg=''):
     print(line.encode('utf-8'), file=sys.stderr)
 
 
+def tests_report(summary):
+    num_tests = len(summary)
+    successes = sum(1 for c in summary if c == '.')
+    fails = sum(1 for c in summary if c in '*F')
+    errors = num_tests - successes - fails
+
+    return num_tests, successes, fails, errors
+
+
 def request_server_tests(tstjson):
+
+    # basic data
     server = tst.Server()
-    url = '/api/assignment/%s/tests' % tstjson['iid']
-    response = server.post(url, payload="", exit_on_fail=True)
-    print("Acceptance tests requested")
-    print(response.stdout)
+    url = '/api/assignment/%s/validation' % tstjson['iid']
+
+    while True:
+
+        # request last validation results
+        response = server.post(url, exit_on_fail=True)
+        status = response.json()['status']
+
+        if status == 'no validation requested':
+            # request last answer validation
+            tst.cprint(tst.LGREEN, "Requesting validation...")
+            response = server.post(url, payload="", exit_on_fail=True)
+
+        elif status == 'waiting worker response':
+            # wait a few seconds
+            tst.cprint(tst.LCYAN, "Waiting for the worker...")
+            time.sleep(3)
+
+        elif status == 'tests finished':
+            break
+
+        else:
+            print(response.stdout)
+            _assert(False, "fatal: please contact developers")
+
+    summary = response.json()['summary']
+    num_tests, successes, fails, errors = tests_report(summary)
+
+    # add num tests to msg
+    msg = "%d tests run, %d errors, %d fails." % (num_tests, errors, fails)
+
+    if all(c == '.' for c in summary):
+        tst.cprint(tst.LCYAN, summary)
+    else:
+        tst.cprint(tst.LRED, summary)
+    print(msg)
+    if not fails and not errors:
+        tst.cprint(tst.LGREEN, "Answer accepted")
+    else:
+        tst.cprint(tst.LGREEN, "Answer rejected")
+    
 
 
 def main():