Run tests on CI + some improvements (#8)

* Run tests on Travis build

* Clone the latest CK for tests

* Some tests refactoring

* Migrate tests to python

* Minor

* Minor

* Minor

* Minor

* Minor

* Minor

* Minor

* No default secret key

* Add secret key to the tests

* Specify correct env vars for ck-crowdnode from tests

* Fix dealing with configs

* Detect the length of incoming request

* Minor

* Migrate the shell test to the new runner

Author: dsavenko

.ck-crowdnode/ck-crowdnode-config.json.linux.sample

@@ -1,5 +1,5 @@
 {
-"port":3334,
+"port":3333,
 "path_to_files":"$HOME/ck-crowdnode-files",
 "secret_key":"c4e239b4-8471-11e6-b24d-cbfef11692ca"
 }

.gitignore

@@ -1 +1,2 @@
 build
+*.pyc

.travis.yml

@@ -1,5 +1,5 @@
 language: c
-script: ./_compile.sh
+script: "./_compile.sh && python -m run_tests.py"
 compiler:
     - clang
     - gcc

appveyor.yml

@@ -1,9 +1,16 @@
+environment:
+    matrix:
+        - PYTHON: "C:\\Python27"
 
 build:
 
 build_script: 
     - mkdir build
     - cd build
     - cmake ..
-    - ls C:/projects/ck-crowdnode/build
     - msbuild /p:Configuration=Release /p:Platform=Win32 C:/projects/ck-crowdnode/build/Project.sln
+    - ls C:/projects/ck-crowdnode/build
+    - ls Release
+    - cp Release/ck-crowdnode-server.exe .
+    - cd ..
+    - "%PYTHON%/python.exe -m run_tests.py"

run_tests.py

@@ -0,0 +1,79 @@
+#!/usr/bin/python
+
+from __future__ import print_function
+import subprocess
+import shutil
+import os
+import sys
+import unittest
+import time
+import platform
+
+def safe_remove(fname):
+    try:
+        os.remove(fname)
+    except Exception:
+        pass
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+os.chdir(script_dir)
+
+config_dir = os.path.join(script_dir, '.ck-crowdnode')
+config_file = os.path.join(config_dir, 'ck-crowdnode-config.json')
+config_file_sample_windows = os.path.join(config_dir, 'ck-crowdnode-config.json.windows.sample')
+config_file_sample_linux = os.path.join(config_dir, 'ck-crowdnode-config.json.linux.sample')
+
+files_dir = os.path.join(script_dir, 'ck-crowdnode-files')
+if not os.path.exists(files_dir):
+    os.makedirs(files_dir)
+
+node_process=None
+ck_dir='tests-ck-master'
+
+def die(retcode):
+    os.chdir(script_dir)
+    shutil.rmtree(ck_dir, ignore_errors=True)
+    shutil.rmtree(files_dir, ignore_errors=True)
+    safe_remove(config_file)
+    if node_process is not None:
+        node_process.kill()
+    exit(retcode)
+
+safe_remove(config_file)
+node_env = os.environ.copy()
+if 'Windows' == platform.system():
+    node_env['LOCALAPPDATA'] = script_dir
+    shutil.copyfile(config_file_sample_windows, config_file)
+else:
+    node_env['HOME'] = script_dir
+    shutil.copyfile(config_file_sample_linux, config_file)
+
+node_process = subprocess.Popen(['build/ck-crowdnode-server'], env=node_env)
+
+shutil.rmtree(ck_dir, ignore_errors=True)
+r = subprocess.call('git clone https://github.com/ctuning/ck.git ' + ck_dir, shell=True)
+if 0 < r:
+    print('Error: failed to clone CK!')
+    die(1)
+
+sys.path.append(ck_dir)
+
+import ck.kernel as ck
+
+r = ck.access({'remote': 'yes', 'module_uoa': 'repo', 'url': 'http://localhost:3333', 'quiet': 'yes', 'data_uoa': 'remote-ck-node', 'action': 'add'})
+
+tests_dir = os.path.join(script_dir, 'tests')
+
+class CkTestLoader(unittest.TestLoader):
+    def loadTestsFromModule(self, module, pattern=None):
+        module.ck = ck
+        module.secret_key = 'c4e239b4-8471-11e6-b24d-cbfef11692ca'
+        module.platform = platform.system()
+        return unittest.TestLoader.loadTestsFromModule(self, module, pattern)
+
+suite = CkTestLoader().discover(tests_dir, pattern='test_*.py')
+
+os.chdir(tests_dir)
+test_result = unittest.TextTestRunner().run(suite)
+
+die(0 if test_result.wasSuccessful() else 1)

src/ck-crowdnode-server.c

@@ -125,6 +125,47 @@ int WSAGetLastError() {
 
 void doProcessing(int sock, char *baseDir);
 
+int sockSend(int sock, const void* buf, size_t len) {
+#ifdef _WIN32
+    return send(sock, buf, len, 0);
+#else
+    return write(sock, buf, len);
+#endif
+}
+
+int sockSendAll(int sock, const void* buf, size_t len) {
+    const char* p = buf;
+    while (0 < len) {
+        int n = sockSend(sock, p, len);
+        if (0 >= n) {
+            return -1;
+        }
+        p += n;
+        len -= n;
+    }
+    return 0;
+}
+
+int sendHttpResponse(int sock, int httpStatus, char* payload, int size) {
+    // send HTTP headers
+    char buf[200];
+    int n = sprintf(buf, "HTTP/1.1 %d OK\r\nContent-Length: %d\r\n\r\n", httpStatus, size);
+    if (0 >= n) {
+        perror("sprintf failed");
+        return -1;
+    }
+    if (0 > sockSendAll(sock, buf, n)) {
+        perror("Failed to send HTTP response headers");
+        return -1;
+    }
+
+    // send payload
+    if (0 > sockSendAll(sock, payload, size)) {
+        perror("Failed to send HTTP response body");
+        return -1;
+    }
+}
+
 void sendErrorMessage(int sock, char * errorMessage, const char *errorCode) {
 	perror(errorMessage);
 
@@ -141,12 +182,7 @@ void sendErrorMessage(int sock, char * errorMessage, const char *errorCode) {
         perror("[ERROR]: resultJSONtext cannot be created");
         return;
     }
-#ifdef _WIN32
-    int n =send( sock, resultJSONtext, strlen(resultJSONtext), 0 );
-#else
-    int n = write(sock, resultJSONtext, strlen(resultJSONtext));
-#endif
-
+    int n = sendHttpResponse(sock, 500, resultJSONtext, strlen(resultJSONtext));
     if (n < 0) {
 		perror("ERROR writing to socket");
 		return ;
@@ -530,7 +566,41 @@ void doProcessingWin (struct thread_win_params* ptwp)
 }
 #endif
 
+/**
+ * Tries to detect message length by the given buffer, which contains the beginning of the message.
+ * The buffer passed must be of at least (size+1) length.
+ * 
+ * Returns -1, if the length is still unknown (in this case the caller must provide a bigger part of the message).
+ * 
+ * Returns -2, if the length can never be determined, i.e. HTTP headers don't contain 'Content-Length'.
+ *
+ * If 0 or more is returned, it is the total size of the message (size of the headers + size of the body).
+ */
+int detectMessageLength(char* buf, int size) {
+    buf[size] = 0;
+    
+    // trying to find where headers end
+    char* s = strstr(buf, "\r\n\r\n");
+    int header_stop_len = 4;
+    if (NULL == s) {
+        s = strstr(buf, "\n\n");
+        header_stop_len = 2;
+    }
+    if (NULL == s) {
+        return -1;
+    }
+    const long header_len = (s - buf) + header_stop_len;
+
+    const char* content_len_key = "Content-Length:";
+    // trying to find Content-Length
+    char* content_len_header = strstr(buf, content_len_key);
+    if (NULL == content_len_header || (content_len_header - buf) >= header_len) {
+        return -2;
+    }
 
+    long l = strtol(content_len_header + strlen(content_len_key), NULL, 10);
+    return header_len + l;
+}
 
 void doProcessing(int sock, char *baseDir) {
     char *client_message = malloc(MAX_BUFFER_SIZE + 1);
@@ -548,6 +618,7 @@ void doProcessing(int sock, char *baseDir) {
     memset(buffer, 0, MAX_BUFFER_SIZE);
     int buffer_read = 0;
     int total_read = 0;
+    int message_len = -1;
 
     //buffered read from socket
     int i = 0;
@@ -564,12 +635,15 @@ void doProcessing(int sock, char *baseDir) {
             total_read = total_read + buffer_read;
             printf("Next %i part of buffer\n", i);
             i++;
+            if (-1 == message_len) {
+                message_len = detectMessageLength(buffer, total_read);
+            }
         } else if (buffer_read < 0) {
             perror("[ERROR]: reading from socket");
             printf("WSAGetLastError() %i\n", WSAGetLastError()); //win
             exit(1);
         }
-        if (buffer_read == 0 || buffer_read < MAX_BUFFER_SIZE) {
+        if (buffer_read == 0 || total_read >= message_len || -2 == message_len) {
             /* message received successfully */
             break;
         }
@@ -607,10 +681,8 @@ void doProcessing(int sock, char *baseDir) {
         return;
     }
     char *clientSecretKey = secretkeyJSON->valuestring;
-    printf("[ERROR]: Get secretkey: %s from client\n", clientSecretKey);
+    printf("[DEBUG]: Got secretkey: %s from client\n", clientSecretKey);
     if (!serverSecretKey || strncmp(clientSecretKey, serverSecretKey, strlen(serverSecretKey)) == 0 ) {
-
-
         cJSON *actionJSON = cJSON_GetObjectItem(commandJSON, JSON_PARAM_NAME_COMMAND);
         if (!actionJSON) {
             printf("[ERROR]: Invalid action JSON format for message: \n");
@@ -879,11 +951,7 @@ void doProcessing(int sock, char *baseDir) {
             sendErrorMessage(sock, "unknown action", ERROR_CODE);
         }
 
-#ifdef _WIN32
-        int n1 =send( sock, resultJSONtext, strlen(resultJSONtext), 0 );
-#else
-        int n1 = write(sock, resultJSONtext, strlen(resultJSONtext));
-#endif
+        int n1 = sendHttpResponse(sock, 200, resultJSONtext, strlen(resultJSONtext));
 
         free(resultJSONtext);