parser on read to accept a None value for header

Author: mxhao2

language/mlsql/mlsql/functions/dataflow.py

@@ -109,8 +109,10 @@ def _model_phase(keywords, filename, header, sep, train, predictors, label, algo
 =============================================
 =============================================""" % (filename, df.head()) )
 
+    df.to_csv('temp.csv')
+
     # Encode all categorical values
-    df = encode_categorical(df)
+    # df = encode_categorical(df)
     #Classification and Regression and Cluster
     if not keywords["classify"] and not keywords["regress"] and not keywords["cluster"]:
         # KI: Rationale behind changining Error to Warning is that the user may

language/mlsql/mlsql/functions/keywords/preprocessing/impute_functions.py

@@ -33,10 +33,8 @@ def impute_missing(data, columns=None, impute_strategy='mode', missing_values='N
     if not cols_to_impute:
         return datacopy
     if impute_strategy == 'mode':
-        print(cols_to_impute)
         for col in cols_to_impute:
             modeVal = data[col].mode()
-            print(modeVal[0])
             datacopy[col] = _fill_col(data[col], missing_values, modeVal[0])
         return datacopy
     elif impute_strategy == 'mean':

language/mlsql/mlsql/functions/keywords/read_functions.py

@@ -13,7 +13,7 @@ def handle_read(userfile, separator, header):
     if is_mlsql_file(userfile):
         model = load_model(userfile)
     else:
-        return _read_data_file(userfile, separator, header)   
+        return _read_data_file(userfile, separator, header)
 
 
 def _read_data_file(userfile, separator, header):
@@ -34,7 +34,10 @@ def _read_data_file(userfile, separator, header):
 
     #attempt to read file with given parameters
     try:
-        df = read_csv(userfile, sep = separ, header = head)
+        if head is None:
+            df = read_csv(userfile, sep=separ, header=None)
+        else:
+            df = read_csv(userfile, sep = separ, header = head)
     except OSError as e:
         print("Error importing file: '" + userfile + "'")
         print(e)
@@ -46,7 +49,8 @@ def _handle_header(header):
     """
     Translates header into a proper value to be read by read_csv functions from pandas
     """
-    if header is None or header == "":
+
+    if header is None or header == "" or header == "None":
         return None
     elif header == "False":
         return None
@@ -69,4 +73,4 @@ def _handle_separator(sep):
     if sep is None or sep == "":
         return ","
     else:
-        return str(sep)
+        return str(sep)

language/mlsql/mlsql/parser/keywords/read.py

@@ -1,5 +1,5 @@
 from .grammer import *
-from pyparsing import Word, Keyword, Optional, MatchFirst, Literal
+from pyparsing import Word, Keyword, Optional, MatchFirst, Literal, oneOf
 
 def define_read():
     filename = Word(everythingWOQuotes).setResultsName("filename")
@@ -9,8 +9,9 @@ def define_read():
 
     #Define Read Optionals
     #header
+    Nones = oneOf('None')
     headerLiteral = (Literal("header") + Literal("=")).suppress()
-    header_choices = MatchFirst([Word(numbers), bool_true, bool_false]).setResultsName("header")
+    header_choices = MatchFirst([Word(numbers), bool_true, bool_false, Nones]).setResultsName("header")
     header = Optional(headerLiteral + header_choices)
 
     #separator
@@ -22,5 +23,5 @@ def define_read():
     readOptions = Optional(openParen + separator + ocomma +  header + closeParen)
 
     read = readKeyword + Quote + filename + Quote + readOptions
-    
+
     return read

language/mlsql/mlsql/test/mlsql_auto.py

@@ -1,8 +1,8 @@
 import mlsql
 from mlsql import execute
 
-query = 'READ "data/auto.csv" (separator = "\s+", header = 0)\
+query = 'READ "data/auto.csv" (separator = "\s+", header = None)\
  REPLACE ("?", "mode") SPLIT (train = .8, test = .2, validation = .0)\
   REGRESS (predictors = [2,3,4,5,6,7,8], label = 1, algorithm = simple)'
 
-execute(query, verbose=True)
+execute(query, verbose=False)