diff --git a/container_files/public_html/doc/builtin/lang/Javascript.md b/container_files/public_html/doc/builtin/lang/Javascript.md index aa88dc39b..7b189694b 100644 --- a/container_files/public_html/doc/builtin/lang/Javascript.md +++ b/container_files/public_html/doc/builtin/lang/Javascript.md @@ -196,6 +196,29 @@ MLDB's atomic types are represented in Javascript as follows: - An array will be used as a compound path with the elements as specified. +### ExpressionValue object + +MLDB's non-atomic types are represented in Javascript by the +`ExpressionValue` class, which represents the various atomic and +structured values of MLDB, plus their associated timestamps. + +That object has the following methods: + +- `toJs()` will return a Javascript representation of the current + values, stripping off the timestamps. Structures and arrays + are supported. +- `when()` will return the timestamp associated with a value. +- `at()` will return a new ExpressionValue with the timestamp + modified to happen at the given point in time. This can be + used to put timestamps back on values which have been processed + with Javascript code. +- `columns()` returns an object with the column names as keys and + their ExpressionValue values as values. Note that this method + only un-nests by one level. + +ExpressionValue objects are primarily used by the `jseval` function +with simplified arguments off. + ### Filesystem access diff --git a/container_files/public_html/doc/builtin/sql/ValueExpression.md b/container_files/public_html/doc/builtin/sql/ValueExpression.md index 412775064..29c7d83f3 100644 --- a/container_files/public_html/doc/builtin/sql/ValueExpression.md +++ b/container_files/public_html/doc/builtin/sql/ValueExpression.md @@ -639,7 +639,7 @@ The SQL function `jseval` allows for the inline definition of functions using Ja 1. A text string containing the text of the function to be evaluated. This must be a valid Javascript function, which will return with the `return` - function. For example, `return x + y`. This must be a constant string, + keyword. For example, `return x + y`. This must be a constant string, it cannot be an expression that is evaluated at run time. 2. A text string containing the names of all of the parameters that will be passed to the function, as they are referred to within the function. For @@ -649,13 +649,21 @@ The SQL function `jseval` allows for the inline definition of functions using Ja any SQL expressions and will be bound to the parameters passed in to the function. +There are two ways that values in arguments can be represented in Javascript: +simplified (the default), and a non-simplified representation that is accessed +by adding a `!` character to the parameter name argument (for example, `!x,y` +instead of `x,y`). + +### Simplified arguments + The result of the function will be the result of calling the function on the supplied arguments. This will be converted into a result as follows: - A `null` will remain a `null` - A Javascript number, string or `Date` will be converted to the equivalent MLDB number, string or timestamp; -- An object (dictionary) will be converted to a row +- An object (dictionary) or array will be converted to a row, with each + element represented as a `[column name, value, timestamp]` tuple. In all cases, the timestamp on the output will be equal to the latest of the timestamps on the arguments passed in to the function. @@ -695,3 +703,16 @@ log to the console to aid debugging. Documentation for this object can be found You can also take a look at the ![](%%nblink _tutorials/Executing JavaScript Code Directly in SQL Queries Using the jseval Function Tutorial) for examples of how to use the `jseval` function. +### Non-simplified arguments + +If the first character of the argument string is `!`, then non-simplified +arguments are used. These are harder to work with in Javascript, but allow +for the entire set of values in MLDB to be represented, especially structured +values or those with repeated columns or multiple timestamps per value. + +For example, the following query yields the same as `(SELECT x:1, y:2)`, +in other words it doesn't mess around with the values: + +```sql +SELECT jseval('return row;', '!row', {*}) AS * FROM (SELECT x:1, y:2)" +``` diff --git a/plugins/lang/js/dataset_js.cc b/plugins/lang/js/dataset_js.cc index cd8262c33..aafa2b50f 100644 --- a/plugins/lang/js/dataset_js.cc +++ b/plugins/lang/js/dataset_js.cc @@ -9,7 +9,7 @@ #include "dataset_js.h" #include "mldb/core/dataset.h" -#include "mldb/types/js/id_js.h" +#include "id_js.h" using namespace std; @@ -25,7 +25,7 @@ namespace MLDB { v8::Handle DatasetJS:: -create(std::shared_ptr dataset, JsPluginContext * context) +create(std::shared_ptr dataset, JsThreadContext * context) { auto obj = context->Dataset->GetFunction()->NewInstance(); auto * wrapped = new DatasetJS(); diff --git a/plugins/lang/js/dataset_js.h b/plugins/lang/js/dataset_js.h index 3b386c046..2df995abd 100644 --- a/plugins/lang/js/dataset_js.h +++ b/plugins/lang/js/dataset_js.h @@ -25,7 +25,7 @@ struct DatasetJS: public JsObjectBase { std::shared_ptr dataset; static v8::Handle - create(std::shared_ptr dataset, JsPluginContext * context); + create(std::shared_ptr dataset, JsThreadContext * context); static Dataset * getShared(const v8::Handle & val); diff --git a/plugins/lang/js/function_js.cc b/plugins/lang/js/function_js.cc index fad58b966..08728da52 100644 --- a/plugins/lang/js/function_js.cc +++ b/plugins/lang/js/function_js.cc @@ -7,7 +7,7 @@ #include "function_js.h" #include "mldb/core/function.h" -#include "mldb/types/js/id_js.h" +#include "id_js.h" using namespace std; @@ -23,7 +23,7 @@ namespace MLDB { v8::Handle FunctionJS:: -create(std::shared_ptr function, JsPluginContext * context) +create(std::shared_ptr function, JsThreadContext * context) { auto obj = context->Function->GetFunction()->NewInstance(); auto * wrapped = new FunctionJS(); diff --git a/plugins/lang/js/function_js.h b/plugins/lang/js/function_js.h index ea9588fb0..89c238835 100644 --- a/plugins/lang/js/function_js.h +++ b/plugins/lang/js/function_js.h @@ -25,7 +25,7 @@ struct FunctionJS: public JsObjectBase { std::shared_ptr function; static v8::Handle - create(std::shared_ptr function, JsPluginContext * context); + create(std::shared_ptr function, JsThreadContext * context); static Function * getShared(const v8::Handle & val); diff --git a/types/js/id_js.h b/plugins/lang/js/id_js.h similarity index 93% rename from types/js/id_js.h rename to plugins/lang/js/id_js.h index eb63a4f8a..db2f44cf6 100644 --- a/types/js/id_js.h +++ b/plugins/lang/js/id_js.h @@ -6,7 +6,7 @@ #pragma once -#include "mldb/soa/js/js_utils.h" +#include "mldb/plugins/lang/js/js_utils.h" #include "mldb/types/id.h" namespace Datacratic { diff --git a/plugins/lang/js/js_common.cc b/plugins/lang/js/js_common.cc index c19bc27e8..11d439809 100644 --- a/plugins/lang/js/js_common.cc +++ b/plugins/lang/js/js_common.cc @@ -98,7 +98,7 @@ CellValue from_js(const JS::JSValue & value, CellValue *) return CellValue(Date::fromSecondsSinceEpoch(value->NumberValue() / 1000.0)); else if (value->IsObject()) { // Look if it's already a CellValue - JsPluginContext * cxt = JsContextScope::current(); + JsThreadContext * cxt = JsContextScope::current(); if (cxt->CellValue->HasInstance(value)) { return CellValueJS::getShared(value); } @@ -126,8 +126,9 @@ void to_js(JS::JSValue & value, const CellValue & val) to_js(value, val.toString()); } else { + cerr << endl << endl << endl << "((((((( CELLVALUE ))))))))" << endl << endl; // Get our context so we can return a proper object - JsPluginContext * cxt = JsContextScope::current(); + JsThreadContext * cxt = JsContextScope::current(); value = CellValueJS::create(val, cxt); } } @@ -178,18 +179,63 @@ void to_js(JS::JSValue & value, const Path & val) void to_js(JS::JSValue & value, const ExpressionValue & val) { - to_js(value, val.getAtom()); + JsThreadContext * cxt = JsContextScope::current(); + value = ExpressionValueJS::create(val, cxt); } ExpressionValue from_js(const JS::JSValue & value, ExpressionValue *) { - // NOTE: we currently pretend that CellValue and ExpressionValue - // are the same thing; they are not. We will eventually need to - // allow proper JS access to full-blown ExpressionValue objects, - // backed with a JS object. + if (value->IsNull() || value->IsUndefined()) + return ExpressionValue::null(Date::notADate()); + else if (value->IsNumber()) + return ExpressionValue(value->NumberValue(), Date::notADate()); + else if (value->IsDate()) + return ExpressionValue(Date::fromSecondsSinceEpoch(value->NumberValue() / 1000.0), + Date::notADate()); + else if (value->IsArray()) { + // It must be an embedding + StructValue result; + + auto arrPtr = v8::Array::Cast(*value); + for(size_t i=0; iLength(); ++i) { + PathElement key(i); + v8::Local val = arrPtr->Get(i); + ExpressionValue ev = from_js(val, (ExpressionValue *)0); + result.emplace_back(std::move(key), std::move(ev)); + } + + return std::move(result); + } + else if (value->IsObject()) { + // Look if it's already an ExpressionValue + JsThreadContext * cxt = JsContextScope::current(); + if (cxt->ExpressionValue->HasInstance(value)) { + return ExpressionValueJS::getShared(value); + } - CellValue val = from_js(value, (CellValue *)0); - return ExpressionValue(val, Date::notADate()); + // Look if it's already a CellValue + if (cxt->CellValue->HasInstance(value)) { + return ExpressionValue(CellValueJS::getShared(value), + Date::notADate()); + } + + auto objPtr = v8::Object::Cast(*value); + + // It must be a nested structure + StructValue result; + + v8::Local properties = objPtr->GetOwnPropertyNames(); + + for (size_t i=0; iLength(); ++i) { + v8::Local key = properties->Get(i); + v8::Local val = objPtr->Get(key); + ExpressionValue ev = from_js(val, (ExpressionValue *)0); + result.emplace_back(PathElement(JS::utf8str(key)), std::move(ev)); + } + + return std::move(result); + } + else return ExpressionValue(JS::utf8str(value), Date::notADate()); } ScriptStackFrame @@ -337,18 +383,18 @@ JsObjectBase:: js_object_.Clear(); } -JsPluginContext * +JsThreadContext * JsObjectBase:: getContext(const v8::Handle & val) { - return reinterpret_cast + return reinterpret_cast (v8::Handle::Cast (val->GetInternalField(1))->Value()); } void JsObjectBase:: -wrap(v8::Handle handle, JsPluginContext * context) +wrap(v8::Handle handle, JsThreadContext * context) { ExcAssert(js_object_.IsEmpty()); @@ -420,9 +466,10 @@ garbageCollectionCallback(v8::Persistent value, void *data) /*****************************************************************************/ JsContextScope:: -JsContextScope(JsPluginContext * context) +JsContextScope(JsThreadContext * context) : context(context) { + ExcAssert(context); enter(context); } @@ -439,9 +486,9 @@ JsContextScope:: exit(context); } -static __thread std::vector * jsContextStack = nullptr; +static __thread std::vector * jsContextStack = nullptr; -JsPluginContext * +JsThreadContext * JsContextScope:: current() { @@ -452,16 +499,16 @@ current() void JsContextScope:: -enter(JsPluginContext * context) +enter(JsThreadContext * context) { if (!jsContextStack) - jsContextStack = new std::vector(); + jsContextStack = new std::vector(); jsContextStack->push_back(context); } void JsContextScope:: -exit(JsPluginContext * context) +exit(JsThreadContext * context) { if (current() != context) throw ML::Exception("JS context stack consistency error"); diff --git a/plugins/lang/js/js_common.h b/plugins/lang/js/js_common.h index 11b30bb26..d9dd1699f 100644 --- a/plugins/lang/js/js_common.h +++ b/plugins/lang/js/js_common.h @@ -8,7 +8,7 @@ */ #include -#include "mldb/soa/js/js_utils.h" +#include "mldb/plugins/lang/js/js_utils.h" #include "mldb/types/value_description.h" #include "mldb/logging/logging.h" #include "mldb/server/script_output.h" @@ -112,19 +112,18 @@ struct JsException: public ML::Exception { ScriptException rep; }; -struct JsPluginContext { - /** Create a JS plugin context. Note that pluginResource may be - a null pointer if the context is for a JS function rather than - an actual plugin. - */ - JsPluginContext(const Utf8String & pluginName, MldbServer * server, - std::shared_ptr pluginResource); - ~JsPluginContext(); +/*****************************************************************************/ +/* JS THREAD CONTEXT */ +/*****************************************************************************/ + +struct JsThreadContext { + JsThreadContext(JsIsolate & isolate, + MldbServer * server, + const Utf8String & pluginName); - JsIsolate isolate; + JsIsolate & isolate; v8::Persistent context; - v8::Persistent script; std::string categoryName, loaderName; std::mutex logMutex; /// protects the categories below @@ -133,18 +132,14 @@ struct JsPluginContext { std::vector logs; - std::function getStatus; - RestRequestRouter router; - RestRequestRouter::OnProcessRequest handleRequest; MldbServer * server; - std::shared_ptr pluginResource; - // These are the function templates for all of the builtin objects v8::Persistent Plugin; v8::Persistent Mldb; v8::Persistent Stream; v8::Persistent CellValue; + v8::Persistent ExpressionValue; v8::Persistent PathElement; v8::Persistent Path; v8::Persistent Dataset; @@ -154,6 +149,31 @@ struct JsPluginContext { }; +/*****************************************************************************/ +/* JS PLUGIN CONTEXT */ +/*****************************************************************************/ + +struct JsPluginContext: public JsIsolate, public JsThreadContext { + + /** Create a JS plugin context. Note that pluginResource may be + a null pointer if the context is for a JS function rather than + an actual plugin. + */ + JsPluginContext(const Utf8String & pluginName, MldbServer * server, + std::shared_ptr pluginResource); + ~JsPluginContext(); + + using JsThreadContext::isolate; + v8::Persistent script; + std::function getStatus; + RestRequestRouter router; + RestRequestRouter::OnProcessRequest handleRequest; + + std::shared_ptr pluginResource; + +}; + + /*****************************************************************************/ /* JS CONTEXT SCOPE */ /*****************************************************************************/ @@ -163,7 +183,7 @@ struct JsPluginContext { */ struct JsContextScope { - JsContextScope(JsPluginContext * context); + JsContextScope(JsThreadContext * context); JsContextScope(const v8::Handle & val); ~JsContextScope(); @@ -172,13 +192,13 @@ struct JsContextScope { JsContextScope(JsContextScope && other) = delete; void operator = (JsContextScope && other) = delete; - static JsPluginContext * current(); + static JsThreadContext * current(); private: - static void enter(JsPluginContext * context); - static void exit(JsPluginContext * context); + static void enter(JsThreadContext * context); + static void exit(JsThreadContext * context); - JsPluginContext * context; + JsThreadContext * context; }; @@ -203,7 +223,7 @@ class JsObjectBase { (handle->GetInternalField(0))->Value()); } - static JsPluginContext * getContext(const v8::Handle & val); + static JsThreadContext * getContext(const v8::Handle & val); v8::Persistent js_object_; @@ -212,7 +232,7 @@ class JsObjectBase { /** Set up the object by making handle contain an external reference to the given object. */ - void wrap(v8::Handle handle, JsPluginContext * context); + void wrap(v8::Handle handle, JsThreadContext * context); /** Set this object up to be garbage collected once there are no more references to it in the javascript. */ diff --git a/plugins/lang/js/js_function.cc b/plugins/lang/js/js_function.cc index cb4911317..de691c31d 100644 --- a/plugins/lang/js/js_function.cc +++ b/plugins/lang/js/js_function.cc @@ -10,7 +10,7 @@ #include "mldb/arch/thread_specific.h" #include "mldb/http/http_exception.h" #include "mldb/types/basic_value_descriptions.h" -#include "mldb/types/js/id_js.h" +#include "id_js.h" #include "mldb/sql/expression_value.h" #include "mldb/sql/sql_expression.h" @@ -27,26 +27,23 @@ struct JsFunctionData; /** Data for a JS function for each thread. */ struct JsFunctionThreadData { - JsFunctionThreadData() - : isolate(0), data(0) - { - } - + bool initialized() const { return isolate; } - JsIsolate * isolate; - v8::Persistent context; + JsIsolate * isolate = nullptr; v8::Persistent script; v8::Persistent function; - const JsFunctionData * data; + const JsFunctionData * data = nullptr; + std::shared_ptr threadContext; void initialize(const JsFunctionData & data); - ExpressionValue run(const std::vector & args, - const SqlRowScope & context) const; + MLDB::ExpressionValue run(const std::vector & args, + const SqlRowScope & context, + bool simplified) const; }; struct JsFunctionData { @@ -55,7 +52,6 @@ struct JsFunctionData { Utf8String scriptSource; std::string filenameForErrorMessages; std::vector params; - std::shared_ptr context; }; void @@ -68,6 +64,8 @@ initialize(const JsFunctionData & data) return; isolate = JsIsolate::getIsolateForMyThread(); + + threadContext.reset(new JsThreadContext(*isolate, data.server, "jseval")); this->data = &data; //v8::Locker locker(this->isolate->isolate); @@ -75,19 +73,11 @@ initialize(const JsFunctionData & data) HandleScope handle_scope; - // Create a new context. - this->context = v8::Persistent::New(Context::New()); - - // Enter the created context for compiling and // running the hello world script. - Context::Scope context_scope(this->context); + Context::Scope context_scope(threadContext->context); // Add the mldb object to the context - auto mldb = MldbJS::registerMe()->NewInstance(); - mldb->SetInternalField(0, v8::External::New(data.server)); - mldb->SetInternalField(1, v8::External::New(data.context.get())); - this->context->Global()->Set(String::New("mldb"), mldb); Utf8String jsFunctionSource = data.scriptSource; @@ -99,7 +89,7 @@ initialize(const JsFunctionData & data) // This is equivalent to fntocall = new Function('arg1', ..., 'script'); v8::Local function - = v8::Local::Cast(this->context->Global()->Get(v8::String::New("Function"))); + = v8::Local::Cast(threadContext->context->Global()->Get(v8::String::New("Function"))); std::vector > argv; for (unsigned i = 0; i != data.params.size(); ++i) argv.push_back(v8::String::New(data.params[i].c_str())); @@ -122,7 +112,8 @@ initialize(const JsFunctionData & data) ExpressionValue JsFunctionThreadData:: run(const std::vector & args, - const SqlRowScope & context) const + const SqlRowScope & context, + bool simplifyArgs) const { using namespace v8; @@ -135,19 +126,27 @@ run(const std::vector & args, // Enter the created context for compiling and // running the hello world script. - Context::Scope context_scope(this->context); + Context::Scope context_scope(threadContext->context); + JsContextScope jsContextScope(threadContext.get()); + Date ts = Date::negativeInfinity(); std::vector > argv; + argv.reserve(args.size() - 1); for (unsigned i = 2; i < args.size(); ++i) { - if (args[i].isRow()) { - RowValue row; - args[i].appendToRow(Path(), row); - argv.push_back(JS::toJS(row)); + if (simplifyArgs) { + if (args[i].isRow()) { + RowValue row; + args[i].appendToRow(Path(), row); + argv.push_back(JS::toJS(row)); + } + else { + argv.push_back(JS::toJS(args[i].getAtom())); + } } else { - argv.push_back(JS::toJS(args[i].getAtom())); + argv.push_back(JS::toJS(args[i])); } ts.setMax(args[i].getEffectiveTimestamp()); } @@ -155,7 +154,8 @@ run(const std::vector & args, TryCatch trycatch; //trycatch.SetVerbose(true); - auto result = this->function->Call(this->context->Global(), argv.size(), &argv[0]); + auto result = this->function->Call(threadContext->context->Global(), + argv.size(), &argv[0]); if (result.IsEmpty()) { auto rep = convertException(trycatch, "Running jseval script"); @@ -170,31 +170,21 @@ run(const std::vector & args, if (result->IsUndefined()) { return ExpressionValue::null(Date::notADate()); } - else if (result->IsString() || result->IsNumber() || result->IsNull() || result->IsDate()) { + else if (result->IsString() || result->IsNumber() || result->IsNull() + || result->IsDate() || result->IsBoolean()) { CellValue res = JS::fromJS(result); - return ExpressionValue(res, ts); } - else if (result->IsObject()) { - std::map cols = JS::fromJS(result); - - std::vector > row; - row.reserve(cols.size()); - for (auto & c: cols) { - row.emplace_back(c.first, ExpressionValue(std::move(c.second), - ts)); - } - return ExpressionValue(std::move(row)); - } else { - throw HttpReturnException(400, "Don't understand expression"); + return JS::fromJS(result); } } ExpressionValue runJsFunction(const std::vector & args, const SqlRowScope & context, - const shared_ptr & data) + const shared_ptr & data, + bool simplifyArgs) { // 1. Find the JS function for this isolate JsFunctionThreadData * threadData = data->threadInfo.get(); @@ -203,7 +193,7 @@ runJsFunction(const std::vector & args, threadData->initialize(*data); // 2. Run the function - return threadData->run(args, context); + return threadData->run(args, context, simplifyArgs); } BoundFunction bindJsEval(const Utf8String & name, @@ -221,10 +211,16 @@ BoundFunction bindJsEval(const Utf8String & name, runner->server = context.getMldbServer(); runner->scriptSource = scriptSource; runner->filenameForErrorMessages = "<>"; - runner->context.reset(new JsPluginContext(name, runner->server, - nullptr /* no plugin context */)); string params = args[1].constantValue().toString(); + + // If the first character of args is an exclamation point, then we + // pass args as full ExpressionValues. Otherwise we pass them + // as simplified values or an array of rows. + bool simplifyArgs = params.empty() || params[0] != '!'; + if (!simplifyArgs) { + params = string(params, 1); + } boost::split(runner->params, params, boost::is_any_of(",")); @@ -235,7 +231,7 @@ BoundFunction bindJsEval(const Utf8String & name, auto fn = [=] (const std::vector & args, const SqlRowScope & context) -> ExpressionValue { - return runJsFunction(args, context, runner); + return runJsFunction(args, context, runner, simplifyArgs); }; // 5. Return it diff --git a/plugins/lang/js/js_loader.cc b/plugins/lang/js/js_loader.cc index 1b36c796b..742b8a945 100644 --- a/plugins/lang/js/js_loader.cc +++ b/plugins/lang/js/js_loader.cc @@ -16,7 +16,7 @@ #include "mldb/sql/cell_value.h" #include "mldb/http/http_exception.h" #include "mldb/rest/rest_request_binding.h" -#include "mldb/types/js/id_js.h" +#include "id_js.h" #include "mldb/jml/utils/file_functions.h" #include "mldb/types/any_impl.h" @@ -27,7 +27,7 @@ #include "procedure_js.h" #include -#include "mldb/soa/js/js_utils.h" +#include "mldb/plugins/lang/js/js_utils.h" #include "mldb/types/string.h" #include @@ -338,25 +338,24 @@ struct JsPluginJS { /*****************************************************************************/ -/* JS PLUGIN CONTEXT */ +/* JS THREAD CONTEXT */ /*****************************************************************************/ -JsPluginContext:: -JsPluginContext(const Utf8String & pluginName, +JsThreadContext:: +JsThreadContext(JsIsolate & isolate, MldbServer * server, - std::shared_ptr pluginResource) - : isolate(false /* for this thread only */), + const Utf8String & pluginName) + : isolate(isolate), categoryName(pluginName.rawString() + " plugin"), loaderName(pluginName.rawString() + " loader"), category(categoryName.c_str()), loader(loaderName.c_str()), - server(server), - pluginResource(pluginResource) + server(server) { using namespace v8; v8::Locker locker(this->isolate.isolate); - v8::Isolate::Scope isolate(this->isolate.isolate); + v8::Isolate::Scope isolateScope(this->isolate.isolate); HandleScope handle_scope; @@ -372,13 +371,6 @@ JsPluginContext(const Utf8String & pluginName, v8::Local globalPrototype = v8::Local::Cast(context->Global()->GetPrototype()); - auto plugin = JsPluginJS::registerMe()->NewInstance(); - plugin->SetInternalField(0, v8::External::New(this)); - plugin->SetInternalField(1, v8::External::New(this)); - if (pluginResource) - plugin->Set(String::New("args"), JS::toJS(jsonEncode(pluginResource->args))); - globalPrototype->Set(String::New("plugin"), plugin); - auto mldb = MldbJS::registerMe()->NewInstance(); mldb->SetInternalField(0, v8::External::New(this->server)); mldb->SetInternalField(1, v8::External::New(this)); @@ -389,9 +381,46 @@ JsPluginContext(const Utf8String & pluginName, Function = v8::Persistent::New(FunctionJS::registerMe()); Procedure = v8::Persistent::New(ProcedureJS::registerMe()); CellValue = v8::Persistent::New(CellValueJS::registerMe()); + ExpressionValue = v8::Persistent::New(ExpressionValueJS::registerMe()); RandomNumberGenerator = v8::Persistent::New(RandomNumberGeneratorJS::registerMe()); } + +/*****************************************************************************/ +/* JS PLUGIN CONTEXT */ +/*****************************************************************************/ + +JsPluginContext:: +JsPluginContext(const Utf8String & pluginName, + MldbServer * server, + std::shared_ptr pluginResource) + : JsIsolate(false /* for this thread only */), + JsThreadContext(*this, server, pluginName), + pluginResource(pluginResource) +{ + using namespace v8; + + v8::Locker locker(this->isolate.isolate); + v8::Isolate::Scope isolateScope(this->isolate.isolate); + + HandleScope handle_scope; + + // Enter the created context for compiling and + // running the hello world script. + Context::Scope context_scope(context); + + v8::Local globalPrototype + = v8::Local::Cast(context->Global()->GetPrototype()); + + auto plugin = JsPluginJS::registerMe()->NewInstance(); + plugin->SetInternalField(0, v8::External::New(this)); + plugin->SetInternalField(1, v8::External::New(this)); + if (pluginResource) + plugin->Set(String::New("args"), + JS::toJS(jsonEncode(pluginResource->args))); + globalPrototype->Set(String::New("plugin"), plugin); +} + JsPluginContext:: ~JsPluginContext() { diff --git a/soa/js/js_utils.cc b/plugins/lang/js/js_utils.cc similarity index 100% rename from soa/js/js_utils.cc rename to plugins/lang/js/js_utils.cc diff --git a/soa/js/js_utils.h b/plugins/lang/js/js_utils.h similarity index 100% rename from soa/js/js_utils.h rename to plugins/lang/js/js_utils.h diff --git a/soa/js/js_value.cc b/plugins/lang/js/js_value.cc similarity index 100% rename from soa/js/js_value.cc rename to plugins/lang/js/js_value.cc diff --git a/soa/js/js_value.h b/plugins/lang/js/js_value.h similarity index 100% rename from soa/js/js_value.h rename to plugins/lang/js/js_value.h diff --git a/soa/js/js_value_fwd.h b/plugins/lang/js/js_value_fwd.h similarity index 100% rename from soa/js/js_value_fwd.h rename to plugins/lang/js/js_value_fwd.h diff --git a/plugins/lang/js/mldb_js.cc b/plugins/lang/js/mldb_js.cc index 1d78a3e51..2bc7dfc5f 100644 --- a/plugins/lang/js/mldb_js.cc +++ b/plugins/lang/js/mldb_js.cc @@ -46,7 +46,7 @@ struct CellValueJS::Methods { v8::Handle CellValueJS:: -create(CellValue value, JsPluginContext * context) +create(CellValue value, JsThreadContext * context) { auto obj = context->CellValue->GetFunction()->NewInstance(); auto * wrapped = new CellValueJS(); @@ -78,6 +78,137 @@ registerMe() } +/*****************************************************************************/ +/* EXPRESSION VALUE JS */ +/*****************************************************************************/ + +struct ExpressionValueJS::Methods { + static v8::Handle + toJs(const v8::Arguments & args) + { + try { + auto & val = getShared(args.This()); + return JS::toJS(val.extractJson()); + } HANDLE_JS_EXCEPTIONS; + } + + static v8::Handle + when(const v8::Arguments & args) + { + try { + auto & val = getShared(args.This()); + return JS::toJS(val.getEffectiveTimestamp()); + } HANDLE_JS_EXCEPTIONS; + } + + static v8::Handle + at(const v8::Arguments & args) + { + try { + auto val = getShared(args.This()); + Date newTs = JS::getArg(args, 0, "New effective timestamp"); + val.setEffectiveTimestamp(newTs); + return JS::toJS(val); + } HANDLE_JS_EXCEPTIONS; + } + + static v8::Handle + columns(const v8::Arguments & args) + { + try { + auto & val = getShared(args.This()); + + v8::HandleScope scope; + v8::Local obj= v8::Object::New(); + + auto onColumn = [&] (const PathElement & col, + const ExpressionValue & val) + { + obj->Set(JS::toJS(col), + JS::toJS(val)); + return true; + }; + + val.forEachColumn(onColumn); + return scope.Close(obj); + } HANDLE_JS_EXCEPTIONS; + } + + +#if 0 + static v8::Handle + getNamed(v8::Local property, + const v8::AccessorInfo& info) + { + try { + cerr << "get named" << endl; + auto val = getShared(info.This()); + return v8::Undefined(); + } HANDLE_JS_EXCEPTIONS; + } + + static v8::Handle + queryNamed(v8::Local property, + const v8::AccessorInfo& info) + { + cerr << "query named" << endl; + auto val = getShared(info.This()); + abort(); + } + + static v8::Handle + enumerateNamed(const v8::AccessorInfo& info) + { + cerr << "enumerate named" << endl; + auto val = getShared(info.This()); + abort(); + } +#endif +}; + +v8::Handle +ExpressionValueJS:: +create(ExpressionValue value, JsThreadContext * context) +{ + auto obj = context->ExpressionValue->GetFunction()->NewInstance(); + auto * wrapped = new ExpressionValueJS(); + wrapped->val = std::move(value); + wrapped->wrap(obj, context); + return obj; +} + +ExpressionValue & +ExpressionValueJS:: +getShared(const v8::Handle & val) +{ + return reinterpret_cast + (v8::Handle::Cast + (val->GetInternalField(0))->Value())->val; +} + +v8::Local +ExpressionValueJS:: +registerMe() +{ + using namespace v8; + + HandleScope scope; + + auto fntmpl = CreateFunctionTemplate("ExpressionValue"); + auto objtmpl = fntmpl->PrototypeTemplate(); + + objtmpl->Set(String::New("toJs"), FunctionTemplate::New(Methods::toJs)); + objtmpl->Set(String::New("when"), FunctionTemplate::New(Methods::when)); + objtmpl->Set(String::New("at"), FunctionTemplate::New(Methods::at)); + objtmpl->Set(String::New("columns"), FunctionTemplate::New(Methods::columns)); + //objtmpl->SetNamedPropertyHandler(Methods::getNamed, nullptr, + // Methods::queryNamed, nullptr, + // Methods::enumerateNamed); + + return scope.Close(fntmpl); +} + + /*****************************************************************************/ /* STREAM JS */ /*****************************************************************************/ @@ -390,7 +521,7 @@ struct StreamJS::Methods { v8::Handle StreamJS:: -create(std::shared_ptr stream, JsPluginContext * context) +create(std::shared_ptr stream, JsThreadContext * context) { auto obj = context->Stream->GetFunction()->NewInstance(); auto * wrapped = new StreamJS(); @@ -531,7 +662,7 @@ struct RandomNumberGeneratorJS::Methods { v8::Handle RandomNumberGeneratorJS:: create(std::shared_ptr randomNumberGenerator, - JsPluginContext * context) + JsThreadContext * context) { auto obj = context->RandomNumberGenerator->GetFunction()->NewInstance(); auto * wrapped = new RandomNumberGeneratorJS(); @@ -578,7 +709,7 @@ struct MldbJS::Methods { openStream(const v8::Arguments & args) { try { - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); auto stream = std::make_shared(JS::cstr(args[0])); return StreamJS::create(stream, context); @@ -589,7 +720,7 @@ struct MldbJS::Methods { createDataset(const v8::Arguments & args) { try { - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); MldbServer * server = MldbJS::getShared(args.This()); Json::Value configJson = JS::getArg(args, 0, "Config"); PolyConfig config = jsonDecode(configJson); @@ -617,7 +748,7 @@ struct MldbJS::Methods { createFunction(const v8::Arguments & args) { try { - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); MldbServer * server = MldbJS::getShared(args.This()); Json::Value configJson = JS::getArg(args, 0, "Config"); PolyConfig config = jsonDecode(configJson); @@ -645,7 +776,7 @@ struct MldbJS::Methods { createProcedure(const v8::Arguments & args) { try { - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); MldbServer * server = MldbJS::getShared(args.This()); Json::Value configJson = JS::getArg(args, 0, "Config"); PolyConfig config = jsonDecode(configJson); @@ -673,7 +804,7 @@ struct MldbJS::Methods { createRandomNumberGenerator(const v8::Arguments & args) { try { - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); int seed = JS::getArg(args, random(), 0.0, "Seed of generator"); auto rng = std::make_shared(seed); @@ -903,7 +1034,7 @@ struct MldbJS::Methods { log(const v8::Arguments & args) { using namespace v8; - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); try { Utf8String line; @@ -918,8 +1049,18 @@ struct MldbJS::Methods { printed = CellValue(Date::fromSecondsSinceEpoch(args[i]->NumberValue() * 0.001)).toUtf8String(); } else if (args[i]->IsObject()) { - Json::Value val = JS::fromJS(args[i]); - printed = val.toStyledString(); + if (context->ExpressionValue->HasInstance(args[i])) { + printed = ExpressionValueJS::getShared(JS::toObject(args[i])) + .extractJson().toStyledString(); + } + else if (context->CellValue->HasInstance(args[i])) { + printed = jsonEncodeStr + (CellValueJS::getShared(JS::toObject(args[i]))); + } + else { + Json::Value val = JS::fromJS(args[i]); + printed = val.toStyledString(); + } } else if (args[i]->IsArray()) { Json::Value val = JS::fromJS(args[i]); @@ -950,7 +1091,7 @@ struct MldbJS::Methods { createInterval(const v8::Arguments & args) { using namespace v8; - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); v8::HandleScope scope; try { @@ -1005,7 +1146,7 @@ struct MldbJS::Methods { createPath(const v8::Arguments & args) { using namespace v8; - JsPluginContext * context = MldbJS::getContext(args.This()); + JsThreadContext * context = MldbJS::getContext(args.This()); v8::HandleScope scope; try { @@ -1090,11 +1231,11 @@ getShared(const v8::Handle & val) (val->GetInternalField(0))->Value()); } -JsPluginContext * +JsThreadContext * MldbJS:: getContext(const v8::Handle & val) { - return reinterpret_cast + return reinterpret_cast (v8::Handle::Cast (val->GetInternalField(1))->Value()); } diff --git a/plugins/lang/js/mldb_js.h b/plugins/lang/js/mldb_js.h index 474649a37..e6476b393 100644 --- a/plugins/lang/js/mldb_js.h +++ b/plugins/lang/js/mldb_js.h @@ -11,6 +11,7 @@ #include "js_common.h" #include "mldb/sql/cell_value.h" +#include "mldb/sql/expression_value.h" namespace Datacratic { namespace MLDB { @@ -26,7 +27,7 @@ struct CellValueJS: public JsObjectBase { CellValue val; static v8::Handle - create(CellValue value, JsPluginContext * context); + create(CellValue value, JsThreadContext * context); static CellValue & getShared(const v8::Handle & val); @@ -38,6 +39,26 @@ struct CellValueJS: public JsObjectBase { }; +/*****************************************************************************/ +/* EXPRESSION VALUE JS */ +/*****************************************************************************/ + +struct ExpressionValueJS: public JsObjectBase { + ExpressionValue val; + + static v8::Handle + create(ExpressionValue value, JsThreadContext * context); + + static ExpressionValue & + getShared(const v8::Handle & val); + + static v8::Local + registerMe(); + + struct Methods; +}; + + /*****************************************************************************/ /* STREAM JS */ /*****************************************************************************/ @@ -49,7 +70,7 @@ struct StreamJS: public JsObjectBase { std::shared_ptr stream; static v8::Handle - create(std::shared_ptr stream, JsPluginContext * context); + create(std::shared_ptr stream, JsThreadContext * context); static std::istream * getShared(const v8::Handle & val); @@ -78,7 +99,7 @@ struct RandomNumberGeneratorJS: public JsObjectBase { static v8::Handle create(std::shared_ptr randomNumberGenerator, - JsPluginContext * context); + JsThreadContext * context); static RandomNumberGenerator * getShared(const v8::Handle & val); @@ -101,12 +122,12 @@ struct MldbJS: public JsObjectBase { std::shared_ptr mldb; static v8::Handle - create(std::shared_ptr mldb, JsPluginContext * context); + create(std::shared_ptr mldb, JsThreadContext * context); static MldbServer * getShared(const v8::Handle & val); - static JsPluginContext * getContext(const v8::Handle & val); + static JsThreadContext * getContext(const v8::Handle & val); static v8::Local registerMe(); diff --git a/plugins/lang/js/mldb_js_plugin.mk b/plugins/lang/js/mldb_js_plugin.mk index b4546ebe8..ab8a7f774 100644 --- a/plugins/lang/js/mldb_js_plugin.mk +++ b/plugins/lang/js/mldb_js_plugin.mk @@ -1,5 +1,17 @@ # This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved. +# JS support library + +LIBJS_SOURCES := \ + js_value.cc \ + js_utils.cc + +LIBJS_LINK := jsoncpp $(V8_LIB) arch utils types + +$(eval $(call library,js,$(LIBJS_SOURCES),$(LIBJS_LINK))) + +$(eval $(call include_sub_make,js_testing,testing,js_testing.mk)) + # JS plugin loader $(eval $(call library,mldb_js_plugin,js_loader.cc js_function.cc js_common.cc mldb_js.cc dataset_js.cc function_js.cc procedure_js.cc,value_description js sql_expression)) diff --git a/plugins/lang/js/procedure_js.cc b/plugins/lang/js/procedure_js.cc index 4701469ff..454b72b47 100644 --- a/plugins/lang/js/procedure_js.cc +++ b/plugins/lang/js/procedure_js.cc @@ -7,7 +7,7 @@ #include "procedure_js.h" #include "mldb/core/procedure.h" -#include "mldb/types/js/id_js.h" +#include "id_js.h" using namespace std; @@ -23,7 +23,7 @@ namespace MLDB { v8::Handle ProcedureJS:: -create(std::shared_ptr procedure, JsPluginContext * context) +create(std::shared_ptr procedure, JsThreadContext * context) { auto obj = context->Procedure->GetFunction()->NewInstance(); auto * wrapped = new ProcedureJS(); diff --git a/plugins/lang/js/procedure_js.h b/plugins/lang/js/procedure_js.h index 177ccf4e9..8ae928388 100644 --- a/plugins/lang/js/procedure_js.h +++ b/plugins/lang/js/procedure_js.h @@ -25,7 +25,7 @@ struct ProcedureJS: public JsObjectBase { std::shared_ptr procedure; static v8::Handle - create(std::shared_ptr procedure, JsPluginContext * context); + create(std::shared_ptr procedure, JsThreadContext * context); static Procedure * getShared(const v8::Handle & val); diff --git a/soa/js/testing/js_call_test.cc b/plugins/lang/js/testing/js_call_test.cc similarity index 94% rename from soa/js/testing/js_call_test.cc rename to plugins/lang/js/testing/js_call_test.cc index 028b7f828..f7a2555b2 100644 --- a/soa/js/testing/js_call_test.cc +++ b/plugins/lang/js/testing/js_call_test.cc @@ -6,7 +6,7 @@ */ -#include "mldb/soa/js/js_call.h" +#include "mldb/plugins/lang/js/js_call.h" using namespace Datacratic::JS; using namespace Datacratic; diff --git a/soa/js/testing/js_testing.mk b/plugins/lang/js/testing/js_testing.mk similarity index 100% rename from soa/js/testing/js_testing.mk rename to plugins/lang/js/testing/js_testing.mk diff --git a/types/js/url_js.h b/plugins/lang/js/url_js.h similarity index 93% rename from types/js/url_js.h rename to plugins/lang/js/url_js.h index cf1f6fb49..a1788c495 100644 --- a/types/js/url_js.h +++ b/plugins/lang/js/url_js.h @@ -6,7 +6,7 @@ #pragma once -#include "mldb/soa/js/js_utils.h" +#include "mldb/plugins/lang/js/js_utils.h" #include "mldb/types/url.h" namespace Datacratic { diff --git a/server/plugin_resource.h b/server/plugin_resource.h index 17214df4e..358230b73 100644 --- a/server/plugin_resource.h +++ b/server/plugin_resource.h @@ -1,18 +1,16 @@ -// This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved. - /** -*- C++ -*- - plugin_resource.cc + plugin_resource.h Francois Maillet, 18 fevrier 2015 - Copyright (c) 2015 Datacratic Inc. All rights reserved. + Copyright (c) 2015 Datacratic Inc. All rights reserved. + This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved. - Transparent resource getter for plugins + Transparent resource getter for plugins */ #pragma once #include "mldb/rest/poly_entity.h" #include "mldb/types/any.h" -#include "mldb/soa/js/js_utils.h" #include "mldb/core/plugin.h" #include "mldb/types/value_description_fwd.h" #include "mldb/types/url.h" diff --git a/soa/js/js.mk b/soa/js/js.mk deleted file mode 100644 index 2ba67f958..000000000 --- a/soa/js/js.mk +++ /dev/null @@ -1,17 +0,0 @@ -# This file is part of MLDB. Copyright 2015 Datacratic. All rights reserved. - -# js.mk -# Jeremy Barnes, 11 May 2010 -# Copyright (c) 2010 Datacratic. All rights reserved. -# -# Support functions for javascript - -LIBJS_SOURCES := \ - js_value.cc \ - js_utils.cc - -LIBJS_LINK := jsoncpp $(V8_LIB) arch utils types - -$(eval $(call library,js,$(LIBJS_SOURCES),$(LIBJS_LINK))) - -$(eval $(call include_sub_make,js_testing,testing,js_testing.mk)) diff --git a/soa/soa.mk b/soa/soa.mk index e4557df81..d61dca16e 100644 --- a/soa/soa.mk +++ b/soa/soa.mk @@ -2,7 +2,6 @@ # SOA makefile -$(eval $(call include_sub_make,js)) $(eval $(call include_sub_make,credentials)) $(eval $(call include_sub_make,service)) $(eval $(call include_sub_make,utils)) diff --git a/testing/MLDB-1190_segfault_sqlexpr_jseval.py b/testing/MLDB-1190_segfault_sqlexpr_jseval.py index be9b0e6a4..628c0e500 100644 --- a/testing/MLDB-1190_segfault_sqlexpr_jseval.py +++ b/testing/MLDB-1190_segfault_sqlexpr_jseval.py @@ -47,7 +47,7 @@ [ "msgStats.msgLen", 28, - "-Inf" + "NaD" ], [ "words.I", diff --git a/testing/MLDB-704-jseval-row.js b/testing/MLDB-704-jseval-row.js index b923b92d8..e0aaf7577 100644 --- a/testing/MLDB-704-jseval-row.js +++ b/testing/MLDB-704-jseval-row.js @@ -40,7 +40,7 @@ dataset.commit() var res1 = mldb.get("/v1/datasets/test/query", { - select: "jseval('mldb.log(''Hello '' + x); return { x: x, y: ''yes''}', 'x', x) AS *", + select: "jseval('mldb.log(''Hello '' + x.toJs()); return { x: x, y: ''yes''}', '!x', x) AS *", format: 'table', orderBy: 'rowName()' }); @@ -57,8 +57,38 @@ var expected = [ assertEqual(res1.json, expected, "row output from JS function"); +var res1 = mldb.get("/v1/datasets/test/query", + { + select: "jseval('mldb.log(''Hello '' + x); return { x: x, y: ''yes''}', 'x', x) AS *", + format: 'table', + orderBy: 'rowName()' + }); + +plugin.log(res1); + +assertEqual(res1.json, expected, "row output from JS function"); + // MLDB-757 +var res2 = mldb.get("/v1/datasets/test/query", + { + select: "jseval('return Object.keys(x.columns()).length', '!x', {*}) AS nvals", + format: 'table', + orderBy: 'rowName()' + }); + +plugin.log(res2); + +var expected2 = [ + [ "_rowName", "nvals" ], + [ "ex1", 2 ], + [ "ex2", 3 ], + [ "ex3", 2 ], + [ "ex4", 3 ] +]; + +assertEqual(res2.json, expected2, "row input to JS function"); + var res2 = mldb.get("/v1/datasets/test/query", { select: "jseval('return Object.keys(x).length', 'x', {*}) AS nvals", @@ -100,4 +130,19 @@ var expected3 = [ assertEqual(res3.json, expected3, "undefined output of JS function"); +var res4 = mldb.query("SELECT jseval('return row;', '!row', {*}) AS * NAMED 'res' FROM (SELECT x:1, y:2)"); + +expected4 = [ + { + "columns" : [ + [ "x", 1, "-Inf" ], + [ "y", 2, "-Inf" ] + ], + "rowHash" : "4ba25cf9b5244b88", + "rowName" : "res" + } +]; + +assertEqual(res4, expected4); + "success" diff --git a/types/date.cc b/types/date.cc index 1bcfda9c9..e176aeac9 100644 --- a/types/date.cc +++ b/types/date.cc @@ -11,7 +11,6 @@ #include #include #include "mldb/arch/format.h" -#include "mldb/soa/js/js_value.h" #include "mldb/ext/jsoncpp/json.h" #include "mldb/base/parse_context.h" #include diff --git a/types/string.cc b/types/string.cc index 1c9477181..f1080cee6 100644 --- a/types/string.cc +++ b/types/string.cc @@ -7,7 +7,6 @@ */ #include "string.h" -#include "mldb/soa/js/js_value.h" #include "mldb/ext/jsoncpp/json.h" #include #include "mldb/arch/exception.h"