diff --git a/exaudfclient/.bazelrc b/exaudfclient/.bazelrc index fe0d63ac4..f6e589f30 100644 --- a/exaudfclient/.bazelrc +++ b/exaudfclient/.bazelrc @@ -25,4 +25,5 @@ build:valgrind --copt -O1 build:valgrind -c dbg build:valgrind --copt -g build:valgrind --strip=never -build:valgrind --copt -DVALGRIND_ACTIVE \ No newline at end of file +build:valgrind --copt -DVALGRIND_ACTIVE +build:swig-string-as-buffer --copt='-DSWIG_STRING_AS_BYTES_ENABLED=1' --action_env=SWIG_STRING_AS_BYTES_ENABLED=1 \ No newline at end of file diff --git a/exaudfclient/base/BUILD b/exaudfclient/base/BUILD index 357c34da9..3a2c87f15 100644 --- a/exaudfclient/base/BUILD +++ b/exaudfclient/base/BUILD @@ -2,7 +2,8 @@ package(default_visibility = ["//visibility:public"]) exports_files(["filter_swig_code.py", "build_integrated.py", "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh", - "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh"]) + "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh", + "add_encoding_decoding.py"]) load("//:variables.bzl", "VM_ENABLED_DEFINES") diff --git a/exaudfclient/base/add_encoding_decoding.py b/exaudfclient/base/add_encoding_decoding.py new file mode 100644 index 000000000..063ab2baf --- /dev/null +++ b/exaudfclient/base/add_encoding_decoding.py @@ -0,0 +1,29 @@ +import os +import sys +from pathlib import Path + +CONVERT_TO_BYTES = """ +decodeUTF8 = lambda x: x.decode(encoding='utf-8') if isinstance(x, bytes) else x +encodeUTF8 = lambda x: x.encode(encoding='utf-8') if isinstance(x, str) else x +""" + +IDENTITY = """ +decodeUTF8 = lambda x: x +encodeUTF8 = lambda x: x +""" + +def add_encoding_decoding(target): + if "SWIG_STRING_AS_BYTES_ENABLED" in os.environ: + decoding_encdecoding = CONVERT_TO_BYTES + else: + decoding_encdecoding = IDENTITY + + with open(target, "wt", encoding="utf-8") as f: + f.write(decoding_encdecoding) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print('Usage: add_encoding_decoding.py target') + sys.exit(1) + add_encoding_decoding(sys.argv[1]) diff --git a/exaudfclient/base/exaudflib/swig/swig_common.h b/exaudfclient/base/exaudflib/swig/swig_common.h index b4e463fed..8080e5916 100644 --- a/exaudfclient/base/exaudflib/swig/swig_common.h +++ b/exaudfclient/base/exaudflib/swig/swig_common.h @@ -1,6 +1,7 @@ #ifndef SWIG_COMMON_H #define SWIG_COMMON_H +#include #include #include #include diff --git a/exaudfclient/base/python/exascript_python_preset_core.py b/exaudfclient/base/python/exascript_python_preset_core.py index 550b31949..5062457cb 100644 --- a/exaudfclient/base/python/exascript_python_preset_core.py +++ b/exaudfclient/base/python/exascript_python_preset_core.py @@ -1,19 +1,17 @@ import sys unicode = str -decodeUTF8 = lambda x: x -encodeUTF8 = lambda x: x from exascript_python import * import decimal import datetime -import imp +import types class ExaUDFError(Exception): pass def clean_stacktrace_line(line): import re - match = re.match("""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line) + match = re.match(r"""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line) if match is not None: filename=match.group(1) lineno=match.group(2) @@ -83,14 +81,14 @@ def ci(x, tbl): colprec = self.__meta.inputColumnPrecision(x) colscale = self.__meta.inputColumnScale(x) colsize = self.__meta.inputColumnSize(x) - coltn = self.__meta.inputColumnTypeName(x) + coltn = decodeUTF8(self.__meta.inputColumnTypeName(x)) elif tbl == 'output': colname = decodeUTF8(self.__meta.outputColumnName(x)) coltype = self.__meta.outputColumnType(x) colprec = self.__meta.outputColumnPrecision(x) colscale = self.__meta.outputColumnScale(x) colsize = self.__meta.outputColumnSize(x) - coltn = self.__meta.outputColumnTypeName(x) + coltn = decodeUTF8(self.__meta.outputColumnTypeName(x)) class exacolumn: def __init__(self, cn, ct, st, cp, cs, l): self.name = cn @@ -126,7 +124,7 @@ def import_script(self, script): modobj = self.__modules[str(code)] else: print("%%% new code", modname, repr(code), code in self.__modules) - modobj = imp.new_module(modname) + modobj = types.ModuleType(modname) modobj.__file__ = "<%s>" % modname modobj.__dict__['exa'] = self self.__modules[str(code)] = modobj diff --git a/exaudfclient/base/python/exascript_python_wrap.py b/exaudfclient/base/python/exascript_python_wrap.py index 00ce02a8d..ebb26dd48 100644 --- a/exaudfclient/base/python/exascript_python_wrap.py +++ b/exaudfclient/base/python/exascript_python_wrap.py @@ -3,8 +3,6 @@ pyextdataframe_pkg = None unicode = str -decodeUTF8 = lambda x: x -encodeUTF8 = lambda x: x long = int if 'LIBPYEXADATAFRAME_DIR' in os.environ: @@ -45,10 +43,10 @@ def resget(): return val return resget def convert_date(x): - val = datetime.datetime.strptime(x, "%Y-%m-%d") + val = datetime.datetime.strptime(decodeUTF8(x), "%Y-%m-%d") return datetime.date(val.year, val.month, val.day) def convert_timestamp(x): - return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f") + return datetime.datetime.strptime(decodeUTF8(x), "%Y-%m-%d %H:%M:%S.%f") self.__incoltypes = [] for col in range(self.__meta.inputColumnCount()): self.__incoltypes.append(self.__meta.inputColumnType(col)) @@ -67,8 +65,8 @@ def convert_timestamp(x): data[colname] = rd(inp.getInt64, inp.wasNull, col) elif self.__incoltypes[col] == NUMERIC: if self.__meta.inputColumnScale(col) == 0: - data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: int(str(x))) - else: data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: decimal.Decimal(str(x))) + data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: int(decodeUTF8(x))) + else: data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: decimal.Decimal(decodeUTF8(x))) elif self.__incoltypes[col] == DATE: data[colname] = rd(inp.getDate, inp.wasNull, col, convert_date) elif self.__incoltypes[col] == TIMESTAMP: @@ -138,7 +136,7 @@ def emit(self, *output): elif type(v) in (int, long): if self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) - elif self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, str(int(v))) + elif self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, encodeUTF8(str(int(v)))) elif self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) else: raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1091: emit column '%s' is of type %s but data given have type %s" \ @@ -147,7 +145,7 @@ def emit(self, *output): if self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) elif self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) - elif self.__outcoltypes[k] == NUMERIC: self.__out.setInt64(k, str(v)) + elif self.__outcoltypes[k] == NUMERIC: self.__out.setInt64(k, encodeUTF8(str(v))) else: raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1092: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) @@ -156,7 +154,7 @@ def emit(self, *output): raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1093: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) self.__out.setBoolean(k, bool(v)) - elif type(v) in (str, unicode): + elif type(v) in (str, unicode, bytes): v = encodeUTF8(v) vl = len(v) if self.__outcoltypes[k] != STRING: @@ -164,7 +162,7 @@ def emit(self, *output): % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) self.__out.setString(k, v, vl) elif type(v) == decimal.Decimal: - if self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, str(v)) + if self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, encodeUTF8(str(v))) elif self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) elif self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) @@ -175,12 +173,12 @@ def emit(self, *output): if self.__outcoltypes[k] != DATE: raise RuntimeError("E-UDF-CL-SL-PYTHON-1096: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setDate(k, v.isoformat()) + self.__out.setDate(k, encodeUTF8(v.isoformat())) elif type(v) == datetime.datetime: if self.__outcoltypes[k] != TIMESTAMP: raise RuntimeError("E-UDF-CL-SL-PYTHON-1097: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setTimestamp(k, v.isoformat(' ')) + self.__out.setTimestamp(k, encodeUTF8(v.isoformat(' '))) else: raise RuntimeError("E-UDF-CL-SL-PYTHON-1098: data type %s is not supported" % str(type(v))) msg = self.__out.checkException() if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1099: "+msg) diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index 9748ef7a1..a82c2a127 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -10,7 +10,7 @@ genrule( swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i """, outs = ["exascript_python_tmp.cc", "exascript_python.py"], - srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"] + srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"], ) genrule( @@ -35,14 +35,23 @@ genrule( tools = ["//base/python:extend_exascript_python_preset_py.sh"] ) +genrule( + name = "gen_encoding_decoding", + cmd = 'python3 "$(location //base:add_encoding_decoding.py)" "$(location exascript_encoding_decoding.py)"', + outs = ["exascript_encoding_decoding.py"], + srcs = [], + tools = ["//base:add_encoding_decoding.py"] +) + genrule( name = "exascript_python_int", cmd = """ cp $(SRCS) . - python3 $(location //base:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" + python3 $(location //base:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" "exascript_encoding_decoding.py" """, outs = ["exascript_python_int.h"], - srcs = [":exascript_python_tmp_cc", "//base/python:exascript_python_wrap.py", ":extend_exascript_python_preset_py"], + srcs = [":exascript_python_tmp_cc", "//base/python:exascript_python_wrap.py", + ":extend_exascript_python_preset_py", ":gen_encoding_decoding"], tools = ["//base:build_integrated.py"] ) diff --git a/exaudfclient/base/python/python3/python_ext_dataframe.cc b/exaudfclient/base/python/python3/python_ext_dataframe.cc index 73051398c..da7d31301 100644 --- a/exaudfclient/base/python/python3/python_ext_dataframe.cc +++ b/exaudfclient/base/python/python3/python_ext_dataframe.cc @@ -175,6 +175,23 @@ inline void checkPyObjIsNotNull(const PyObject *obj) { throw std::runtime_error("F-UDF-CL-SL-PYTHON-1142"); } +inline PyObject* convertPyStringToPyBytes(PyObject* pyString) { +#ifdef SWIG_STRING_AS_BYTES_ENABLED + return PyObject_CallMethod(pyString, "encode", NULL); +#else + Py_INCREF(pyString); + return pyString; +#endif +} + +inline PyObject* convertPyBytesToPyString(PyObject* pyString) { +#ifdef SWIG_STRING_AS_BYTES_ENABLED + return PyObject_CallMethod(pyString, "decode", NULL); +#else + Py_INCREF(pyString); + return pyString; +#endif +} struct ColumnInfo @@ -923,8 +940,11 @@ inline void handleEmitPyInt( break; } case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyInt)); + { + PyPtr pyStrInt(PyObject_Str(pyInt)); + pyValue.reset(convertPyStringToPyBytes(pyStrInt.get())); break; + } case SWIGVMContainers::DOUBLE: { double value = PyFloat_AsDouble(pyInt); @@ -977,8 +997,11 @@ inline void handleEmitPyFloat( break; } case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyFloat)); + { + PyPtr pyStrFloat(PyObject_Str(pyFloat)); + pyValue.reset(convertPyStringToPyBytes(pyStrFloat.get())); break; + } case SWIGVMContainers::DOUBLE: { //pyFloat points to a 'borrowed' reference. We need to explicitly increase the ref counter here, as pyValue will decrease it again later. @@ -1020,15 +1043,21 @@ inline void handleEmitPyDecimal( case SWIGVMContainers::INT64: case SWIGVMContainers::INT32: { + std::cout << "handleEmitPyDecimal: INT" << std::endl; PyPtr pyInt(PyObject_CallMethodObjArgs(pyDecimal, pyIntMethodName.get(), NULL)); pyValue.reset(pyInt.release()); break; } case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyDecimal)); + { + std::cout << "handleEmitPyDecimal: NUMERIC" << std::endl; + PyPtr pyStrDecimal(PyObject_Str(pyDecimal)); + pyValue.reset(convertPyStringToPyBytes(pyStrDecimal.get())); break; + } case SWIGVMContainers::DOUBLE: { + std::cout << "handleEmitPyDecimal: DOUBLE" << std::endl; PyPtr pyFloat(PyObject_CallMethodObjArgs(pyDecimal, pyFloatMethodName.get(), NULL)); pyValue.reset(pyFloat.release()); break; @@ -1067,11 +1096,12 @@ inline void handleEmitPyStr( case SWIGVMContainers::STRING: { Py_ssize_t size = -1; + PyPtr bytesString(convertPyStringToPyBytes(pyString)); const char *str = PyUnicode_AsUTF8AndSize(pyString, &size); if (!str && size < 0) throw std::runtime_error("F-UDF-CL-SL-PYTHON-1137: invalid size of string"); PyPtr pySize(PyLong_FromSsize_t(size)); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyString, pySize.get(), NULL)); + pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesString.get(), pySize.get(), NULL)); break; } default: @@ -1105,7 +1135,8 @@ inline void handleEmitPyDate( case SWIGVMContainers::DATE: { PyPtr pyIsoDate(PyObject_CallMethodObjArgs(pyDate, pyIsoformatMethodName.get(), NULL)); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDate.get(), NULL)); + PyPtr bytesIsoDate(convertPyStringToPyBytes(pyIsoDate.get())); + pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesIsoDate.get(), NULL)); break; } default: @@ -1141,8 +1172,9 @@ inline void handleEmitPyTimestamp( // it to the generated string. PyPtr pyTzLocalize(PyObject_CallMethod(pyTimestamp, "tz_localize", "z", NULL)); PyPtr pyIsoDatetime(PyObject_CallMethod(pyTzLocalize.get(), "isoformat", "s", " ")); + PyPtr bytesDateTime(convertPyStringToPyBytes(pyIsoDatetime.get())); pyResult.reset(PyObject_CallMethodObjArgs( - resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDatetime.get(), NULL)); + resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesDateTime.get(), NULL)); break; } default: @@ -1177,8 +1209,9 @@ inline void handleEmitNpyDateTime( case SWIGVMContainers::TIMESTAMP: { PyPtr pyIsoDatetime(PyObject_CallMethod(pyTimestamp, "isoformat", "s", " ")); + PyPtr bytesDateTime(convertPyStringToPyBytes(pyIsoDatetime.get())); pyResult.reset(PyObject_CallMethodObjArgs( - resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDatetime.get(), NULL)); + resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesDateTime.get(), NULL)); break; } default: @@ -1377,7 +1410,8 @@ void emit(PyObject *resultHandler, std::vector& colInfo, PyObject *d PyPtr pyCheckException(PyObject_CallMethodObjArgs(resultHandler, pyCheckExceptionMethodName.get(), NULL)); if (pyCheckException.get() != Py_None) { - const char *exMsg = PyUnicode_AsUTF8(pyCheckException.get()); + PyPtr exceptionAsPyBytes(convertPyBytesToPyString(pyCheckException.get())); + const char *exMsg = PyUnicode_AsUTF8(exceptionAsPyBytes.get()); if (exMsg) { std::stringstream ss; ss << "F-UDF-CL-SL-PYTHON-1075: emit(): " << exMsg; diff --git a/exaudfclient/base/python/pythoncontainer.cc b/exaudfclient/base/python/pythoncontainer.cc index a52b29f52..171cf2930 100644 --- a/exaudfclient/base/python/pythoncontainer.cc +++ b/exaudfclient/base/python/pythoncontainer.cc @@ -194,16 +194,22 @@ PythonVMImpl::PythonVMImpl(bool checkOnly): m_checkOnly(checkOnly) check("F-UDF-CL-SL-PYTHON-1010"); if (exatable == NULL) throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1011: Failed to import code module"); + code = Py_CompileString(integrated_exascript_encoding_decoding_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1143"); + if (code == NULL) {check("F-UDF-CL-SL-PYTHON-1144");} + + PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1145"); + Py_DECREF(code); + code = Py_CompileString(integrated_exascript_python_preset_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1012"); if (code == NULL) {check("F-UDF-CL-SL-PYTHON-1013");} - PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1014"); + PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1014"); Py_DECREF(code); - PyObject *runobj = PyDict_GetItemString(globals, "__pythonvm_wrapped_parse"); check("F-UDF-CL-SL-PYTHON-1016"); - //PyObject *retvalue = PyObject_CallFunction(runobj, NULL); check(); - PyObject *retvalue = PyObject_CallFunctionObjArgs(runobj, globals, NULL); check("F-UDF-CL-SL-PYTHON-1017"); - Py_XDECREF(retvalue); retvalue = NULL; + PyObject *runobj = PyDict_GetItemString(globals, "__pythonvm_wrapped_parse"); check("F-UDF-CL-SL-PYTHON-1016"); + //PyObject *retvalue = PyObject_CallFunction(runobj, NULL); check(); + PyObject *retvalue = PyObject_CallFunctionObjArgs(runobj, globals, NULL); check("F-UDF-CL-SL-PYTHON-1017"); + Py_XDECREF(retvalue); retvalue = NULL; code = Py_CompileString(integrated_exascript_python_wrap_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1018"); if (code == NULL) throw PythonVM::exception("Failed to compile wrapping script"); @@ -489,6 +495,7 @@ const char* PythonVMImpl::singleCall(single_call_function_id_e fn, const Executi PyObject* p3str = PyUnicode_AsEncodedString(repr, "utf-8", "ignore"); const char *bytes = PyBytes_AS_STRING(p3str); singleCallResult = string(bytes); + std::cout << "pythoncontainer: singleCallResult='" << singleCallResult << "'" << std::endl; Py_XDECREF(retvalue); retvalue = NULL; return singleCallResult.c_str(); } diff --git a/ext/scripts/install_scripts/install_via_pip.pl b/ext/scripts/install_scripts/install_via_pip.pl index da93abea6..95ecdbf6b 100755 --- a/ext/scripts/install_scripts/install_via_pip.pl +++ b/ext/scripts/install_scripts/install_via_pip.pl @@ -40,6 +40,7 @@ =head1 SYNOPSIS my $use_deprecated_legacy_resolver = 0; my $ancestor_pip_package_root_path = ''; my $install_build_tools_ephemerally = 0; +my $pip_needs_break_system_packages = 0; GetOptions ( "help" => \$help, "dry-run" => \$dry_run, @@ -51,7 +52,8 @@ =head1 SYNOPSIS "use-deprecated-legacy-resolver" => \$use_deprecated_legacy_resolver, "python-binary=s" => \$python_binary, "ancestor-pip-package-root-path=s" => \$ancestor_pip_package_root_path, - "install-build-tools-ephemerally" => \$install_build_tools_ephemerally + "install-build-tools-ephemerally" => \$install_build_tools_ephemerally, + "pip-needs-break-system-packages" => \$pip_needs_break_system_packages ) or package_mgmt_utils::print_usage_and_abort(__FILE__,"Error in command line arguments",2); package_mgmt_utils::print_usage_and_abort(__FILE__,"",0) if $help; @@ -79,6 +81,10 @@ =head1 SYNOPSIS push @pip_parameters, "--use-deprecated=legacy-resolver"; } +if($pip_needs_break_system_packages){ + push @pip_parameters, "--break-system-packages"; +} + my $pip_parameters_str = join( ' ', @pip_parameters); my $element_separator = '\\|'; my $combining_template = "$python_binary -m pip install $pip_parameters_str --no-cache-dir <<<<0>>>>"; diff --git a/ext/scripts/tests/install_scripts/run_pip_tests.sh b/ext/scripts/tests/install_scripts/run_pip_tests.sh index b507202c5..c5c87954c 100644 --- a/ext/scripts/tests/install_scripts/run_pip_tests.sh +++ b/ext/scripts/tests/install_scripts/run_pip_tests.sh @@ -111,5 +111,10 @@ TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file te assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'" echo +echo ./install_via_pip.pl with pip-needs-break-system-packages +TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --pip-needs-break-system-packages --python-binary python3 "$DRY_RUN_OPTION") +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly' 'requests'" +echo + check_for_failed_tests echo "All pip tests passed" diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile index a18d69b23..992344671 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile @@ -13,7 +13,7 @@ RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfc RUN addgroup --gid 1000 exasolution RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" -RUN addgroup --gid 500 exausers +RUN addgroup --gid 500 exausers RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" ENV LANG en_US.UTF-8 diff --git a/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md b/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md new file mode 100644 index 000000000..7398e1a51 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md @@ -0,0 +1,14 @@ +# Details for template-Exasol-all-python-3.12 + +## Packages + +**Remarks:** The packages are grouped by there main usage and their type. + +- [UDFclient dependencies](flavor_base/udfclient_deps/packages/apt_get_packages) +- [Language dependencies](flavor_base/language_deps/packages/apt_get_packages) +- Flavor packages + - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages) + - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages) +- Customization + - [Ubuntu packages](flavor_customization/packages/apt_get_packages) + - [Python3 pip packages](flavor_customization/packages/python3_pip_packages) diff --git a/flavors/template-Exasol-all-python-3.12/ci.json b/flavors/template-Exasol-all-python-3.12/ci.json new file mode 100644 index 000000000..1bda959cd --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/ci.json @@ -0,0 +1,32 @@ +{ + "build_runner": "ubuntu-24.04", + "test_config": { + "default_test_runner": "ubuntu-24.04", + "test_sets": [ + { + "name": "python", + "folders": ["python3/all"], + "goal": "release", + "generic_language_tests": [] + }, + { + "name": "pandas", + "folders": ["pandas/all", "pandas/pandas2"], + "goal": "release", + "generic_language_tests": [] + }, + { + "name": "linker_namespace", + "folders": ["linker_namespace_sanity"], + "goal": "base_test_build_run", + "generic_language_tests": [] + }, + { + "name": "generic", + "folders": [], + "goal": "release", + "generic_language_tests": ["python3"] + } + ] + } +} diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile new file mode 100644 index 000000000..f289dc767 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile @@ -0,0 +1,41 @@ +FROM {{language_deps}} + +RUN mkdir /conf /buckets + +COPY --from={{base_test_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /env /env +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY /exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries --config swig-string-as-buffer"] +RUN cp -r -L bazel-bin/* /exaudf + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient +RUN ./test_udfclient.sh /exaudf/exaudfclient_static + +WORKDIR / +RUN mkdir /exasol_emulator +COPY emulator/ /exasol_emulator +COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile new file mode 100644 index 000000000..3a8ca3f61 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile @@ -0,0 +1,6 @@ +FROM {{build_deps}} + +RUN mkdir -p /build_info/packages +COPY base_test_deps/packages /build_info/packages/base_test_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages new file mode 100644 index 000000000..04a04253f --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages @@ -0,0 +1,5 @@ +gdb|15.0.50.20240403-0ubuntu1 +valgrind|1:3.22.0-0ubuntu3 +binutils|2.42-4ubuntu2.5 +patchelf|0.18.0-1.1build1 +strace|6.8-0ubuntu2 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile new file mode 100644 index 000000000..fecd4ef27 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive + +ENV ARCHIVE_UBUNTU_PREFIX="" +RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + +RUN mkdir -p /build_info/packages +COPY build_deps/packages /build_info/packages/build_deps + +ENV BAZEL_PACKAGE_VERSION="7.2.1" +ENV BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" +ENV BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/build_deps/apt_get_packages --with-versions + +RUN apt-get -y update && \ + curl -L --output "$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" && \ + apt-get install -y "./$BAZEL_PACKAGE_FILE" && \ + rm "$BAZEL_PACKAGE_FILE" && \ + apt-get -y clean && \ + apt-get -y autoremove + +RUN curl -L -o swig-4.3.1.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-4.3.1.tar.gz && \ + tar zxf swig-4.3.1.tar.gz && \ + (cd swig-4.3.1 && ./configure --prefix=/usr && make && make install) && \ + rm -rf swig-4.3.1 swig-4.3.1.tar.gz + +RUN locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + ldconfig + +RUN touch /env && \ + echo "export PROTOBUF_BIN=/usr/bin/protoc" >> /env && \ + echo "export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64" >> /env diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages new file mode 100644 index 000000000..75e3ba6e5 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages @@ -0,0 +1,9 @@ +coreutils|9.4-3ubuntu6 +tar|1.35+dfsg-3build1 +curl|8.5.0-2ubuntu10.6 +openjdk-11-jdk|11.0.27+6~us1-0ubuntu1~24.04 +build-essential|12.10ubuntu1 +libpcre2-dev|10.42-4ubuntu2.1 +protobuf-compiler|3.21.12-8.2ubuntu0.1 +chrpath|0.16-2build1 +locales|2.39-0ubuntu8.4 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile new file mode 100644 index 000000000..162d19a2a --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile @@ -0,0 +1,43 @@ +FROM {{language_deps}} + +COPY --from={{build_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /env /env +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary --config swig-string-as-buffer"] +RUN cp -r -L bazel-bin/* /exaudf +RUN rm -r /root/.cache/bazel #Delete bazel cache as it contains java rules with special characters which will cause a problem for Bucketfs + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient + +WORKDIR /exaudf + +RUN rm -r /exaudfclient + +COPY --from={{build_deps}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN mkdir /conf /buckets diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py b/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py new file mode 100644 index 000000000..d6a10c28f --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py @@ -0,0 +1,147 @@ +from typing import Dict + +from exasol.slc.internal.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask + + +class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "udfclient_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeLanguageDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "language_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"scripts": "ext/scripts"} + + def requires_tasks(self): + return {"udfclient_deps": AnalyzeUDFClientDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "build_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "build_run" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps, + "language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "exaudfclient"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "base_test_deps" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "base_test_build_run" + + def requires_tasks(self): + return {"base_test_deps": AnalyzeBaseTestDeps, + "language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "exaudfclient", "emulator": "emulator"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeFlavorBaseDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_base_deps" + + def requires_tasks(self): + return {"language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self): + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeFlavorCustomization(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_customization" + + def requires_tasks(self): + return {"flavor_base_deps": AnalyzeFlavorBaseDeps} + + +class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_test_build_run" + + def requires_tasks(self): + return {"flavor_customization": AnalyzeFlavorCustomization, + "base_test_build_run": AnalyzeBaseTestBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeRelease(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "release" + + def requires_tasks(self): + return {"flavor_customization": AnalyzeFlavorCustomization, + "build_run": AnalyzeBuildRun, + "language_deps": AnalyzeLanguageDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class SecurityScan(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "security_scan" + + def requires_tasks(self): + return {"release": AnalyzeRelease} + + def get_path_in_flavor(self): + return "flavor_base" diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from b/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from new file mode 100644 index 000000000..fb0760244 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from @@ -0,0 +1 @@ +flavors/template-Exasol-all-python-3.10 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile new file mode 100644 index 000000000..33c65c6ee --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile @@ -0,0 +1,8 @@ +FROM {{ language_deps }} + +RUN mkdir -p /build_info/packages +COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions + +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages new file mode 100644 index 000000000..3b66a1381 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages @@ -0,0 +1,4 @@ +unzip|6.0-28ubuntu4 +git|1:2.43.0-1ubuntu7 +libcurl4-openssl-dev|8.5.0-2ubuntu10.6 +build-essential|12.10ubuntu1 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/python3_pip_packages new file mode 100644 index 000000000..e69de29bb diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile new file mode 100644 index 000000000..32230a4d9 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile @@ -0,0 +1,25 @@ +FROM {{base_test_build_run}} + +RUN mkdir -p /conf /buckets + +COPY --from={{flavor_customization}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition new file mode 100644 index 000000000..0576fe703 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition @@ -0,0 +1 @@ +PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile new file mode 100644 index 000000000..f1ca7dfc2 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile @@ -0,0 +1,13 @@ +FROM {{udfclient_deps}} + +RUN mkdir -p /build_info/packages/language_deps +COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions + +COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages + +ENV PYTHON3_PREFIX /usr +ENV PYTHON3_VERSION python3.12 + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages new file mode 100644 index 000000000..1c5cf2d71 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages @@ -0,0 +1,4 @@ +ca-certificates|20240203 +python3.12-dev|3.12.3-1ubuntu0.7 +curl|8.5.0-2ubuntu10.6 +python3-pip|24.0+dfsg-1ubuntu1.2 \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages new file mode 100644 index 000000000..63697c0e3 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages @@ -0,0 +1,3 @@ +pandas|2.2.2 +numpy|1.26.4 +pyarrow|16.0.0 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile new file mode 100644 index 000000000..b5e1d4708 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:24.04 +RUN mkdir /conf /buckets + +COPY --from={{language_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /var /var +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /scripts /scripts +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +COPY --from={{flavor_customization}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /var /var +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +COPY --from={{build_run}} /exaudf /exaudf +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_run}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN mkdir -p /build_info/actual_installed_packages/release && \ + /scripts/list_installed_scripts/list_installed_apt.sh > /build_info/actual_installed_packages/release/apt_get_packages && \ + /scripts/list_installed_scripts/list_installed_pip.sh python3.12 > /build_info/actual_installed_packages/release/python3_pip_packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/.trivyignore b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/.trivyignore new file mode 100644 index 000000000..e69de29bb diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile new file mode 100644 index 000000000..5dabc1ff6 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile @@ -0,0 +1,15 @@ +FROM {{release}} +ENV DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /build_info/packages +COPY security_scan/packages /build_info/packages/security_scan +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages_trivy_deps +RUN curl -s https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add - && \ + /scripts/install_scripts/install_ppa.pl --ppa 'deb https://aquasecurity.github.io/trivy-repo/deb noble main' --out-file trivy.list +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages + +ENV SECURITY_SCANNERS="trivy" +COPY /security_scan/.trivyignore /.trivyignore +COPY /security_scan/trivy.rego /trivy.rego + +ENTRYPOINT ["/scripts/security_scan/run.sh"] \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages new file mode 100644 index 000000000..6bd628d4c --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages @@ -0,0 +1 @@ +trivy diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps new file mode 100644 index 000000000..8920661bf --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps @@ -0,0 +1,2 @@ +apt-transport-https +gnupg \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego new file mode 100644 index 000000000..ea531bfaf --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego @@ -0,0 +1,10 @@ +package trivy + +import data.lib.trivy + +default ignore = false + +ignore { + input.PkgName == "linux-libc-dev" + regex.match("^(kernel:|In the Linux kernel)", input.Title) +} \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig b/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig new file mode 100644 index 000000000..d1bdb8113 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig @@ -0,0 +1,2 @@ +generic_language_tests=python3 +test_folders=python3/all pandas/all pandas/pandas2 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile new file mode 100644 index 000000000..c67b999ec --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive + +ENV ARCHIVE_UBUNTU_PREFIX="" +RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + +RUN mkdir -p /build_info/packages +COPY udfclient_deps/packages /build_info/packages/udfclient_deps +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions + +RUN addgroup --gid 1001 exasolution +RUN adduser --disabled-login --uid 1001 --gid 1001 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 501 exausers +RUN adduser --disabled-login --uid 501 --gid 501 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +ENV PROTOBUF_LIBRARY_PREFIX=/usr/lib/ +ENV PROTOBUF_INCLUDE_PREFIX=/usr/include/ +ENV ZMQ_LIBRARY_PREFIX=/usr/lib +ENV ZMQ_INCLUDE_PREFIX=/usr/include diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages new file mode 100644 index 000000000..8d9e27f08 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages @@ -0,0 +1,8 @@ +coreutils|9.4-3ubuntu6 +adduser|3.137ubuntu1 +locales|2.39-0ubuntu8.4 +libnss-db|2.2.3pre1-9build3 +libzmq3-dev|4.3.5-1build2 +cppzmq-dev|4.10.0-1build1 +libprotobuf-dev|3.21.12-8.2ubuntu0.1 +libssl-dev|3.0.13-0ubuntu3.5 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile new file mode 100644 index 000000000..e83813dbe --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile @@ -0,0 +1,77 @@ +############################################################################################ +############################################################################################ +# This Dockerfile allows you to extend this flavor by installing packages or adding files. +# IF you didn't change the lines below, you can add packages and their version to the +# files in ./packages and they get automatically installed. +############################################################################################ +############################################################################################ + +####################################################################### +####################################################################### +# Do not change the following lines unless you know what you are doing +####################################################################### +####################################################################### + +FROM {{flavor_base_deps}} + +RUN mkdir -p /build_info/packages/flavor_customization + +COPY flavor_customization/packages/apt_get_packages /build_info/packages/flavor_customization +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_customization/apt_get_packages --with-versions + +COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3 --with-versions --allow-no-version + + +########################################################################## +########################################################################## +# Below this text you can add any Dockerfile commands except of FROM. +# However only commands which change the filesystem will be +# reflected to the final script-language container. +# We recommend to use only RUN, COPY and ADD. For example, WORKDIR, USER +# and ENV will be not carried over into the final container. For more +# information about Dockerfile commands, please check the reference +# https://docs.docker.com/engine/reference/builder/#dockerfile-reference +########################################################################## +########################################################################## + + +########### +########### +# Examples: +########### +########### + +################################################# +# Execute a command during the build of the image +################################################# + +# With RUN you can excute any command you usally would run in a bash shell. +# It usually is a good idea to cleanup downloaded files or packages +# caches in the same in the same RUN statement to keep the image size +# small. You can execute multiple shell commands in one RUN statement +# by combine them with +# - `&&` (only execute if the previous command was successfull), +# - `||` (only execute if the previous command was unsuccesfull) +# - `;` (execute regardless of the success of the previous command) +# To format multiple command per RUN statement you can add a line break with `\`. +# If you need to execute complex sequences of commands you should execute +# scripts which you can copy into the image. +# ------------------------------------------------------------------------------- + +# RUN git clone https://github.com/exasol/data-science-examples +# RUN curl -o data-science-examples.zip https://codeload.github.com/exasol/data-science-examples/zip/master && \ +# unzip data-science-examples.zip && \ +# rm data-science-examples.zip + +##################################### +# Copy a file from ./ into the image. +##################################### + +# In case of exaslct it is important to prefix source files or directories with `flavor_customization` +# if you want to copy files or directories located in the same directory as this Dockerfile +# ----------------------------------------------------------------------------------------------------- + +# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization +# ADD flavor_customization/code.tar.gz /code + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages new file mode 100644 index 000000000..d78cd6de0 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages @@ -0,0 +1,9 @@ +# This file specifies the package list which gets installed via apt. +# You must specify the the package and its version separated by a |. +# We recommend here the usage of package versions, to ensure that the container +# builds are reproducible. However, we allow also packages without version. +# As you can see, this file can contain comments which start with #. +# If a line starts with # the whole line is a comment, however you can +# also start a comment after the package definition. + +# p7zip-full|16.02+dfsg-6 # you can check which versions of a package are available with apt-cache policy diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages new file mode 100644 index 000000000..a9f22f993 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages @@ -0,0 +1,9 @@ +# This file specifies the package list which gets installed via pip for python3. +# You must specify the the package and its version separated by a |. +# We recommend here the usage of package versions, to ensure that the container +# builds are reproducible. However, we allow also packages without version. +# As you can see, this file can contain comments which start with #. +# If a line starts with # the whole line is a comment, however you can +# also start a comment after the package definition. + +#tensorflow-probability|0.9.0 diff --git a/test_container/tests/test/generic/get_connection.py b/test_container/tests/test/generic/get_connection.py index 7f5ad9d39..f4733f11c 100644 --- a/test_container/tests/test/generic/get_connection.py +++ b/test_container/tests/test/generic/get_connection.py @@ -172,8 +172,7 @@ def testAccessConnectionInAdapter(self): CREATE OR REPLACE PYTHON3 ADAPTER SCRIPT adapter.fast_adapter_conn AS import json import string - encodeUTF8 = lambda x: x - + def adapter_call(request): root = json.loads(request) if root["type"] == "createVirtualSchema": @@ -197,9 +196,9 @@ def adapter_call(request): }] } } - return encodeUTF8(json.dumps(res)) + return json.dumps(res) elif root["type"] == "dropVirtualSchema": - return encodeUTF8(json.dumps({"type": "dropVirtualSchema"})) + return json.dumps({"type": "dropVirtualSchema"}) else: raise ValueError('Unsupported callback') /