From 778c411fe4558871de6819461433976d926cc166 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 13 May 2024 17:05:51 +0200 Subject: [PATCH 01/13] #896: Add template-python3.12 flavor --- .../base/exaudflib/swig/swig_common.h | 1 + .../python/exascript_python_preset_core.py | 4 +- .../install_scripts/install_via_pip.pl | 2 +- .../flavor_base/udfclient_deps/Dockerfile | 8 +- .../FLAVOR_DESCRIPTION.md | 14 ++ .../template-Exasol-all-python-3.12/ci.json | 32 ++++ .../base_test_build_run/Dockerfile | 41 +++++ .../flavor_base/base_test_deps/Dockerfile | 6 + .../base_test_deps/packages/apt_get_packages | 5 + .../flavor_base/build_deps/Dockerfile | 37 +++++ .../build_deps/packages/apt_get_packages | 9 ++ .../flavor_base/build_run/Dockerfile | 43 +++++ .../flavor_base/build_steps.py | 147 ++++++++++++++++++ .../flavor_base/derived_from | 1 + .../flavor_base/flavor_base_deps/Dockerfile | 8 + .../packages/apt_get_packages | 4 + .../packages/python3_pip_packages | 0 .../flavor_test_build_run/Dockerfile | 25 +++ .../flavor_base/language_definition | 1 + .../flavor_base/language_deps/Dockerfile | 13 ++ .../language_deps/packages/apt_get_packages | 4 + .../packages/python3_pip_packages | 4 + .../flavor_base/release/Dockerfile | 62 ++++++++ .../flavor_base/security_scan/.trivyignore | 0 .../flavor_base/security_scan/Dockerfile | 14 ++ .../security_scan/packages/apt_get_packages | 1 + .../packages/apt_get_packages_trivy_deps | 2 + .../flavor_base/security_scan/trivy.rego | 10 ++ .../flavor_base/testconfig | 2 + .../flavor_base/udfclient_deps/Dockerfile | 26 ++++ .../udfclient_deps/packages/apt_get_packages | 8 + .../flavor_customization/Dockerfile | 77 +++++++++ .../packages/apt_get_packages | 9 ++ .../packages/python3_pip_packages | 9 ++ 34 files changed, 622 insertions(+), 7 deletions(-) create mode 100644 flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md create mode 100644 flavors/template-Exasol-all-python-3.12/ci.json create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/derived_from create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/python3_pip_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/language_definition create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/.trivyignore create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/testconfig create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages create mode 100644 flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages diff --git a/exaudfclient/base/exaudflib/swig/swig_common.h b/exaudfclient/base/exaudflib/swig/swig_common.h index b4e463fed..8080e5916 100644 --- a/exaudfclient/base/exaudflib/swig/swig_common.h +++ b/exaudfclient/base/exaudflib/swig/swig_common.h @@ -1,6 +1,7 @@ #ifndef SWIG_COMMON_H #define SWIG_COMMON_H +#include #include #include #include diff --git a/exaudfclient/base/python/exascript_python_preset_core.py b/exaudfclient/base/python/exascript_python_preset_core.py index 550b31949..2b7cf8636 100644 --- a/exaudfclient/base/python/exascript_python_preset_core.py +++ b/exaudfclient/base/python/exascript_python_preset_core.py @@ -6,7 +6,7 @@ from exascript_python import * import decimal import datetime -import imp +import types class ExaUDFError(Exception): pass @@ -126,7 +126,7 @@ def import_script(self, script): modobj = self.__modules[str(code)] else: print("%%% new code", modname, repr(code), code in self.__modules) - modobj = imp.new_module(modname) + modobj = types.ModuleType(modname) modobj.__file__ = "<%s>" % modname modobj.__dict__['exa'] = self self.__modules[str(code)] = modobj diff --git a/ext/scripts/install_scripts/install_via_pip.pl b/ext/scripts/install_scripts/install_via_pip.pl index da93abea6..9248ca720 100755 --- a/ext/scripts/install_scripts/install_via_pip.pl +++ b/ext/scripts/install_scripts/install_via_pip.pl @@ -81,7 +81,7 @@ =head1 SYNOPSIS my $pip_parameters_str = join( ' ', @pip_parameters); my $element_separator = '\\|'; -my $combining_template = "$python_binary -m pip install $pip_parameters_str --no-cache-dir <<<<0>>>>"; +my $combining_template = "$python_binary -m pip install $pip_parameters_str --break-system-packages --no-cache-dir <<<<0>>>>"; my @templates = ("'<<<<0>>>>'"); if($with_versions){ @templates=("'<<<<0>>>>==<<<<1>>>>'") diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile index a18d69b23..d359a98c2 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile @@ -11,10 +11,10 @@ RUN mkdir -p /build_info/packages COPY udfclient_deps/packages /build_info/packages/udfclient_deps RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions -RUN addgroup --gid 1000 exasolution -RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" -RUN addgroup --gid 500 exausers -RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 1001 exasolution +RUN adduser --disabled-login --uid 1001 --gid 1001 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 501 exausers +RUN adduser --disabled-login --uid 501 --gid 501 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en diff --git a/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md b/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md new file mode 100644 index 000000000..7398e1a51 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/FLAVOR_DESCRIPTION.md @@ -0,0 +1,14 @@ +# Details for template-Exasol-all-python-3.12 + +## Packages + +**Remarks:** The packages are grouped by there main usage and their type. + +- [UDFclient dependencies](flavor_base/udfclient_deps/packages/apt_get_packages) +- [Language dependencies](flavor_base/language_deps/packages/apt_get_packages) +- Flavor packages + - [Ubuntu packages](flavor_base/flavor_base_deps/packages/apt_get_packages) + - [Python3 pip packages](flavor_base/flavor_base_deps/packages/python3_pip_packages) +- Customization + - [Ubuntu packages](flavor_customization/packages/apt_get_packages) + - [Python3 pip packages](flavor_customization/packages/python3_pip_packages) diff --git a/flavors/template-Exasol-all-python-3.12/ci.json b/flavors/template-Exasol-all-python-3.12/ci.json new file mode 100644 index 000000000..1bda959cd --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/ci.json @@ -0,0 +1,32 @@ +{ + "build_runner": "ubuntu-24.04", + "test_config": { + "default_test_runner": "ubuntu-24.04", + "test_sets": [ + { + "name": "python", + "folders": ["python3/all"], + "goal": "release", + "generic_language_tests": [] + }, + { + "name": "pandas", + "folders": ["pandas/all", "pandas/pandas2"], + "goal": "release", + "generic_language_tests": [] + }, + { + "name": "linker_namespace", + "folders": ["linker_namespace_sanity"], + "goal": "base_test_build_run", + "generic_language_tests": [] + }, + { + "name": "generic", + "folders": [], + "goal": "release", + "generic_language_tests": ["python3"] + } + ] + } +} diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile new file mode 100644 index 000000000..b87c2bfc5 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile @@ -0,0 +1,41 @@ +FROM {{language_deps}} + +RUN mkdir /conf /buckets + +COPY --from={{base_test_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{base_test_deps}} /env /env +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY /exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"] +RUN cp -r -L bazel-bin/* /exaudf + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient +RUN ./test_udfclient.sh /exaudf/exaudfclient_static + +WORKDIR / +RUN mkdir /exasol_emulator +COPY emulator/ /exasol_emulator +COPY /exaudfclient/base/exaudflib/zmqcontainer.proto /exasol_emulator +RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=. diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile new file mode 100644 index 000000000..3a8ca3f61 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/Dockerfile @@ -0,0 +1,6 @@ +FROM {{build_deps}} + +RUN mkdir -p /build_info/packages +COPY base_test_deps/packages /build_info/packages/base_test_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/base_test_deps/apt_get_packages --with-versions diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages new file mode 100644 index 000000000..04a04253f --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_deps/packages/apt_get_packages @@ -0,0 +1,5 @@ +gdb|15.0.50.20240403-0ubuntu1 +valgrind|1:3.22.0-0ubuntu3 +binutils|2.42-4ubuntu2.5 +patchelf|0.18.0-1.1build1 +strace|6.8-0ubuntu2 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile new file mode 100644 index 000000000..93d58af5d --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile @@ -0,0 +1,37 @@ +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive + +ENV ARCHIVE_UBUNTU_PREFIX="" +RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + +RUN mkdir -p /build_info/packages +COPY build_deps/packages /build_info/packages/build_deps + +ENV BAZEL_PACKAGE_VERSION="7.2.1" +ENV BAZEL_PACKAGE_FILE="bazel_$BAZEL_PACKAGE_VERSION-linux-x86_64.deb" +ENV BAZEL_PACKAGE_URL="https://github.com/bazelbuild/bazel/releases/download/$BAZEL_PACKAGE_VERSION/$BAZEL_PACKAGE_FILE" + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/build_deps/apt_get_packages --with-versions + +RUN apt-get -y update && \ + curl -L --output "$BAZEL_PACKAGE_FILE" "$BAZEL_PACKAGE_URL" && \ + apt-get install -y "./$BAZEL_PACKAGE_FILE" && \ + rm "$BAZEL_PACKAGE_FILE" && \ + apt-get -y clean && \ + apt-get -y autoremove + +RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \ + tar zxf swig-2.0.4.tar.gz && \ + (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \ + rm -rf swig-2.0.4 swig-2.0.4.tar.gz + +RUN locale-gen en_US.UTF-8 && \ + update-locale LC_ALL=en_US.UTF-8 && \ + ldconfig + +RUN touch /env && \ + echo "export PROTOBUF_BIN=/usr/bin/protoc" >> /env && \ + echo "export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64" >> /env diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages new file mode 100644 index 000000000..d92f66436 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages @@ -0,0 +1,9 @@ +coreutils|9.4-3ubuntu6 +tar|1.35+dfsg-3build1 +curl|8.5.0-2ubuntu10.6 +openjdk-11-jdk|11.0.27+6~us1-0ubuntu1~24.04 +build-essential|12.10ubuntu1 +libpcre3-dev|2:8.39-15build1 +protobuf-compiler|3.21.12-8.2ubuntu0.1 +chrpath|0.16-2build1 +locales|2.39-0ubuntu8.4 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile new file mode 100644 index 000000000..1d1524f16 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile @@ -0,0 +1,43 @@ +FROM {{language_deps}} + +COPY --from={{build_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_deps}} /env /env +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +RUN mkdir /exaudfclient /exaudf +COPY exaudfclient/ /exaudfclient/ + +WORKDIR /exaudfclient/ +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"] +RUN cp -r -L bazel-bin/* /exaudf +RUN rm -r /root/.cache/bazel #Delete bazel cache as it contains java rules with special characters which will cause a problem for Bucketfs + +WORKDIR /exaudfclient/base +RUN ./test_udfclient.sh /exaudf/exaudfclient + +WORKDIR /exaudf + +RUN rm -r /exaudfclient + +COPY --from={{build_deps}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN mkdir /conf /buckets diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py b/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py new file mode 100644 index 000000000..d6a10c28f --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_steps.py @@ -0,0 +1,147 @@ +from typing import Dict + +from exasol.slc.internal.tasks.build.docker_flavor_image_task import DockerFlavorAnalyzeImageTask + + +class AnalyzeUDFClientDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "udfclient_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeLanguageDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "language_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"scripts": "ext/scripts"} + + def requires_tasks(self): + return {"udfclient_deps": AnalyzeUDFClientDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "build_deps" + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "build_run" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps, + "language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "exaudfclient"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "base_test_deps" + + def requires_tasks(self): + return {"build_deps": AnalyzeBuildDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeBaseTestBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "base_test_build_run" + + def requires_tasks(self): + return {"base_test_deps": AnalyzeBaseTestDeps, + "language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self) -> Dict[str, str]: + return {"exaudfclient": "exaudfclient", "emulator": "emulator"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeFlavorBaseDeps(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_base_deps" + + def requires_tasks(self): + return {"language_deps": AnalyzeLanguageDeps} + + def get_additional_build_directories_mapping(self): + return {"01_nodoc": "ext/01_nodoc", "scripts": "ext/scripts"} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeFlavorCustomization(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_customization" + + def requires_tasks(self): + return {"flavor_base_deps": AnalyzeFlavorBaseDeps} + + +class AnalyzeFlavorTestBuildRun(DockerFlavorAnalyzeImageTask): + + def get_build_step(self) -> str: + return "flavor_test_build_run" + + def requires_tasks(self): + return {"flavor_customization": AnalyzeFlavorCustomization, + "base_test_build_run": AnalyzeBaseTestBuildRun} + + def get_path_in_flavor(self): + return "flavor_base" + + +class AnalyzeRelease(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "release" + + def requires_tasks(self): + return {"flavor_customization": AnalyzeFlavorCustomization, + "build_run": AnalyzeBuildRun, + "language_deps": AnalyzeLanguageDeps} + + def get_path_in_flavor(self): + return "flavor_base" + + +class SecurityScan(DockerFlavorAnalyzeImageTask): + def get_build_step(self) -> str: + return "security_scan" + + def requires_tasks(self): + return {"release": AnalyzeRelease} + + def get_path_in_flavor(self): + return "flavor_base" diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from b/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from new file mode 100644 index 000000000..fb0760244 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/derived_from @@ -0,0 +1 @@ +flavors/template-Exasol-all-python-3.10 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile new file mode 100644 index 000000000..5c6e69974 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile @@ -0,0 +1,8 @@ +FROM {{ language_deps }} + +RUN mkdir -p /build_info/packages +COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions + +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages new file mode 100644 index 000000000..3b66a1381 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/apt_get_packages @@ -0,0 +1,4 @@ +unzip|6.0-28ubuntu4 +git|1:2.43.0-1ubuntu7 +libcurl4-openssl-dev|8.5.0-2ubuntu10.6 +build-essential|12.10ubuntu1 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/packages/python3_pip_packages new file mode 100644 index 000000000..e69de29bb diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile new file mode 100644 index 000000000..32230a4d9 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_test_build_run/Dockerfile @@ -0,0 +1,25 @@ +FROM {{base_test_build_run}} + +RUN mkdir -p /conf /buckets + +COPY --from={{flavor_customization}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition new file mode 100644 index 000000000..82e76acf3 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition @@ -0,0 +1 @@ +PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile new file mode 100644 index 000000000..2267ca877 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile @@ -0,0 +1,13 @@ +FROM {{udfclient_deps}} + +RUN mkdir -p /build_info/packages/language_deps +COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps + +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions + +COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages + +ENV PYTHON3_PREFIX /usr +ENV PYTHON3_VERSION python3.12 + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages new file mode 100644 index 000000000..a827416df --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages @@ -0,0 +1,4 @@ +ca-certificates|20240203 +python3.12-dev|3.12.3-1ubuntu0.5 +curl|8.5.0-2ubuntu10.6 +python3-pip|24.0+dfsg-1ubuntu1.1 \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages new file mode 100644 index 000000000..5b0edb320 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages @@ -0,0 +1,4 @@ +pandas|2.2.2 +numpy|1.26.4 +pyarrow|16.0.0 +setuptools|69.5.1 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile new file mode 100644 index 000000000..b5e1d4708 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/release/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:24.04 +RUN mkdir /conf /buckets + +COPY --from={{language_deps}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /var /var +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{language_deps}} /scripts /scripts +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +COPY --from={{flavor_customization}} /usr /usr +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /lib /lib +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /bin /bin +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /opt /opt +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /etc /etc +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{flavor_customization}} /var /var +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN ldconfig + +COPY --from={{build_run}} /exaudf /exaudf +RUN true # workaround for https://github.com/moby/moby/issues/37965 + +COPY --from={{build_run}} /build_info /build_info +RUN true # workaround for https://github.com/moby/moby/issues/37965 + + +RUN mkdir -p /build_info/actual_installed_packages/release && \ + /scripts/list_installed_scripts/list_installed_apt.sh > /build_info/actual_installed_packages/release/apt_get_packages && \ + /scripts/list_installed_scripts/list_installed_pip.sh python3.12 > /build_info/actual_installed_packages/release/python3_pip_packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/.trivyignore b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/.trivyignore new file mode 100644 index 000000000..e69de29bb diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile new file mode 100644 index 000000000..fd948691d --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile @@ -0,0 +1,14 @@ +FROM {{release}} +ENV DEBIAN_FRONTEND=noninteractive + +RUN mkdir -p /build_info/packages +COPY security_scan/packages /build_info/packages/security_scan +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages_trivy_deps +RUN curl -s https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add - && \ + /scripts/install_scripts/install_ppa.pl --ppa 'deb https://aquasecurity.github.io/trivy-repo/deb noble main' --out-file trivy.list +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/security_scan/apt_get_packages + +ENV SECURITY_SCANNERS="trivy" +COPY /security_scan/.trivyignore /.trivyignore + +ENTRYPOINT ["/scripts/security_scan/run.sh"] \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages new file mode 100644 index 000000000..6bd628d4c --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages @@ -0,0 +1 @@ +trivy diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps new file mode 100644 index 000000000..8920661bf --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/packages/apt_get_packages_trivy_deps @@ -0,0 +1,2 @@ +apt-transport-https +gnupg \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego new file mode 100644 index 000000000..ea531bfaf --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/trivy.rego @@ -0,0 +1,10 @@ +package trivy + +import data.lib.trivy + +default ignore = false + +ignore { + input.PkgName == "linux-libc-dev" + regex.match("^(kernel:|In the Linux kernel)", input.Title) +} \ No newline at end of file diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig b/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig new file mode 100644 index 000000000..d1bdb8113 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/testconfig @@ -0,0 +1,2 @@ +generic_language_tests=python3 +test_folders=python3/all pandas/all pandas/pandas2 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile new file mode 100644 index 000000000..c67b999ec --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive + +ENV ARCHIVE_UBUNTU_PREFIX="" +RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list + +COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc +COPY scripts /scripts + +RUN mkdir -p /build_info/packages +COPY udfclient_deps/packages /build_info/packages/udfclient_deps +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions + +RUN addgroup --gid 1001 exasolution +RUN adduser --disabled-login --uid 1001 --gid 1001 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 501 exausers +RUN adduser --disabled-login --uid 501 --gid 501 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" + +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +ENV PROTOBUF_LIBRARY_PREFIX=/usr/lib/ +ENV PROTOBUF_INCLUDE_PREFIX=/usr/include/ +ENV ZMQ_LIBRARY_PREFIX=/usr/lib +ENV ZMQ_INCLUDE_PREFIX=/usr/include diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages new file mode 100644 index 000000000..8d9e27f08 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/packages/apt_get_packages @@ -0,0 +1,8 @@ +coreutils|9.4-3ubuntu6 +adduser|3.137ubuntu1 +locales|2.39-0ubuntu8.4 +libnss-db|2.2.3pre1-9build3 +libzmq3-dev|4.3.5-1build2 +cppzmq-dev|4.10.0-1build1 +libprotobuf-dev|3.21.12-8.2ubuntu0.1 +libssl-dev|3.0.13-0ubuntu3.5 diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile new file mode 100644 index 000000000..e83813dbe --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/Dockerfile @@ -0,0 +1,77 @@ +############################################################################################ +############################################################################################ +# This Dockerfile allows you to extend this flavor by installing packages or adding files. +# IF you didn't change the lines below, you can add packages and their version to the +# files in ./packages and they get automatically installed. +############################################################################################ +############################################################################################ + +####################################################################### +####################################################################### +# Do not change the following lines unless you know what you are doing +####################################################################### +####################################################################### + +FROM {{flavor_base_deps}} + +RUN mkdir -p /build_info/packages/flavor_customization + +COPY flavor_customization/packages/apt_get_packages /build_info/packages/flavor_customization +RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_customization/apt_get_packages --with-versions + +COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_customization/python3_pip_packages --python-binary python3 --with-versions --allow-no-version + + +########################################################################## +########################################################################## +# Below this text you can add any Dockerfile commands except of FROM. +# However only commands which change the filesystem will be +# reflected to the final script-language container. +# We recommend to use only RUN, COPY and ADD. For example, WORKDIR, USER +# and ENV will be not carried over into the final container. For more +# information about Dockerfile commands, please check the reference +# https://docs.docker.com/engine/reference/builder/#dockerfile-reference +########################################################################## +########################################################################## + + +########### +########### +# Examples: +########### +########### + +################################################# +# Execute a command during the build of the image +################################################# + +# With RUN you can excute any command you usally would run in a bash shell. +# It usually is a good idea to cleanup downloaded files or packages +# caches in the same in the same RUN statement to keep the image size +# small. You can execute multiple shell commands in one RUN statement +# by combine them with +# - `&&` (only execute if the previous command was successfull), +# - `||` (only execute if the previous command was unsuccesfull) +# - `;` (execute regardless of the success of the previous command) +# To format multiple command per RUN statement you can add a line break with `\`. +# If you need to execute complex sequences of commands you should execute +# scripts which you can copy into the image. +# ------------------------------------------------------------------------------- + +# RUN git clone https://github.com/exasol/data-science-examples +# RUN curl -o data-science-examples.zip https://codeload.github.com/exasol/data-science-examples/zip/master && \ +# unzip data-science-examples.zip && \ +# rm data-science-examples.zip + +##################################### +# Copy a file from ./ into the image. +##################################### + +# In case of exaslct it is important to prefix source files or directories with `flavor_customization` +# if you want to copy files or directories located in the same directory as this Dockerfile +# ----------------------------------------------------------------------------------------------------- + +# COPY flavor_customization/packages/python3_pip_packages /build_info/packages/flavor_customization +# ADD flavor_customization/code.tar.gz /code + diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages new file mode 100644 index 000000000..d78cd6de0 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/apt_get_packages @@ -0,0 +1,9 @@ +# This file specifies the package list which gets installed via apt. +# You must specify the the package and its version separated by a |. +# We recommend here the usage of package versions, to ensure that the container +# builds are reproducible. However, we allow also packages without version. +# As you can see, this file can contain comments which start with #. +# If a line starts with # the whole line is a comment, however you can +# also start a comment after the package definition. + +# p7zip-full|16.02+dfsg-6 # you can check which versions of a package are available with apt-cache policy diff --git a/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages new file mode 100644 index 000000000..a9f22f993 --- /dev/null +++ b/flavors/template-Exasol-all-python-3.12/flavor_customization/packages/python3_pip_packages @@ -0,0 +1,9 @@ +# This file specifies the package list which gets installed via pip for python3. +# You must specify the the package and its version separated by a |. +# We recommend here the usage of package versions, to ensure that the container +# builds are reproducible. However, we allow also packages without version. +# As you can see, this file can contain comments which start with #. +# If a line starts with # the whole line is a comment, however you can +# also start a comment after the package definition. + +#tensorflow-probability|0.9.0 From 36ec6b2e9ba99c6ef314c761345b0d5be92d8e21 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 16 Jun 2025 16:07:26 +0200 Subject: [PATCH 02/13] Reverted changes in template-Exasol-all-python-3.10 --- .../flavor_base/udfclient_deps/Dockerfile | 8 ++++---- .../flavor_base/udfclient_deps/Dockerfile | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile index d359a98c2..992344671 100644 --- a/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.10/flavor_base/udfclient_deps/Dockerfile @@ -11,10 +11,10 @@ RUN mkdir -p /build_info/packages COPY udfclient_deps/packages /build_info/packages/udfclient_deps RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions -RUN addgroup --gid 1001 exasolution -RUN adduser --disabled-login --uid 1001 --gid 1001 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" -RUN addgroup --gid 501 exausers -RUN adduser --disabled-login --uid 501 --gid 501 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 1000 exasolution +RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 500 exausers +RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile index c67b999ec..e4443a61f 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile @@ -11,10 +11,10 @@ RUN mkdir -p /build_info/packages COPY udfclient_deps/packages /build_info/packages/udfclient_deps RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions -RUN addgroup --gid 1001 exasolution -RUN adduser --disabled-login --uid 1001 --gid 1001 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" -RUN addgroup --gid 501 exausers -RUN adduser --disabled-login --uid 501 --gid 501 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 1000 exasolution +RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 500 exausers +RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en From 64689869df775379e692215d90ac1bcba114ab2e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 17 Jun 2025 07:14:48 +0200 Subject: [PATCH 03/13] Updated gids for exasolution and exausers --- .../flavor_base/udfclient_deps/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile index e4443a61f..c67b999ec 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/udfclient_deps/Dockerfile @@ -11,10 +11,10 @@ RUN mkdir -p /build_info/packages COPY udfclient_deps/packages /build_info/packages/udfclient_deps RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/udfclient_deps/apt_get_packages --with-versions -RUN addgroup --gid 1000 exasolution -RUN adduser --disabled-login --uid 1000 --gid 1000 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" -RUN addgroup --gid 500 exausers -RUN adduser --disabled-login --uid 500 --gid 500 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 1001 exasolution +RUN adduser --disabled-login --uid 1001 --gid 1001 exasolution --gecos "First Last,RoomNumber,WorkPhone,HomePhone" +RUN addgroup --gid 501 exausers +RUN adduser --disabled-login --uid 501 --gid 501 exadefusr --gecos "First Last,RoomNumber,WorkPhone,HomePhone" ENV LANG en_US.UTF-8 ENV LANGUAGE en_US:en From a9f0d7d1183903b022e3065f094ca88f5edf7b6d Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 17 Jun 2025 07:43:13 +0200 Subject: [PATCH 04/13] 1. Updated python3.12-dev 2. Tried to fix install script tests --- .../tests/install_scripts/run_pip_tests.sh | 28 +++++++++---------- .../language_deps/packages/apt_get_packages | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/ext/scripts/tests/install_scripts/run_pip_tests.sh b/ext/scripts/tests/install_scripts/run_pip_tests.sh index b507202c5..3c69b6d50 100644 --- a/ext/scripts/tests/install_scripts/run_pip_tests.sh +++ b/ext/scripts/tests/install_scripts/run_pip_tests.sh @@ -17,43 +17,43 @@ echo echo ./install_via_pip.pl without versions TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl without versions and --ignore-installed TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --ignore-installed --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --ignore-installed --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --ignore-installed --break-system-packages --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl without versions and --use-deprecated-legacy-resolver TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --use-deprecated-legacy-resolver --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --use-deprecated=legacy-resolver --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --use-deprecated=legacy-resolver --break-system-packages --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl with versions, without allow-no-version TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" echo echo ./install_via_pip.pl with versions, with allow-no-version, all versions specified TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" echo echo ./install_via_pip.pl with versions, with allow-no-version, some versions missing TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/some_missing_versions --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.27.1' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests==2.27.1' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo @@ -64,13 +64,13 @@ echo echo ./install_via_pip.pl with pip version syntax TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/pip_version_syntax --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests>=2.21.0' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests>=2.21.0' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl installing a package twice with different versions must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/same_pkg/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/same_pkg/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.4' 'azure-common==1.1.28'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.4' 'azure-common==1.1.28'" echo @@ -84,31 +84,31 @@ echo echo ./install_via_pip.pl installing with ancestors and correct dependency TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/no_version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/no_version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.4' 'azure-batch==1.0.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.4' 'azure-batch==1.0.0'" echo echo ./install_via_pip.pl installing with ancestors and dependency with wrong version must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==1.0.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.28' 'azure-batch==1.0.0'" echo echo ./install_via_pip.pl installing with ancestors and package which has a dependency to an older package must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-storage-queue==1.1.0' 'azure-batch==1.0.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-storage-queue==1.1.0' 'azure-batch==1.0.0'" echo echo ./install_via_pip.pl installing with ancestors and package which has a dependency to a newer package must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-batch==1.0.0' 'azure-storage-queue==1.1.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-batch==1.0.0' 'azure-storage-queue==1.1.0'" echo echo ./install_via_pip.pl installing with multiple ancestors TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/no_version_conflict/multiple_ancestors/step3 --ancestor-pip-package-root-path test_files/pip/no_version_conflict/multiple_ancestors/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'" echo check_for_failed_tests diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages index a827416df..5057a9190 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages @@ -1,4 +1,4 @@ ca-certificates|20240203 -python3.12-dev|3.12.3-1ubuntu0.5 +python3.12-dev|3.12.3-1ubuntu0.6 curl|8.5.0-2ubuntu10.6 python3-pip|24.0+dfsg-1ubuntu1.1 \ No newline at end of file From 0df77c97a13ec046551839e297f8c7e4daa21738 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 23 Jun 2025 10:06:59 +0200 Subject: [PATCH 05/13] Updated language_definition --- .../flavor_base/language_definition | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition index 82e76acf3..0576fe703 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_definition @@ -1 +1 @@ -PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient_py3 +PYTHON3=localzmq+protobuf:///{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}?lang=python#buckets/{{ bucketfs_name }}/{{ bucket_name }}/{{ path_in_bucket }}{{ release_name }}/exaudf/exaudfclient From d86e4d4c6dead3ba0615a67f177bb1ad40f6131f Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 23 Jun 2025 15:49:12 +0200 Subject: [PATCH 06/13] Updated python package --- .../flavor_base/language_deps/packages/apt_get_packages | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages index 5057a9190..8bad8824a 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages @@ -1,4 +1,4 @@ ca-certificates|20240203 -python3.12-dev|3.12.3-1ubuntu0.6 +python3.12-dev|3.12.3-1ubuntu0.7 curl|8.5.0-2ubuntu10.6 python3-pip|24.0+dfsg-1ubuntu1.1 \ No newline at end of file From 8a54dbfe6a07140d2bf580c803888b8a45a1c91f Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Mon, 23 Jun 2025 16:07:04 +0200 Subject: [PATCH 07/13] Make option --pip-needs-break-system-packages optional --- .../install_scripts/install_via_pip.pl | 10 ++++-- .../tests/install_scripts/run_pip_tests.sh | 33 +++++++++++-------- .../flavor_base/flavor_base_deps/Dockerfile | 2 +- .../flavor_base/language_deps/Dockerfile | 2 +- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/ext/scripts/install_scripts/install_via_pip.pl b/ext/scripts/install_scripts/install_via_pip.pl index 9248ca720..95ecdbf6b 100755 --- a/ext/scripts/install_scripts/install_via_pip.pl +++ b/ext/scripts/install_scripts/install_via_pip.pl @@ -40,6 +40,7 @@ =head1 SYNOPSIS my $use_deprecated_legacy_resolver = 0; my $ancestor_pip_package_root_path = ''; my $install_build_tools_ephemerally = 0; +my $pip_needs_break_system_packages = 0; GetOptions ( "help" => \$help, "dry-run" => \$dry_run, @@ -51,7 +52,8 @@ =head1 SYNOPSIS "use-deprecated-legacy-resolver" => \$use_deprecated_legacy_resolver, "python-binary=s" => \$python_binary, "ancestor-pip-package-root-path=s" => \$ancestor_pip_package_root_path, - "install-build-tools-ephemerally" => \$install_build_tools_ephemerally + "install-build-tools-ephemerally" => \$install_build_tools_ephemerally, + "pip-needs-break-system-packages" => \$pip_needs_break_system_packages ) or package_mgmt_utils::print_usage_and_abort(__FILE__,"Error in command line arguments",2); package_mgmt_utils::print_usage_and_abort(__FILE__,"",0) if $help; @@ -79,9 +81,13 @@ =head1 SYNOPSIS push @pip_parameters, "--use-deprecated=legacy-resolver"; } +if($pip_needs_break_system_packages){ + push @pip_parameters, "--break-system-packages"; +} + my $pip_parameters_str = join( ' ', @pip_parameters); my $element_separator = '\\|'; -my $combining_template = "$python_binary -m pip install $pip_parameters_str --break-system-packages --no-cache-dir <<<<0>>>>"; +my $combining_template = "$python_binary -m pip install $pip_parameters_str --no-cache-dir <<<<0>>>>"; my @templates = ("'<<<<0>>>>'"); if($with_versions){ @templates=("'<<<<0>>>>==<<<<1>>>>'") diff --git a/ext/scripts/tests/install_scripts/run_pip_tests.sh b/ext/scripts/tests/install_scripts/run_pip_tests.sh index 3c69b6d50..c5c87954c 100644 --- a/ext/scripts/tests/install_scripts/run_pip_tests.sh +++ b/ext/scripts/tests/install_scripts/run_pip_tests.sh @@ -17,43 +17,43 @@ echo echo ./install_via_pip.pl without versions TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl without versions and --ignore-installed TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --ignore-installed --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --ignore-installed --break-system-packages --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --ignore-installed --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl without versions and --use-deprecated-legacy-resolver TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/without_versions --use-deprecated-legacy-resolver --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --use-deprecated=legacy-resolver --break-system-packages --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --use-deprecated=legacy-resolver --no-cache-dir 'humanfriendly' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl with versions, without allow-no-version TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" echo echo ./install_via_pip.pl with versions, with allow-no-version, all versions specified TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.21.0'" echo echo ./install_via_pip.pl with versions, with allow-no-version, some versions missing TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/some_missing_versions --with-versions --allow-no-version --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl with versions, with allow-no-version-for-urls, file with urls TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/with_urls --with-versions --allow-no-version-for-urls --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests==2.27.1' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests==2.27.1' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo @@ -64,13 +64,13 @@ echo echo ./install_via_pip.pl with pip version syntax TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/pip_version_syntax --python-binary python3 "$DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly==9.1' 'requests>=2.21.0' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'humanfriendly==9.1' 'requests>=2.21.0' 'git+http://github.com/exasol/bucketfs-utils-python.git@0.2.0#egg=exasol-bucketfs-utils-python'" echo echo ./install_via_pip.pl installing a package twice with different versions must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/same_pkg/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/same_pkg/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.4' 'azure-common==1.1.28'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.4' 'azure-common==1.1.28'" echo @@ -84,31 +84,36 @@ echo echo ./install_via_pip.pl installing with ancestors and correct dependency TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/no_version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/no_version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.4' 'azure-batch==1.0.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.4' 'azure-batch==1.0.0'" echo echo ./install_via_pip.pl installing with ancestors and dependency with wrong version must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.28' 'azure-batch==1.0.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==1.0.0'" echo echo ./install_via_pip.pl installing with ancestors and package which has a dependency to an older package must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_older_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-storage-queue==1.1.0' 'azure-batch==1.0.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-storage-queue==1.1.0' 'azure-batch==1.0.0'" echo echo ./install_via_pip.pl installing with ancestors and package which has a dependency to a newer package must fail TEST_OUTPUT=$(run_install_must_fail "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/step2 --ancestor-pip-package-root-path test_files/pip/version_conflict/other_package_with_newer_dependency_already_installed/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-batch==1.0.0' 'azure-storage-queue==1.1.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-batch==1.0.0' 'azure-storage-queue==1.1.0'" echo echo ./install_via_pip.pl installing with multiple ancestors TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl --file test_files/pip/no_version_conflict/multiple_ancestors/step3 --ancestor-pip-package-root-path test_files/pip/no_version_conflict/multiple_ancestors/build_info/packages --python-binary python3 --with-versions $DRY_RUN_OPTION") -assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'" +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --no-cache-dir 'azure-common==1.1.28' 'azure-batch==14.2.0' 'azure-storage-queue==1.1.0'" +echo + +echo ./install_via_pip.pl with pip-needs-break-system-packages +TEST_OUTPUT=$(run_install "$PATH_TO_INSTALL_SCRIPTS/install_via_pip.pl" --file test_files/pip/with_versions/all_versions_specified --pip-needs-break-system-packages --python-binary python3 "$DRY_RUN_OPTION") +assert "$TEST_OUTPUT" "Dry-Run: python3 -m pip install --break-system-packages --no-cache-dir 'humanfriendly' 'requests'" echo check_for_failed_tests diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile index 5c6e69974..33c65c6ee 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/flavor_base_deps/Dockerfile @@ -5,4 +5,4 @@ COPY flavor_base_deps/packages /build_info/packages/flavor_base_deps RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/flavor_base_deps/apt_get_packages --with-versions -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/flavor_base_deps/python3_pip_packages --python-binary python3 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile index 2267ca877..f1ca7dfc2 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/Dockerfile @@ -6,7 +6,7 @@ COPY language_deps/packages/apt_get_packages /build_info/packages/language_deps RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/language_deps/apt_get_packages --with-versions COPY language_deps/packages/python3_pip_packages /build_info/packages/language_deps -RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --ancestor-pip-package-root-path /build_info/packages +RUN /scripts/install_scripts/install_via_pip.pl --file /build_info/packages/language_deps/python3_pip_packages --python-binary python3 --with-versions --pip-needs-break-system-packages --ancestor-pip-package-root-path /build_info/packages ENV PYTHON3_PREFIX /usr ENV PYTHON3_VERSION python3.12 From 6eb00325e05fe7b7ee7a626d99f16f282b37069a Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 24 Jun 2025 07:26:49 +0200 Subject: [PATCH 08/13] Fixed security scan --- .../flavor_base/security_scan/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile index fd948691d..5dabc1ff6 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/security_scan/Dockerfile @@ -10,5 +10,6 @@ RUN /scripts/install_scripts/install_via_apt.pl --file /build_info/packages/secu ENV SECURITY_SCANNERS="trivy" COPY /security_scan/.trivyignore /.trivyignore +COPY /security_scan/trivy.rego /trivy.rego ENTRYPOINT ["/scripts/security_scan/run.sh"] \ No newline at end of file From bf66b03b6b04e299f1a748e6fb7434af45cc2205 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Tue, 24 Jun 2025 10:58:18 +0200 Subject: [PATCH 09/13] Replace legacy swig python code with import statement --- exaudfclient/base/BUILD | 3 +- exaudfclient/base/python/python3/BUILD | 5 ++- .../base/replace_swig_import_helper.py | 43 +++++++++++++++++++ .../packages/python3_pip_packages | 1 - 4 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 exaudfclient/base/replace_swig_import_helper.py diff --git a/exaudfclient/base/BUILD b/exaudfclient/base/BUILD index 357c34da9..9d6728354 100644 --- a/exaudfclient/base/BUILD +++ b/exaudfclient/base/BUILD @@ -2,7 +2,8 @@ package(default_visibility = ["//visibility:public"]) exports_files(["filter_swig_code.py", "build_integrated.py", "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh", - "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh"]) + "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh", + "replace_swig_import_helper.py"]) load("//:variables.bzl", "VM_ENABLED_DEFINES") diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index 9748ef7a1..6c74d1ff2 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -8,9 +8,12 @@ genrule( cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib cd build_exascript_python_tmp_cc swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i + cd .. + python3 "$(location //base:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" """, outs = ["exascript_python_tmp.cc", "exascript_python.py"], - srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"] + srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"], + tools = ["//base:replace_swig_import_helper.py"], ) genrule( diff --git a/exaudfclient/base/replace_swig_import_helper.py b/exaudfclient/base/replace_swig_import_helper.py new file mode 100644 index 000000000..51cc5f6b6 --- /dev/null +++ b/exaudfclient/base/replace_swig_import_helper.py @@ -0,0 +1,43 @@ +import sys +from pathlib import Path + +old_function = """ + def swig_import_helper(): + from os.path import dirname + import imp + fp = None + try: + fp, pathname, description = imp.find_module('_exascript_python', [dirname(__file__)]) + except ImportError: + import _exascript_python + return _exascript_python + if fp is not None: + try: + _mod = imp.load_module('_exascript_python', fp, pathname, description) + finally: + fp.close() + return _mod + _exascript_python = swig_import_helper() + del swig_import_helper +""" + +new_function = """ + import _exascript_python +""" + +def replace_swig_import_helper(target, source): + source_code = Path(source).read_text() + + assert old_function in source_code + + new_source_code = source_code.replace(old_function, new_function) + + with open(target, 'w') as f: + f.write(new_source_code) + +if __name__ == "__main__": + if len(sys.argv) != 3: + print('Usage: replace_swig_import_helper.py target source') + sys.exit(1) + print(f"replacing function swig_import_helper() using source={sys.argv[2]} and target={sys.argv[1]}") + replace_swig_import_helper(sys.argv[1], sys.argv[2]) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages index 5b0edb320..63697c0e3 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/python3_pip_packages @@ -1,4 +1,3 @@ pandas|2.2.2 numpy|1.26.4 pyarrow|16.0.0 -setuptools|69.5.1 From 6d08841e9e242b151a869fd57b7c11433cf27c41 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 25 Jun 2025 07:45:49 +0200 Subject: [PATCH 10/13] New swig --- exaudfclient/base/python/python3/BUILD | 2 +- .../flavor_base/build_deps/Dockerfile | 8 ++++---- .../flavor_base/build_deps/packages/apt_get_packages | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index 6c74d1ff2..bec79ed50 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -9,7 +9,7 @@ genrule( cd build_exascript_python_tmp_cc swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i cd .. - python3 "$(location //base:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" + #python3 "$(location //base:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" """, outs = ["exascript_python_tmp.cc", "exascript_python.py"], srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"], diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile index 93d58af5d..fecd4ef27 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/Dockerfile @@ -23,10 +23,10 @@ RUN apt-get -y update && \ apt-get -y clean && \ apt-get -y autoremove -RUN curl -L -o swig-2.0.4.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-2.0.4.tar.gz && \ - tar zxf swig-2.0.4.tar.gz && \ - (cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \ - rm -rf swig-2.0.4 swig-2.0.4.tar.gz +RUN curl -L -o swig-4.3.1.tar.gz https://exasol-script-languages-dependencies.s3.eu-central-1.amazonaws.com/swig-4.3.1.tar.gz && \ + tar zxf swig-4.3.1.tar.gz && \ + (cd swig-4.3.1 && ./configure --prefix=/usr && make && make install) && \ + rm -rf swig-4.3.1 swig-4.3.1.tar.gz RUN locale-gen en_US.UTF-8 && \ update-locale LC_ALL=en_US.UTF-8 && \ diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages index d92f66436..75e3ba6e5 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_deps/packages/apt_get_packages @@ -3,7 +3,7 @@ tar|1.35+dfsg-3build1 curl|8.5.0-2ubuntu10.6 openjdk-11-jdk|11.0.27+6~us1-0ubuntu1~24.04 build-essential|12.10ubuntu1 -libpcre3-dev|2:8.39-15build1 +libpcre2-dev|10.42-4ubuntu2.1 protobuf-compiler|3.21.12-8.2ubuntu0.1 chrpath|0.16-2build1 locales|2.39-0ubuntu8.4 From 9228bd574eb1d8c1fb81d3d6cf13e8ef2d7b6e4e Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Wed, 25 Jun 2025 10:42:40 +0200 Subject: [PATCH 11/13] Fixed syntax warning --- exaudfclient/base/python/exascript_python_preset_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exaudfclient/base/python/exascript_python_preset_core.py b/exaudfclient/base/python/exascript_python_preset_core.py index 2b7cf8636..b2dcc7766 100644 --- a/exaudfclient/base/python/exascript_python_preset_core.py +++ b/exaudfclient/base/python/exascript_python_preset_core.py @@ -13,7 +13,7 @@ class ExaUDFError(Exception): def clean_stacktrace_line(line): import re - match = re.match("""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line) + match = re.match(r"""^\s+File "(.+)", line ([0-9]+), in (.+)$""",line) if match is not None: filename=match.group(1) lineno=match.group(2) From 9a8cc663c878cfb0dfa554a6065de7529d3a7f2f Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 27 Jun 2025 07:42:37 +0200 Subject: [PATCH 12/13] 1. Fixed encoding/decoding for legacy swig versions by implementing a new auto-generated python module for decoding/encoding 2. Remove replace_swig_import_helper.py 3. Fixed test in get_connection.py by removing usage of "encodeUTF8", which actually was used from the global namespace, and not the lambda in the UDF 4. Fixed missing decodings/encodings in exascript_python_preset_core.py and exascript_python_wrap.py 5. Fixed some errors in python_ext_dataframe.cc --- exaudfclient/.bazelrc | 3 +- exaudfclient/base/BUILD | 2 +- exaudfclient/base/add_encoding_decoding.py | 29 +++++++++++ .../python/exascript_python_preset_core.py | 6 +-- .../base/python/exascript_python_wrap.py | 22 ++++---- exaudfclient/base/python/python3/BUILD | 16 ++++-- .../python/python3/python_ext_dataframe.cc | 50 ++++++++++++++++--- exaudfclient/base/python/pythoncontainer.cc | 17 +++++-- .../base/replace_swig_import_helper.py | 43 ---------------- .../flavor_base/build_run/Dockerfile | 2 +- .../language_deps/packages/apt_get_packages | 2 +- .../tests/test/generic/get_connection.py | 7 ++- 12 files changed, 114 insertions(+), 85 deletions(-) create mode 100644 exaudfclient/base/add_encoding_decoding.py delete mode 100644 exaudfclient/base/replace_swig_import_helper.py diff --git a/exaudfclient/.bazelrc b/exaudfclient/.bazelrc index fe0d63ac4..f6e589f30 100644 --- a/exaudfclient/.bazelrc +++ b/exaudfclient/.bazelrc @@ -25,4 +25,5 @@ build:valgrind --copt -O1 build:valgrind -c dbg build:valgrind --copt -g build:valgrind --strip=never -build:valgrind --copt -DVALGRIND_ACTIVE \ No newline at end of file +build:valgrind --copt -DVALGRIND_ACTIVE +build:swig-string-as-buffer --copt='-DSWIG_STRING_AS_BYTES_ENABLED=1' --action_env=SWIG_STRING_AS_BYTES_ENABLED=1 \ No newline at end of file diff --git a/exaudfclient/base/BUILD b/exaudfclient/base/BUILD index 9d6728354..3a2c87f15 100644 --- a/exaudfclient/base/BUILD +++ b/exaudfclient/base/BUILD @@ -3,7 +3,7 @@ package(default_visibility = ["//visibility:public"]) exports_files(["filter_swig_code.py", "build_integrated.py", "create_binary_wrapper.sh", "create_binary_wrapper_valgrind.sh", "create_binary_wrapper_valgrind_massif.sh", "exaudfclient.template.sh", - "replace_swig_import_helper.py"]) + "add_encoding_decoding.py"]) load("//:variables.bzl", "VM_ENABLED_DEFINES") diff --git a/exaudfclient/base/add_encoding_decoding.py b/exaudfclient/base/add_encoding_decoding.py new file mode 100644 index 000000000..063ab2baf --- /dev/null +++ b/exaudfclient/base/add_encoding_decoding.py @@ -0,0 +1,29 @@ +import os +import sys +from pathlib import Path + +CONVERT_TO_BYTES = """ +decodeUTF8 = lambda x: x.decode(encoding='utf-8') if isinstance(x, bytes) else x +encodeUTF8 = lambda x: x.encode(encoding='utf-8') if isinstance(x, str) else x +""" + +IDENTITY = """ +decodeUTF8 = lambda x: x +encodeUTF8 = lambda x: x +""" + +def add_encoding_decoding(target): + if "SWIG_STRING_AS_BYTES_ENABLED" in os.environ: + decoding_encdecoding = CONVERT_TO_BYTES + else: + decoding_encdecoding = IDENTITY + + with open(target, "wt", encoding="utf-8") as f: + f.write(decoding_encdecoding) + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print('Usage: add_encoding_decoding.py target') + sys.exit(1) + add_encoding_decoding(sys.argv[1]) diff --git a/exaudfclient/base/python/exascript_python_preset_core.py b/exaudfclient/base/python/exascript_python_preset_core.py index b2dcc7766..5062457cb 100644 --- a/exaudfclient/base/python/exascript_python_preset_core.py +++ b/exaudfclient/base/python/exascript_python_preset_core.py @@ -1,7 +1,5 @@ import sys unicode = str -decodeUTF8 = lambda x: x -encodeUTF8 = lambda x: x from exascript_python import * import decimal @@ -83,14 +81,14 @@ def ci(x, tbl): colprec = self.__meta.inputColumnPrecision(x) colscale = self.__meta.inputColumnScale(x) colsize = self.__meta.inputColumnSize(x) - coltn = self.__meta.inputColumnTypeName(x) + coltn = decodeUTF8(self.__meta.inputColumnTypeName(x)) elif tbl == 'output': colname = decodeUTF8(self.__meta.outputColumnName(x)) coltype = self.__meta.outputColumnType(x) colprec = self.__meta.outputColumnPrecision(x) colscale = self.__meta.outputColumnScale(x) colsize = self.__meta.outputColumnSize(x) - coltn = self.__meta.outputColumnTypeName(x) + coltn = decodeUTF8(self.__meta.outputColumnTypeName(x)) class exacolumn: def __init__(self, cn, ct, st, cp, cs, l): self.name = cn diff --git a/exaudfclient/base/python/exascript_python_wrap.py b/exaudfclient/base/python/exascript_python_wrap.py index 00ce02a8d..ebb26dd48 100644 --- a/exaudfclient/base/python/exascript_python_wrap.py +++ b/exaudfclient/base/python/exascript_python_wrap.py @@ -3,8 +3,6 @@ pyextdataframe_pkg = None unicode = str -decodeUTF8 = lambda x: x -encodeUTF8 = lambda x: x long = int if 'LIBPYEXADATAFRAME_DIR' in os.environ: @@ -45,10 +43,10 @@ def resget(): return val return resget def convert_date(x): - val = datetime.datetime.strptime(x, "%Y-%m-%d") + val = datetime.datetime.strptime(decodeUTF8(x), "%Y-%m-%d") return datetime.date(val.year, val.month, val.day) def convert_timestamp(x): - return datetime.datetime.strptime(x, "%Y-%m-%d %H:%M:%S.%f") + return datetime.datetime.strptime(decodeUTF8(x), "%Y-%m-%d %H:%M:%S.%f") self.__incoltypes = [] for col in range(self.__meta.inputColumnCount()): self.__incoltypes.append(self.__meta.inputColumnType(col)) @@ -67,8 +65,8 @@ def convert_timestamp(x): data[colname] = rd(inp.getInt64, inp.wasNull, col) elif self.__incoltypes[col] == NUMERIC: if self.__meta.inputColumnScale(col) == 0: - data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: int(str(x))) - else: data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: decimal.Decimal(str(x))) + data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: int(decodeUTF8(x))) + else: data[colname] = rd(inp.getNumeric, inp.wasNull, col, lambda x: decimal.Decimal(decodeUTF8(x))) elif self.__incoltypes[col] == DATE: data[colname] = rd(inp.getDate, inp.wasNull, col, convert_date) elif self.__incoltypes[col] == TIMESTAMP: @@ -138,7 +136,7 @@ def emit(self, *output): elif type(v) in (int, long): if self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) - elif self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, str(int(v))) + elif self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, encodeUTF8(str(int(v)))) elif self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) else: raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1091: emit column '%s' is of type %s but data given have type %s" \ @@ -147,7 +145,7 @@ def emit(self, *output): if self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) elif self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) - elif self.__outcoltypes[k] == NUMERIC: self.__out.setInt64(k, str(v)) + elif self.__outcoltypes[k] == NUMERIC: self.__out.setInt64(k, encodeUTF8(str(v))) else: raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1092: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) @@ -156,7 +154,7 @@ def emit(self, *output): raise RuntimeError(u"E-UDF-CL-SL-PYTHON-1093: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) self.__out.setBoolean(k, bool(v)) - elif type(v) in (str, unicode): + elif type(v) in (str, unicode, bytes): v = encodeUTF8(v) vl = len(v) if self.__outcoltypes[k] != STRING: @@ -164,7 +162,7 @@ def emit(self, *output): % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) self.__out.setString(k, v, vl) elif type(v) == decimal.Decimal: - if self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, str(v)) + if self.__outcoltypes[k] == NUMERIC: self.__out.setNumeric(k, encodeUTF8(str(v))) elif self.__outcoltypes[k] == INT32: self.__out.setInt32(k, int(v)) elif self.__outcoltypes[k] == INT64: self.__out.setInt64(k, int(v)) elif self.__outcoltypes[k] == DOUBLE: self.__out.setDouble(k, float(v)) @@ -175,12 +173,12 @@ def emit(self, *output): if self.__outcoltypes[k] != DATE: raise RuntimeError("E-UDF-CL-SL-PYTHON-1096: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setDate(k, v.isoformat()) + self.__out.setDate(k, encodeUTF8(v.isoformat())) elif type(v) == datetime.datetime: if self.__outcoltypes[k] != TIMESTAMP: raise RuntimeError("E-UDF-CL-SL-PYTHON-1097: emit column '%s' is of type %s but data given have type %s" \ % (decodeUTF8(self.__meta.outputColumnName(k)), type_names.get(self.__outcoltypes[k], 'UNKONWN'), str(type(v)))) - self.__out.setTimestamp(k, v.isoformat(' ')) + self.__out.setTimestamp(k, encodeUTF8(v.isoformat(' '))) else: raise RuntimeError("E-UDF-CL-SL-PYTHON-1098: data type %s is not supported" % str(type(v))) msg = self.__out.checkException() if msg: raise RuntimeError("F-UDF-CL-SL-PYTHON-1099: "+msg) diff --git a/exaudfclient/base/python/python3/BUILD b/exaudfclient/base/python/python3/BUILD index bec79ed50..a82c2a127 100644 --- a/exaudfclient/base/python/python3/BUILD +++ b/exaudfclient/base/python/python3/BUILD @@ -8,12 +8,9 @@ genrule( cp "$(location //base/exaudflib:swig/script_data_transfer_objects_wrapper.h)" "$(location //base/exaudflib:exascript.i)" build_exascript_python_tmp_cc/exaudflib cd build_exascript_python_tmp_cc swig -v $$INCLUDES -O -DEXTERNAL_PROCESS -Wall -c++ -python -py3 -addextern -module exascript_python -o "../$(location exascript_python_tmp.cc)" exaudflib/exascript.i - cd .. - #python3 "$(location //base:replace_swig_import_helper.py)" "$(location exascript_python.py)" "$(location exascript_python.py)" """, outs = ["exascript_python_tmp.cc", "exascript_python.py"], srcs = ["//base/exaudflib:exascript.i","//base/exaudflib:swig/script_data_transfer_objects_wrapper.h"], - tools = ["//base:replace_swig_import_helper.py"], ) genrule( @@ -38,14 +35,23 @@ genrule( tools = ["//base/python:extend_exascript_python_preset_py.sh"] ) +genrule( + name = "gen_encoding_decoding", + cmd = 'python3 "$(location //base:add_encoding_decoding.py)" "$(location exascript_encoding_decoding.py)"', + outs = ["exascript_encoding_decoding.py"], + srcs = [], + tools = ["//base:add_encoding_decoding.py"] +) + genrule( name = "exascript_python_int", cmd = """ cp $(SRCS) . - python3 $(location //base:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" + python3 $(location //base:build_integrated.py) "$(location exascript_python_int.h)" "exascript_python.py" "exascript_python_wrap.py" "exascript_python_preset.py" "exascript_encoding_decoding.py" """, outs = ["exascript_python_int.h"], - srcs = [":exascript_python_tmp_cc", "//base/python:exascript_python_wrap.py", ":extend_exascript_python_preset_py"], + srcs = [":exascript_python_tmp_cc", "//base/python:exascript_python_wrap.py", + ":extend_exascript_python_preset_py", ":gen_encoding_decoding"], tools = ["//base:build_integrated.py"] ) diff --git a/exaudfclient/base/python/python3/python_ext_dataframe.cc b/exaudfclient/base/python/python3/python_ext_dataframe.cc index 73051398c..da7d31301 100644 --- a/exaudfclient/base/python/python3/python_ext_dataframe.cc +++ b/exaudfclient/base/python/python3/python_ext_dataframe.cc @@ -175,6 +175,23 @@ inline void checkPyObjIsNotNull(const PyObject *obj) { throw std::runtime_error("F-UDF-CL-SL-PYTHON-1142"); } +inline PyObject* convertPyStringToPyBytes(PyObject* pyString) { +#ifdef SWIG_STRING_AS_BYTES_ENABLED + return PyObject_CallMethod(pyString, "encode", NULL); +#else + Py_INCREF(pyString); + return pyString; +#endif +} + +inline PyObject* convertPyBytesToPyString(PyObject* pyString) { +#ifdef SWIG_STRING_AS_BYTES_ENABLED + return PyObject_CallMethod(pyString, "decode", NULL); +#else + Py_INCREF(pyString); + return pyString; +#endif +} struct ColumnInfo @@ -923,8 +940,11 @@ inline void handleEmitPyInt( break; } case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyInt)); + { + PyPtr pyStrInt(PyObject_Str(pyInt)); + pyValue.reset(convertPyStringToPyBytes(pyStrInt.get())); break; + } case SWIGVMContainers::DOUBLE: { double value = PyFloat_AsDouble(pyInt); @@ -977,8 +997,11 @@ inline void handleEmitPyFloat( break; } case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyFloat)); + { + PyPtr pyStrFloat(PyObject_Str(pyFloat)); + pyValue.reset(convertPyStringToPyBytes(pyStrFloat.get())); break; + } case SWIGVMContainers::DOUBLE: { //pyFloat points to a 'borrowed' reference. We need to explicitly increase the ref counter here, as pyValue will decrease it again later. @@ -1020,15 +1043,21 @@ inline void handleEmitPyDecimal( case SWIGVMContainers::INT64: case SWIGVMContainers::INT32: { + std::cout << "handleEmitPyDecimal: INT" << std::endl; PyPtr pyInt(PyObject_CallMethodObjArgs(pyDecimal, pyIntMethodName.get(), NULL)); pyValue.reset(pyInt.release()); break; } case SWIGVMContainers::NUMERIC: - pyValue.reset(PyObject_Str(pyDecimal)); + { + std::cout << "handleEmitPyDecimal: NUMERIC" << std::endl; + PyPtr pyStrDecimal(PyObject_Str(pyDecimal)); + pyValue.reset(convertPyStringToPyBytes(pyStrDecimal.get())); break; + } case SWIGVMContainers::DOUBLE: { + std::cout << "handleEmitPyDecimal: DOUBLE" << std::endl; PyPtr pyFloat(PyObject_CallMethodObjArgs(pyDecimal, pyFloatMethodName.get(), NULL)); pyValue.reset(pyFloat.release()); break; @@ -1067,11 +1096,12 @@ inline void handleEmitPyStr( case SWIGVMContainers::STRING: { Py_ssize_t size = -1; + PyPtr bytesString(convertPyStringToPyBytes(pyString)); const char *str = PyUnicode_AsUTF8AndSize(pyString, &size); if (!str && size < 0) throw std::runtime_error("F-UDF-CL-SL-PYTHON-1137: invalid size of string"); PyPtr pySize(PyLong_FromSsize_t(size)); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyString, pySize.get(), NULL)); + pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesString.get(), pySize.get(), NULL)); break; } default: @@ -1105,7 +1135,8 @@ inline void handleEmitPyDate( case SWIGVMContainers::DATE: { PyPtr pyIsoDate(PyObject_CallMethodObjArgs(pyDate, pyIsoformatMethodName.get(), NULL)); - pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDate.get(), NULL)); + PyPtr bytesIsoDate(convertPyStringToPyBytes(pyIsoDate.get())); + pyResult.reset(PyObject_CallMethodObjArgs(resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesIsoDate.get(), NULL)); break; } default: @@ -1141,8 +1172,9 @@ inline void handleEmitPyTimestamp( // it to the generated string. PyPtr pyTzLocalize(PyObject_CallMethod(pyTimestamp, "tz_localize", "z", NULL)); PyPtr pyIsoDatetime(PyObject_CallMethod(pyTzLocalize.get(), "isoformat", "s", " ")); + PyPtr bytesDateTime(convertPyStringToPyBytes(pyIsoDatetime.get())); pyResult.reset(PyObject_CallMethodObjArgs( - resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDatetime.get(), NULL)); + resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesDateTime.get(), NULL)); break; } default: @@ -1177,8 +1209,9 @@ inline void handleEmitNpyDateTime( case SWIGVMContainers::TIMESTAMP: { PyPtr pyIsoDatetime(PyObject_CallMethod(pyTimestamp, "isoformat", "s", " ")); + PyPtr bytesDateTime(convertPyStringToPyBytes(pyIsoDatetime.get())); pyResult.reset(PyObject_CallMethodObjArgs( - resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), pyIsoDatetime.get(), NULL)); + resultHandler, pyColSetMethods[c].second.get(), pyColSetMethods[c].first.get(), bytesDateTime.get(), NULL)); break; } default: @@ -1377,7 +1410,8 @@ void emit(PyObject *resultHandler, std::vector& colInfo, PyObject *d PyPtr pyCheckException(PyObject_CallMethodObjArgs(resultHandler, pyCheckExceptionMethodName.get(), NULL)); if (pyCheckException.get() != Py_None) { - const char *exMsg = PyUnicode_AsUTF8(pyCheckException.get()); + PyPtr exceptionAsPyBytes(convertPyBytesToPyString(pyCheckException.get())); + const char *exMsg = PyUnicode_AsUTF8(exceptionAsPyBytes.get()); if (exMsg) { std::stringstream ss; ss << "F-UDF-CL-SL-PYTHON-1075: emit(): " << exMsg; diff --git a/exaudfclient/base/python/pythoncontainer.cc b/exaudfclient/base/python/pythoncontainer.cc index a52b29f52..171cf2930 100644 --- a/exaudfclient/base/python/pythoncontainer.cc +++ b/exaudfclient/base/python/pythoncontainer.cc @@ -194,16 +194,22 @@ PythonVMImpl::PythonVMImpl(bool checkOnly): m_checkOnly(checkOnly) check("F-UDF-CL-SL-PYTHON-1010"); if (exatable == NULL) throw PythonVM::exception("F-UDF-CL-SL-PYTHON-1011: Failed to import code module"); + code = Py_CompileString(integrated_exascript_encoding_decoding_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1143"); + if (code == NULL) {check("F-UDF-CL-SL-PYTHON-1144");} + + PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1145"); + Py_DECREF(code); + code = Py_CompileString(integrated_exascript_python_preset_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1012"); if (code == NULL) {check("F-UDF-CL-SL-PYTHON-1013");} - PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1014"); + PyEval_EvalCode(code, globals, globals); check("F-UDF-CL-SL-PYTHON-1014"); Py_DECREF(code); - PyObject *runobj = PyDict_GetItemString(globals, "__pythonvm_wrapped_parse"); check("F-UDF-CL-SL-PYTHON-1016"); - //PyObject *retvalue = PyObject_CallFunction(runobj, NULL); check(); - PyObject *retvalue = PyObject_CallFunctionObjArgs(runobj, globals, NULL); check("F-UDF-CL-SL-PYTHON-1017"); - Py_XDECREF(retvalue); retvalue = NULL; + PyObject *runobj = PyDict_GetItemString(globals, "__pythonvm_wrapped_parse"); check("F-UDF-CL-SL-PYTHON-1016"); + //PyObject *retvalue = PyObject_CallFunction(runobj, NULL); check(); + PyObject *retvalue = PyObject_CallFunctionObjArgs(runobj, globals, NULL); check("F-UDF-CL-SL-PYTHON-1017"); + Py_XDECREF(retvalue); retvalue = NULL; code = Py_CompileString(integrated_exascript_python_wrap_py, "", Py_file_input); check("F-UDF-CL-SL-PYTHON-1018"); if (code == NULL) throw PythonVM::exception("Failed to compile wrapping script"); @@ -489,6 +495,7 @@ const char* PythonVMImpl::singleCall(single_call_function_id_e fn, const Executi PyObject* p3str = PyUnicode_AsEncodedString(repr, "utf-8", "ignore"); const char *bytes = PyBytes_AS_STRING(p3str); singleCallResult = string(bytes); + std::cout << "pythoncontainer: singleCallResult='" << singleCallResult << "'" << std::endl; Py_XDECREF(retvalue); retvalue = NULL; return singleCallResult.c_str(); } diff --git a/exaudfclient/base/replace_swig_import_helper.py b/exaudfclient/base/replace_swig_import_helper.py deleted file mode 100644 index 51cc5f6b6..000000000 --- a/exaudfclient/base/replace_swig_import_helper.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys -from pathlib import Path - -old_function = """ - def swig_import_helper(): - from os.path import dirname - import imp - fp = None - try: - fp, pathname, description = imp.find_module('_exascript_python', [dirname(__file__)]) - except ImportError: - import _exascript_python - return _exascript_python - if fp is not None: - try: - _mod = imp.load_module('_exascript_python', fp, pathname, description) - finally: - fp.close() - return _mod - _exascript_python = swig_import_helper() - del swig_import_helper -""" - -new_function = """ - import _exascript_python -""" - -def replace_swig_import_helper(target, source): - source_code = Path(source).read_text() - - assert old_function in source_code - - new_source_code = source_code.replace(old_function, new_function) - - with open(target, 'w') as f: - f.write(new_source_code) - -if __name__ == "__main__": - if len(sys.argv) != 3: - print('Usage: replace_swig_import_helper.py target source') - sys.exit(1) - print(f"replacing function swig_import_helper() using source={sys.argv[2]} and target={sys.argv[1]}") - replace_swig_import_helper(sys.argv[1], sys.argv[2]) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile index 1d1524f16..162d19a2a 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/build_run/Dockerfile @@ -25,7 +25,7 @@ RUN mkdir /exaudfclient /exaudf COPY exaudfclient/ /exaudfclient/ WORKDIR /exaudfclient/ -RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary"] +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty --config optimize --config python --config fast-binary --config swig-string-as-buffer"] RUN cp -r -L bazel-bin/* /exaudf RUN rm -r /root/.cache/bazel #Delete bazel cache as it contains java rules with special characters which will cause a problem for Bucketfs diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages index 8bad8824a..1c5cf2d71 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/language_deps/packages/apt_get_packages @@ -1,4 +1,4 @@ ca-certificates|20240203 python3.12-dev|3.12.3-1ubuntu0.7 curl|8.5.0-2ubuntu10.6 -python3-pip|24.0+dfsg-1ubuntu1.1 \ No newline at end of file +python3-pip|24.0+dfsg-1ubuntu1.2 \ No newline at end of file diff --git a/test_container/tests/test/generic/get_connection.py b/test_container/tests/test/generic/get_connection.py index 7f5ad9d39..f4733f11c 100644 --- a/test_container/tests/test/generic/get_connection.py +++ b/test_container/tests/test/generic/get_connection.py @@ -172,8 +172,7 @@ def testAccessConnectionInAdapter(self): CREATE OR REPLACE PYTHON3 ADAPTER SCRIPT adapter.fast_adapter_conn AS import json import string - encodeUTF8 = lambda x: x - + def adapter_call(request): root = json.loads(request) if root["type"] == "createVirtualSchema": @@ -197,9 +196,9 @@ def adapter_call(request): }] } } - return encodeUTF8(json.dumps(res)) + return json.dumps(res) elif root["type"] == "dropVirtualSchema": - return encodeUTF8(json.dumps({"type": "dropVirtualSchema"})) + return json.dumps({"type": "dropVirtualSchema"}) else: raise ValueError('Unsupported callback') / From f2b1b77bd1034569459b23f319d6a91dbd730a35 Mon Sep 17 00:00:00 2001 From: Thomas Ubensee <34603111+tomuben@users.noreply.github.com> Date: Fri, 27 Jun 2025 08:09:01 +0200 Subject: [PATCH 13/13] Fixed build step base_test_build_run --- .../flavor_base/base_test_build_run/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile index b87c2bfc5..f289dc767 100644 --- a/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile +++ b/flavors/template-Exasol-all-python-3.12/flavor_base/base_test_build_run/Dockerfile @@ -27,7 +27,7 @@ RUN mkdir /exaudfclient /exaudf COPY /exaudfclient/ /exaudfclient/ WORKDIR /exaudfclient/ -RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries"] +RUN ["/bin/bash", "-c", "source /env && bash build.sh --config no-tty -c dbg --config python --config test-binaries --config swig-string-as-buffer"] RUN cp -r -L bazel-bin/* /exaudf WORKDIR /exaudfclient/base