diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..480d76e2b --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + target-branch: "develop" diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 008cd2d18..cb7f0539e 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -12,9 +12,14 @@ on: - '*' branches: - release/* - - feature/* - main - develop + - feature/* + +# prevent duplicate CI runs from separate triggers +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true jobs: build_wheels: @@ -36,12 +41,12 @@ jobs: - [windows-latest, AMD64, win_amd64, 0.0] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 # need git tags available for setuptools_scm to grab tags with: fetch-depth: 0 - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 with: script: | core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); @@ -107,7 +112,7 @@ jobs: - name: build windows wheels if: runner.OS == 'Windows' - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.2.1 with: package-dir: ${{github.workspace}}/install/python_installer config-file: ${{github.workspace}}/install/python_installer/pyproject.toml @@ -115,17 +120,19 @@ jobs: CIBW_ARCHS: ${{matrix.build-platform[1]}} CIBW_PLATFORM: windows CIBW_BUILD: cp3*-${{matrix.build-platform[2]}} - CIBW_SKIP: cp36* cp37* cp313* + # CIBW_SKIP: cp36* cp37* cp313* + CIBW_SKIP: cp36* cp37* CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path ${{github.workspace}}/install/lib" CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel" - name: Build manylinux2014 wheels # this is a slow-running build. Only execute on tag + # && + # github.event_name == 'push' && + # startsWith(github.ref, 'refs/tags/') if: > - startswith(matrix.build-platform[2], 'manylinux2014') && - github.event_name == 'push' && - startsWith(github.ref, 'refs/tags/') - uses: pypa/cibuildwheel@v2.22.0 + startswith(matrix.build-platform[2], 'manylinux2014') + uses: pypa/cibuildwheel@v3.2.1 with: package-dir: ./source/wrappers/python config-file: ./source/wrappers/python/pyproject.toml @@ -156,11 +163,11 @@ jobs: wget https://github.com/capnproto/capnproto/archive/refs/tags/v0.10.4.tar.gz && tar xzf v0.10.4.tar.gz && cd capnproto-0.10.4 && - cmake -Bbuild -H. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_SHARED_LIBS=ON && + cmake -Bbuild -H. -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 -DCMAKE_INSTALL_PREFIX=/usr/local -DBUILD_SHARED_LIBS=ON && cmake --build build && cmake --install build && cd /tmp && - wget https://boostorg.jfrog.io/artifactory/main/release/1.80.0/source/boost_1_80_0.tar.gz && + wget https://sourceforge.net/projects/boost/files/boost/1.80.0/boost_1_80_0.tar.gz && tar xzf boost_1_80_0.tar.gz && cd boost_1_80_0 && ./bootstrap.sh --prefix=/usr/local && @@ -176,7 +183,7 @@ jobs: startswith(matrix.build-platform[2], 'manylinux_2_28') && ! ( startswith(matrix.build-platform[1], 'aarch64') && !startsWith(github.ref, 'refs/tags/') ) - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.2.1 with: package-dir: ./source/wrappers/python config-file: ./source/wrappers/python/pyproject.toml @@ -230,21 +237,42 @@ jobs: - name: Build macos wheels if: runner.os == 'macOS' - uses: pypa/cibuildwheel@v2.22.0 + uses: pypa/cibuildwheel@v3.2.1 with: package-dir: ./source/wrappers/python config-file: ./source/wrappers/python/pyproject.toml env: CIBW_ARCHS: ${{matrix.build-platform[1]}} CIBW_PLATFORM: macos + # CIBW_SKIP: cp313* CIBW_BUILD: cp*-${{matrix.build-platform[2]}} MACOSX_DEPLOYMENT_TARGET: ${{matrix.build-platform[3]}} - - uses: actions/upload-artifact@v4 + - name: upload pyuda cibw build artifacts + uses: actions/upload-artifact@v5 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl + # only build dummy uda package on a single runner (pure python) + - name: build uda (dummy package) wheels + if: runner.os == 'macOS' && startswith(matrix.build-platform[1], 'arm64') + run: | + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip + pip install --upgrade build setuptools + cd ${{github.workspace}}/source/wrappers/python/uda && python -m build --wheel --sdist + + - name: Upload uda build artifacts + # only build and upload pure python wheels on a single runner + if: runner.os == 'macOS' && startswith(matrix.build-platform[1], 'arm64') + uses: actions/upload-artifact@v5 + with: + name: uda-wheels + path: ${{github.workspace}}/source/wrappers/python/uda/dist/* + + upload_pypi: # only upload to pypi for tagged releases if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') @@ -252,7 +280,7 @@ jobs: runs-on: ubuntu-latest environment: name: pypi - url: https://pypi.org/p/uda + url: https://pypi.org/p/pyuda permissions: id-token: write steps: @@ -277,7 +305,7 @@ jobs: permissions: id-token: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v6 with: # unpacks all CIBW artifacts into dist/ pattern: cibw-* @@ -287,3 +315,47 @@ jobs: - uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ + verbose: true + + upload_dummy_repo_to_pypi: + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') + # only upload dummy repo if normal one exists + needs: upload_pypi + runs-on: ubuntu-latest + environment: + name: pypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v6 + with: + # unpacks all build artifacts into dist/ + name: uda-wheels + path: dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://upload.pypi.org/legacy/ + verbose: true + + upload_dummy_repo_to_testpypi: + if: > + github.event_name == 'push' && !startsWith(github.ref, 'refs/heads/feature/') + # only upload dummy repo if normal one exists + needs: upload_test_pypi + runs-on: ubuntu-latest + environment: + name: testpypi + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v6 + with: + # unpacks all build artifacts into dist/ + name: uda-wheels + path: dist + + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ + verbose: true diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index dbf2c4f88..6dee13b35 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -7,18 +7,22 @@ on: - develop - 'feature/**' - 'release/**' + - 'bugfix/**' pull_request: branches: - main + - develop -#env: -# BUILD_TYPE: Release +# prevent duplicate CI runs from separate triggers +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true jobs: build: strategy: matrix: - os: [ubuntu-22.04, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] release: [Release] ssl: [ON, OFF] client-only: [ON, OFF] @@ -26,7 +30,7 @@ jobs: exclude: - os: windows-latest client-only: OFF - - os: ubuntu-22.04 + - os: ubuntu-latest client-only: ON - os: macos-latest client-only: ON @@ -38,16 +42,16 @@ jobs: shell: bash steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - - uses: actions/github-script@v7 + - uses: actions/github-script@v8 with: script: | core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Install linux dependencies - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-latest' run: > sudo apt update && sudo apt install -y git @@ -56,6 +60,7 @@ jobs: libssl-dev cmake build-essential + libtirpc-dev pkg-config libxml2-dev libspdlog-dev @@ -66,8 +71,15 @@ jobs: python3-pip python3-venv + # Capnproto package on Ubuntu 24.04 has broken CMake file + - name: Fix Capnproto on Ubuntu + if: matrix.os == 'ubuntu-latest' + run: > + sudo sed -i 's/if (yes) # WITH_LIBATOMIC/if (no) # WITH_LIBATOMIC/' \ + /usr/lib/x86_64-linux-gnu/cmake/CapnProto/CapnProtoConfig.cmake + - name: Install Intel compiler - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-latest' run: > sudo apt install -y wget && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && @@ -113,9 +125,10 @@ jobs: openssl dlfcn-win32 spdlog + liblzma - name: Configure CMake (linux) - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-latest' run: > cmake -G Ninja -B build -DBUILD_SHARED_LIBS=ON @@ -125,7 +138,7 @@ jobs: -DENABLE_CAPNP=${{ matrix.capnp }} - name: Configure CMake (linux Intel) - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-latest' run: > source /opt/intel/oneapi/setvars.sh && CXX=icpx CC=icx cmake -G Ninja -B build-intel @@ -176,7 +189,7 @@ jobs: run: cmake --build build --config ${{ matrix.release }} - name: Build Intel - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-latest' run: cmake --build build-intel --config ${{ matrix.release }} - name: Install @@ -188,7 +201,7 @@ jobs: run: cmake --install build --config ${{ matrix.release }} - name: Install pyuda - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-latest' run: > cp -r /usr/local/python_installer ${{github.workspace}}/python_installer && python3 -m venv ${{github.workspace}}/venv && @@ -198,41 +211,49 @@ jobs: pip3 install ${{github.workspace}}/python_installer - name: Test pyuda import - if: matrix.os == 'ubuntu-22.04' + if: matrix.os == 'ubuntu-latest' run: > source ${{github.workspace}}/venv/bin/activate && python3 -c 'import pyuda; client=pyuda.Client()' - name: Run non-SSL system tests - if: matrix.os == 'ubuntu-22.04' && matrix.ssl == 'OFF' + if: matrix.os == 'ubuntu-latest' && matrix.ssl == 'OFF' run: > sudo cp /usr/local/etc/uda.socket /usr/local/etc/uda@.service /etc/systemd/system && + sudo chown -R $USER:$USER /usr/local/etc && + echo "export UDA_SERVER_SSL_AUTHENTICATE=OFF" >> /usr/local/etc/udaserver.cfg && sudo systemctl start uda.socket && sudo systemctl enable uda.socket && - sudo chown -R $USER:$USER /usr/local/etc && nc -4zv localhost 56565 && export UDA_HOST=localhost && export UDA_PORT=56565 && - ./build/test/plugins/plugin_test_testplugin + ./build/test/plugins/plugin_test_testplugin && + ctest -V --test-dir build/test/unit_tests --output-on-failure && + uda_cli --help && uda_cli --request "help::help()" - name: Run SSL system tests if: matrix.os == 'ubuntu-22.04' && matrix.ssl == 'ON' - run: > - sudo cp /usr/local/etc/uda.socket /usr/local/etc/uda@.service /etc/systemd/system && - sudo chown -R $USER:$USER /usr/local/etc && - echo "export UDAHOSTNAME=github-ci-ssl" >> /usr/local/etc/udaserver.cfg && - ./scripts/create_certs.sh && - mkdir /usr/local/etc/certs && - cp rootCA.crt server.crt server.key /usr/local/etc/certs && - sudo systemctl start uda.socket && - sudo systemctl enable uda.socket && - nc -4zv localhost 56565 && - export UDA_HOST=localhost && - export UDA_PORT=56565 && - export UDA_CLIENT_SSL_AUTHENTICATE=1 && - export UDA_CLIENT_CA_SSL_CERT=$PWD/rootCA.crt && - export UDA_CLIENT_SSL_CERT=$PWD/client.crt && - export UDA_CLIENT_SSL_KEY=$PWD/client.key && + run: | + sudo cp /usr/local/etc/uda.socket /usr/local/etc/uda@.service /etc/systemd/system + sudo chown -R $USER:$USER /usr/local/etc + # pulls in extra config options from machine.d + echo "export UDAHOSTNAME=github-ci-ssl" >> /usr/local/etc/udaserver.cfg + ./scripts/create_certs.sh + mkdir /usr/local/etc/certs + cp rootCA.crt server.crt server.key /usr/local/etc/certs + sudo systemctl start uda.socket + sudo systemctl enable uda.socket + nc -4zv localhost 56565 + export UDA_HOST=localhost + export UDA_PORT=56565 + export UDA_CLIENT_SSL_AUTHENTICATE=OFF + export UDA_CLIENT_CA_SSL_CERT=$PWD/rootCA.crt + export UDA_CLIENT_SSL_CERT=$PWD/client.crt + export UDA_CLIENT_SSL_KEY=$PWD/client.key + set +e + /usr/local/bin/uda_cli --request "help::help()" && exit 1 || echo "process failed successfully" + set -e + export UDA_CLIENT_SSL_AUTHENTICATE=ON ./build/test/plugins/plugin_test_testplugin # - name: Test diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 09f4a1e0b..00ffb78cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,41 +1,206 @@ -# use the official gcc image, based on debian -# can use verions as well, like gcc:5.2 -# see https://hub.docker.com/_/gcc/ -image: gcc:7.3 +workflow: + rules: + - if: '$CI_COMMIT_BRANCH == "main"' + - if: '$CI_COMMIT_BRANCH == "develop"' + - if: '$CI_COMMIT_TAG' + - if: '$CI_PIPELINE_SOURCE == "web"' + +stages: + - build + - test + - package + +default: + tags: + - freia + before_script: + - . /etc/profile + - module use /usr/local/modules/default + - module use /common/software/micromamba/modules build: stage: build - # instead of calling g++ directly you can also use some build toolkit like make - # install the necessary build tools when needed - before_script: - - apt update && apt -y install cmake python3-dev libssl-dev libboost-dev swig libxml2-dev pkg-config ninja-build libtirpc-dev script: - - cmake -GNinja -Bbuild -H. -DCMAKE_INSTALL_PREFIX=. -DTARGET_TYPE=OTHER -DNO_MODULES=ON -DFAT_TESTS=ON - - ninja -C build - - ninja -C build install + - source scripts/cmake-freia.sh -DCMAKE_INSTALL_PREFIX=install -DCLIENT_ONLY=ON -DNO_JAVA_WRAPPER=ON -DSSLAUTHENTICATION=OFF + - cmake --build build_freia -j --config Release --target install artifacts: paths: - - lib/ - - include/ - - bin/ - - etc/ - - python_installer/ - - build/ - # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time - # cache: - # paths: - # - build/ + - install/ + +build_idl84: + stage: build + script: + - source scripts/cmake-freia-idl84.sh -DCMAKE_INSTALL_PREFIX=install_idl84 -DCLIENT_ONLY=ON -DNO_JAVA_WRAPPER=ON -DSSLAUTHENTICATION=OFF + - cmake --build build_freia -j --config Release --target install + artifacts: + paths: + - install_idl84/ -# run tests using the binary built before -test: +.build_pyuda_template: + stage: test dependencies: - build - stage: test script: - - cd build/test && LD_LIBRARY_PATH=$PWD/../../lib:$LD_LIBRARY_PATH ctest --timeout 60 + - module load capnproto/0.10.4 fmt/10.0.0 spdlog/1.11.0 + - module swap gcc/11.2.0 + - module use /common/software/micromamba/modules + - module load $PYTHON_MODULE + + - python3 -m venv venv${PYTHON_VERSION} --system-site-packages --symlinks + - source venv${PYTHON_VERSION}/bin/activate + - python3 -m pip install --upgrade pip wheel setuptools + - printf "${PYTHON_REQUIRES}" > requirements.txt + - python3 -m pip install -r requirements.txt + + # - sed -i '/oldest-supported-numpy/,+1d' install/python_installer/pyproject.toml + - COMPILER_DIR=`echo $PATH | awk -F':' '{for (i =1 ; i< NF ; ++i) {print $i}}' | grep gcc` + - export CC=$COMPILER_DIR/gcc + - export CXX=$COMPILER_DIR/g++ + - python3 -m pip install --prefix=install install/python_installer + - mkdir -p dist + - cp -r install/lib/python${PYTHON_VERSION}/site-packages/*uda* dist + + - export PYTHONPATH=dist:$PYTHONPATH + # - export LD_LIBRARY_PATH=install/lib:$LD_LIBRARY_PATH + - module use install/modulefiles + - UDA_MODULE=`ls install/modulefiles | grep uda` + - module load $UDA_MODULE + - python -c "import pyuda; client=pyuda.Client();" + + - module load uda-mast + - python -c "import mast" + - python3 -m pip install pytest + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@git.ccfe.ac.uk/MAST-U/mastcodes.git + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_geometry.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_images.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_meta.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_signals.py + - python3 -m pytest -v mastcodes/uda/python/tests/test_uda_xpad.py + artifacts: + paths: + - dist + +# python3.5: +# extends: .build_pyuda_template +# variables: +# PYTHON_VERSION: "3.5" +# PYTHON_MODULE: "python/3.5" +# PYTHON_REQUIRES: "Cython==0.29.32\nprogress" +# INSTALL_OPTION: "--prefix=install" + +python3.7: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.7" + PYTHON_MODULE: "python/3.7" + PYTHON_REQUIRES: "progress" + +python3.9: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.9" + PYTHON_MODULE: "python/3.9" + PYTHON_REQUIRES: "progress" + +python3.9_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.9" + PYTHON_MODULE: "python/3.9-minibundle" + PYTHON_REQUIRES: "progress" + +python3.10_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.10" + PYTHON_MODULE: "python/3.10-minibundle" + PYTHON_REQUIRES: "progress" + +python3.11_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.11" + PYTHON_MODULE: "python/3.11-minibundle" + PYTHON_REQUIRES: "progress" + +python3.12_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.12" + PYTHON_MODULE: "python/3.12-minibundle" + PYTHON_REQUIRES: "progress" + +python3.13_minibundle: + extends: .build_pyuda_template + variables: + PYTHON_VERSION: "3.13" + PYTHON_MODULE: "python/3.13-minibundle" + PYTHON_REQUIRES: "progress" + +.package-template: &package_template + stage: package + dependencies: + - build + - build_idl84 + # - python3.5 + - python3.7 + - python3.9 + - python3.9_minibundle + - python3.10_minibundle + - python3.11_minibundle + - python3.12_minibundle + - python3.13_minibundle + script: + - mkdir -p install/idl/8.3/idl + - mkdir -p install/idl/8.4/idl + + # Move IDL 8.3 files + - mv install/idl/*.pro install/idl/8.3/idl/ + - mv install/dlm install/idl/8.3/ + + # Move IDL 8.4 files + - mv install_idl84/idl/*.pro install/idl/8.4/idl/ + - mv install_idl84/dlm install/idl/8.4/ + + # Create modulefiles + # - UDA_GIT_TAG=$(git describe --tags --abbrev=0) + - MODULEFILE=$(ls install/modulefiles/uda/* | head -n 1) + - cp ${MODULEFILE} ${MODULEFILE}-idl-8.4 + - sed -ri '/([^\$]UDA_IDL_DIR)/ s/$/\/idl\/8.3/' $MODULEFILE + - sed -ri '/([^\$]UDA_IDL_DIR)/ s/$/\/idl\/8.4/' ${MODULEFILE}-idl-8.4 + + # Create uda-devel modulefile + - MODULE_TAG=${MODULEFILE##*/} + - mkdir -p install/modulefiles/uda-devel + - DEV_MODULEFILE=install/modulefiles/uda-devel/${MODULE_TAG} + - cp ${MODULEFILE} ${DEV_MODULEFILE} + - sed -i '/idam/a module load capnproto\nmodule load fmt' ${DEV_MODULEFILE} + + - mkdir -p install/python + - cp -r dist/* install/python + + - TAG=$(git describe --tags || echo "untagged") + - tar -czf uda-$TAG.tar.gz -C install . + # artifacts: + # paths: + # - uda-*.tar.gz + +package:branch: + <<: *package_template artifacts: - reports: - junit: - - "*_out.xml" - - plugins/*_out.xml - - imas/*_out.xml + paths: + - uda-*.tar.gz + expire_in: 1 week + only: + - branches + except: + - tags + +package:tag: + <<: *package_template + artifacts: + paths: + - uda-*.tar.gz + expire_in: 2 years + only: + - tags diff --git a/README.md b/README.md index 976c4a292..3138573d8 100755 --- a/README.md +++ b/README.md @@ -31,10 +31,10 @@ The UDA git repository can be cloned from: ## Getting the UDA client -The easiest way to obtain the client is to pip install the python wrapper (pyuda), wheels are uploaded for every tagged release from version 2.7.6. Further details are available on [pypi](https://pypi.org/project/uda/). +The easiest way to obtain the client is to pip install the python wrapper (pyuda), wheels are uploaded for every tagged release from version 2.7.6. Further details are available on [pypi](https://pypi.org/project/pyuda/). ```sh -pip install uda +pip install pyuda ``` For any other use cases please see the documentation to build from source [here](https://ukaea.github.io/UDA/client_installation/). diff --git a/docs/authentication.md b/docs/authentication.md index 46eb6aa8e..1924c2e73 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -1,7 +1,7 @@ --- layout: default title: Authentication -nav_order: 6 +nav_order: 4 --- TODO: Notes on configuring authenticated connections diff --git a/docs/client_installation.md b/docs/client_installation.md index 5a864e054..591ffbb67 100644 --- a/docs/client_installation.md +++ b/docs/client_installation.md @@ -1,6 +1,6 @@ --- layout: default -title: Installing a UDA client +title: Client installation nav_order: 3 --- @@ -20,7 +20,7 @@ project CMake configuration. It's worth noting, however, that pre-built Docker i available for your platform. For python specifically, wheels are built for a range of architecure/OS/python-version combinations and all tagged releases -past version 2.7.6 are available to pip install through pypi [here](https://pypi.org/project/uda/). Windows is supported +past version 2.7.6 are available to pip install through pypi [here](https://pypi.org/project/pyuda/). Windows is supported from version 2.8.0. The range of available dockerfiles are stored in the UDA repository [here](https://github.com/ukaea/UDA/tree/main/docker), @@ -40,7 +40,7 @@ additional client wrapper langauges or additinal platforms for python wheels or python3 -m venv venv source venv/bin/activate python -m pip install --upgrade pip -python -m pip install uda +python -m pip install pyuda ``` The python syntax to request a data item would then be: diff --git a/docs/client_usage.md b/docs/client_usage.md new file mode 100644 index 000000000..9f5248ab3 --- /dev/null +++ b/docs/client_usage.md @@ -0,0 +1,94 @@ +--- +layout: default +title: Using the UDA client +nav_order: 5 +--- + +# Using the UDA client +{:.no_toc} + +This section will describe the main features to be aware of in the UDA client library. + +## Contents +{:.no_toc} +1. TOC +{:toc} + +## Connection details + +Two fields define the server that a client will connect to: the server address (as an IP or DNS name) and a port number. These details can either be set as environment variables in the shell before launching your client application, or can be set through API functions (the exact syntax will depend on the wrapper you use). + +syntax using environment variables: +```sh +export UDA_HOST= +export UDA_PORT= +``` + +example syntax using the python wrapper: +```py +pyuda.Client.server = "" +pyuda.Client.port = +``` + +### Authentication + +UDA currently supports authenticaed server access using SSL certificates. The options and certificate locations must be set using environment variables before you connect to an authenticated server. See the Authentication page for more details. + +## Configuration flags and settings + +There are a number of options that can be set to configure the client properties. + +### Timeout + +When you establish a connection with a UDA server the same connection stays alive for either the duration of the client process (ie multiple data requests are all served by the same server) or when the server timeout duration has passed. + +There are currently 2 modes for how this timeout behavior works and a configuration option exists to toggle between the two. The first (and default) setting is where the timeout time describes the total lifetime of the server. Once this has passed the server process will be killed and a new one will start to server further requests from the client. The second setting is where the timeout time only accumulates between data requests and is reset after each one is fulfilled, so the server is only killed due to inactivity after a specified idle time. + +The value of the timeout duration is also customisable and by default is set to 10 minutes. + +example syntax using the python wrapper: +```py + +# query the current status of the IDLE_TIMEOUT flag (default is false for total-lifetime behaviour) +client.get_property(pyuda.Properties.IDLE_TIMEOUT) + +# change the timeour mode to idle time instead of total lifetime +client.set_property(pyuda.Properties.IDLE_TIMEOUT, True) + +# change the timeout duration from 600s (default) to 30 seconds +client.set_property(pyuda.Properties.TIMEOUT, 30) + +``` + +### Type-casting of returned data + +A number of flags exist to force a conversion of returned data to a specific data type. These optionas all default to false which means the original data type from the data source will be returned unless otherwise specified. + +```py + +# cast data to double precision floating point numbers +client.set_property(pyuda.Properties.GET_DATA_DOUBLE, True) + +# cast dimension data to double precision floating point numbers +client.set_property(pyuda.Properties.GET_DIM_DOUBLE, True) + +# cast time data to double precision floating point numbers +client.set_property(pyuda.Properties.GET_TIME_DOUBLE, True) +``` + +### Ignoring dimension data + +UDA usually returns array data in a rich structure containing the data itself along with labels, units, and dimension data. Dimensions could be the time points the data was recorded at for a 1D array, or perhaps time and 2 position vectors for a 3D array. + +An option exists to request just the data itself and not any dimension values. The default behaviour is dimension data is always returned to the client where it exists. + +```py +client.set_property(pyuda.Properties.GET_NO_DIM_DATA, True) + +``` + +## TODO: The get and put APIs + +### TODO: calling plugin functions + +### TODO: subsetting syntax diff --git a/docs/creating_plugins.md b/docs/creating_plugins.md index c0dffbc65..920dbd9bf 100644 --- a/docs/creating_plugins.md +++ b/docs/creating_plugins.md @@ -1,7 +1,7 @@ --- layout: default title: Creating a UDA plugin -nav_order: 2 +nav_order: 8 --- ## UDA plugin diff --git a/docs/development.md b/docs/development.md index 13a015ee7..47b153c7e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,7 +1,7 @@ --- layout: default title: UDA Development -nav_order: 5 +nav_order: 7 --- # UDA Development @@ -14,24 +14,24 @@ nav_order: 5 ## Setting up a development environment -To begin developing you will need to fork the uda repository from github, build the project, and run a local development server for testing before making your changes. +To begin developing, you will need to fork the uda repository from GitHub, build the project, and run a local development server for testing before making your changes. ### Creating a development build Follow the instructions in [server installation](/UDA/server_installation) for building an uda server from source for your platform. You can optionally change the cmake config option to generate debug symbols if required: ```sh -cmake --build build --config Debug --target intall +cmake --build build --config Debug --target install ``` ### Run the development server -There is a simple way to temporarily run the uda super-server daemon under xinted for testing or development only that doesn't require the same elevated privileges as the system-level install described in the [server installation](/UDA/server_installation) section; this method is currently only available for xinetd and is described in the next section. +There is a simple way to temporarily run the uda super-server daemon under xinetd for testing or development only that doesn't require the same elevated privileges as the system-level install described in the [server installation](/UDA/server_installation) section; this method is currently only available for xinetd and is described in the next section. -Note that if you need to run your server under `launchd` (for MacOS) or `systemd`, you will need to follow the instructions on the [server installation](/UDA/server_installation) page to run your development server. +Note that if you need to run your server under `launchd` (for macOS) or `systemd`, you will need to follow the instructions on the [server installation](/UDA/server_installation) page to run your development server. ### Running a development server under xinetd Once uda has been built there will be a script called `rc.uda` in the `etc` subdirectory of your uda install which can be used as shown below to start a xinetd process listening on the port specified in `etc/xinetd.conf`. Xinetd here is used as a super-server daemon which will launch new uda server processes for each new incoming client connection. -The `rc.uda` script takes one of 3 possible arguments: `start` to start the xinetd process, `stop` to kill a running xinetd process (from the pid written to a file called `xinetd..pid`), and `status` which will print whether a valid pid file is found or not, or if one is found but the process no longer exists. +The `rc.uda` script takes one of three possible arguments: `start` to start the xinetd process, `stop` to kill a running xinetd process (from the pid written to a file called `xinetd..pid`), and `status` which will print whether a valid pid file is found or not, or if one is found but the process no longer exists. ```sh ./rc.uda start @@ -39,7 +39,7 @@ The `rc.uda` script takes one of 3 possible arguments: `start` to start the xine ``` The text `server running` will be output to the console if this executes successfully. If the `start` command failed for any reason the status will output `xinetd..pid not found, server not running` instead. Logging from the xinetd process (including messages from successful or failed startup attempts) will be written to a file called `mylog.`. Note that the port number specified in `xinetd.conf` must be unique for each running uda server installation, so may need to be changed from the default 56565 value if that is already in use by another server on the same host. -Note that this method of running the xinetd super-server daemon is not appropriate for running a production server and should be used for development purposes only. See the instructions in the [server installation](/UDA/server_installation) guide instead for the recommended deployment methods. +Note that this method of running the xinetd super-server daemon is not appropriate for running a production server and should be used for development only. See the instructions in the [server installation](/UDA/server_installation) guide instead for the recommended deployment methods. ### Running a development server under launchd (MacOS) See the launchd section of the instructions for [building and running a server](/UDA/server_installation#launchd). @@ -49,11 +49,11 @@ TODO: fat client instructions ## Running tests -The first two tests here will verify that a server can receive connections, the final section describes how to run the automated test suite. +The first two sections here describe tests that verify that a server can receive connections, and the final section describes how to run the automated test suite. ### Server socket connection is open and receiving connections -You can use the ncat commandline tool to attempt to connect to the server socket. Here the `-v` option sets the output level to verbose, `-z` is used to report the connection status only, and and the `-4` option specifies the use of IPv4 only. +You can use the ncat commandline tool to attempt to connect to the server socket. Here the `-v` option sets the output level to verbose, `-z` is used to report the connection status only, and the `-4` option specifies the use of IPv4 only. ```sh nc -zv -4 localhost 56565 @@ -95,16 +95,16 @@ export UDA_PORT=56565 TODO: testing strategy and CTest integration ## Debugging -This section will mainly focus on how to debug an uda server because of the complications of the client-server communication. For debugging either the client or the fat-client there aren't any additional considerations to be aware of in the same way, simply compile with debug symbols and use your usual choice of debugging tool (`gdb`, `lldb`, etc.). +This section will mainly focus on how to debug an uda server because of the complications of the client-server communication. For debugging either the client or the fat-client, there aren't any additional considerations to be aware of in the same way, compile with debug symbols and use your usual choice of debugging tool (`gdb`, `lldb`, etc.). ### Server debugging procedure -The uda server process is launched by the super-server daemon (i.e. `xinetd` or `systemd`) only when a client request is received, and only runs while that client connection is alive. The server debugging process must be launched after a request has been receieved (and the server process has actually started), but before the request has been completed by the server. +The uda server process is launched by the super-server daemon (i.e. `xinetd` or `systemd`) only when a client request is received, and only runs while that client connection is alive. The server debugging process must be launched after a request has been received (and the server process has actually started), but before the request has been completed by the server. The general server debugging procedure therefore involves using 2 separate debugging processes (ie `gdb` or `lldb` sessions): one on the client side which will pause the execution of the clientside routine at the correct time, and a separate debugging session on the server which can only be started when the client-side session hits its first breakpoint. ### Pausing the client-side execution using gdb -The simplest client to use for debugging will be the `uda_cli` launched from the same host where the server is installed. The example below shows the syntax to use, the `"help::help()"` request is a placehodlder and can be replaced with whatever you are interested in testing. The line-number of the break point is selected to pause the client-side execution just after the request has been sent from the client to the server. +The simplest client to use for debugging will be the `uda_cli` launched from the same host where the server is installed. The example below shows the syntax to use, the `"help::help()"` request is a placeholder and can be replaced with whatever you are interested in testing. The line-number of the break point is selected to pause the client-side execution just after the request has been sent from the client to the server. ```sh gdb --args uda_cli -h localhost -p 56565 "help::help()" @@ -113,7 +113,7 @@ run ``` ### Breaking into the server process -First you need to find the PID corresponding to the specific test-server process you're interested in. It's useful to note that the default name of all uda server processes will be `uda_server` (the name of the binary in the `bin` directory of your install location). If you have more than one server running, or more than one client connection open at that time to your test server, you will have to manually locate the correct PID. +First, you need to find the PID corresponding to the specific test-server process you're interested in. It's useful to note that the default name of all uda server processes will be `uda_server` (the name of the binary in the `bin` directory of your install-location). If you have more than one server running, or more than one client connection open at that time to your test server, you will have to manually locate the correct PID. ```sh # list all uda_server process ID numbers, all that's needed if there's only 1 @@ -133,7 +133,7 @@ cont ### Unlocking the client-side process and debugging the server Use the `cont` command on the client-side debugging process when you're ready for the server-side execution to progress. -From this point you can continue to use `gdb` on the server process as you would usually. +From this point you can continue to use `gdb` on the server process as you usually would. ## Code style conventions @@ -145,7 +145,7 @@ TODO: UML diagrams ### Server -### Clientserver +### Client-server ### Client diff --git a/docs/server_installation.md b/docs/server_installation.md index 16004d1dd..27a30c99b 100644 --- a/docs/server_installation.md +++ b/docs/server_installation.md @@ -1,7 +1,7 @@ --- layout: default title: Server Installation -nav_order: 4 +nav_order: 2 --- # UDA server installation @@ -148,7 +148,17 @@ There is a directory in `/etc` called `machine.d` in which general Most general server options are set in `/etc/udaserver.cfg` including the debug level, the file locations of plugin-registration files, and to add the library locations of uda and any uda-plugin builds (as well as any other dependencies such as imas) to the LD_LIBRARY_PATH. -Note that the UDA_LOG_LEVEL should Generally be set to ERROR for a production deployment. The DEBUG level will impose severe a performance penalty, but can be used to print some very verbose logs for each incoming request during server development. +Note that the UDA_LOG_LEVEL should Generally be set to ERROR for a production deployment. The DEBUG level will impose severe a performance penalty, but can be used to print some very verbose logs for each incoming request during server development. + +##### UDA_ALLOWED_PATHS +This environment variable defines the permitted file paths for the UDA server. By default, all file access is prohibited if this is unset or empty. Multiple paths can be specified using semi-colons (`;`). Setting the path to `/` allows unrestricted access. + +**Example** +```sh +export UDA_ALLOWED_PATHS=/ +``` +**Security Note** +Restrict the paths as narrowly as possible to limit server file access and reduce risk. ### Plugin-specific options diff --git a/docs/training_events.md b/docs/training_events.md index 982f6bbbd..9f7b8e99a 100644 --- a/docs/training_events.md +++ b/docs/training_events.md @@ -1,7 +1,7 @@ --- layout: default title: Training and Events (WIP) -nav_order: 8 +nav_order: 9 --- # Training and Events diff --git a/docs/wrappers/index.md b/docs/wrappers/index.md index 00973d1c5..336011378 100644 --- a/docs/wrappers/index.md +++ b/docs/wrappers/index.md @@ -1,7 +1,7 @@ --- layout: default title: Wrappers -# nav_order: 6 +nav_order: 6 has_children: true --- diff --git a/extlib/portablexdr-4.9.1/CMakeLists.txt b/extlib/portablexdr-4.9.1/CMakeLists.txt index 2e7603672..9b0d56f42 100644 --- a/extlib/portablexdr-4.9.1/CMakeLists.txt +++ b/extlib/portablexdr-4.9.1/CMakeLists.txt @@ -5,12 +5,16 @@ cmake_policy( SET CMP0048 NEW ) project( xdr VERSION 4.9.1 ) if( NOT WIN32 ) - set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -fno-strict-aliasing" ) + set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -fno-strict-aliasing -std=gnu89 -Wno-old-style-definition -Wno-strict-prototypes" ) endif() include( TestBigEndian ) if( WIN32 ) + if( CMAKE_CXX_COMPILER_ID STREQUAL "GNU" ) + set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=gnu89 -Wno-old-style-definition -Wno-strict-prototypes" ) + endif() + add_definitions( -DBIG_ENDIAN=1 -DLITTLE_ENDIAN=2 ) test_big_endian( BIG_ENDIAN ) @@ -37,6 +41,10 @@ else() add_library( ${PROJECT_NAME} STATIC ${SOURCES} ) endif() +if( CMAKE_C_COMPILER_ID MATCHES "GNU|Clang" ) + target_compile_options( ${PROJECT_NAME} PRIVATE -Wno-incompatible-pointer-types ) +endif() + install( FILES rpc/rpc.h rpc/types.h rpc/xdr.h ${CMAKE_BINARY_DIR}/config.h DESTINATION include/rpc diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 5ba20c8b1..ea767580e 100755 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -27,6 +27,12 @@ else() add_definitions( -Wno-use-after-free ) endif() endif() + if( APPLE ) + add_definitions( -Wno-cast-function-type ) + if ( ${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL 17 ) + add_definitions( -Wno-cast-function-type-mismatch ) + endif( ) + endif() endif() if( NOT WIN32 AND NOT MINGW ) diff --git a/source/authentication/udaClientSSL.cpp b/source/authentication/udaClientSSL.cpp index 16aff91ba..ee1845b97 100755 --- a/source/authentication/udaClientSSL.cpp +++ b/source/authentication/udaClientSSL.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -327,7 +328,7 @@ int initUdaClientSSL() // Has the user directly specified SSL/TLS authentication? // Does the connection entry in the client host configuration file have the three SSL authentication files - if (!g_sslProtocol && !getenv("UDA_CLIENT_SSL_AUTHENTICATE")) { + if (!g_sslProtocol && !uda::common::env_config::evaluate_bool_param("UDA_CLIENT_SSL_AUTHENTICATE", false)) { g_sslDisabled = true; if (g_host != nullptr) { diff --git a/source/authentication/udaServerSSL.cpp b/source/authentication/udaServerSSL.cpp index 62ba90ac3..5ba3690b0 100644 --- a/source/authentication/udaServerSSL.cpp +++ b/source/authentication/udaServerSSL.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -280,11 +281,9 @@ int startUdaServerSSL() } // Has the server disabled SSL/TLS authentication? - if (!getenv("UDA_SERVER_SSL_AUTHENTICATE")) { - g_sslDisabled = true; + g_sslDisabled = !uda::common::env_config::evaluate_bool_param("UDA_SERVER_SSL_AUTHENTICATE", false); + if (g_sslDisabled) { return 0; - } else { - g_sslDisabled = false; } UDA_LOG(UDA_LOG_DEBUG, "SSL Authentication is Enabled!\n"); diff --git a/source/bin/CMakeLists.txt b/source/bin/CMakeLists.txt index 8267775fb..43e6031f5 100644 --- a/source/bin/CMakeLists.txt +++ b/source/bin/CMakeLists.txt @@ -50,3 +50,9 @@ install( DESTINATION bin PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ ) + +install( + FILES log_parser.py + DESTINATION bin + PERMISSIONS OWNER_EXECUTE OWNER_READ OWNER_WRITE GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ +) diff --git a/source/bin/log_parser.py b/source/bin/log_parser.py new file mode 100644 index 000000000..9540f181c --- /dev/null +++ b/source/bin/log_parser.py @@ -0,0 +1,65 @@ +#!/bin/env python3 + +import re +import json +from collections import namedtuple + +RE_STRING = r'(?P\d+\.\d+\.\d+\.\d+) - (?P[\w\d_\-]+)? \[(?P[a-zA-Z0-9: ]+)\] \[(?P\d+) (?P.+) (?P-?\d+) (?P-?\d+) (?P[\w\d\-_\.\/+]+)? (?P[\w\d\-_\.\/+]+)? (?P[a-zA-Z_]+)? (?P[a-zA-Z_]+)? (?P[a-zA-Z_]+)? \] (?P\d+) (?P\d+) \[(?P.*)\] (?P\d+(?:\.\d*(?:e[+\-]?\d+)?)?) (?P\d+) (?P\d+) \[(?P\d+) (?P\d+)\] \[\]' + +log_regex = re.compile(RE_STRING) + +LogLine = namedtuple('LogLine', + ['host', 'uid', 'datetime', 'req_id', 'signal', 'exp_no', 'pass_no', 'path', + 'file', 'format', 'archive', 'device', 'err_code', 'db_size', 'err_msg', + 'elapsed_time', 'client_ver', 'server_ver', 'client_pid', 'server_pid']) + +line_start_regex = re.compile(r'^\d+\.\d+\.\d+\.\d+ - ') + + +def log_line_generator(lines) -> (int, str): + """ + Generator which joins log entries split by spurious newline + characters, yielding full lines and the starting line-number of + that entry in the original file + """ + buffer = [] + start_line_num = None + + for i, line in enumerate(lines): + if line_start_regex.match(line): + if buffer: + yield start_line_num, ''.join(buffer) + buffer = [] + start_line_num = i + buffer.append(line.rstrip('\r\n')) + if buffer: + yield start_line_num, ''.join(buffer) + + +def parse(logfile, outfile): + lines = [] + with open(logfile) as file: + for i, line in log_line_generator(file): + match = log_regex.match(line) + if not match: + print(f'failed to parse log line {i + 1}:\n{line}', file=sys.stderr) + continue + groups = match.groups() + log_line = LogLine(*groups) + lines.append(log_line._asdict()) + with open(outfile, 'w') as file: + json.dump(lines, file, indent=2) + + +if __name__ == '__main__': + import sys + + args = sys.argv + if len(args) != 3: + print(f'usage: {args[0]} logfile outfile', file=sys.stderr) + raise SystemExit() + + logfile = args[1] + outfile = args[2] + + parse(logfile, outfile) diff --git a/source/bin/uda_cli.cpp b/source/bin/uda_cli.cpp index 527dc6217..b95014a38 100644 --- a/source/bin/uda_cli.cpp +++ b/source/bin/uda_cli.cpp @@ -7,6 +7,30 @@ #include #include #include +#include +#include +#include + +//NOTE: redefinition of UDA complex types here to avoid dragging in unnecessary dependencies. +// To be removed when headers are cleaned up in uda v3.0 +typedef struct DComplex { + double real; + double imaginary; +} DCOMPLEX; + +typedef struct Complex { + float real; + float imaginary; +} COMPLEX; + +template +struct is_uda_complex : std::false_type {}; + +template<> +struct is_uda_complex : std::true_type {}; + +template<> +struct is_uda_complex : std::true_type {}; struct CLIException : public uda::UDAException { @@ -42,6 +66,21 @@ std::ostream& operator<<(std::ostream& out, const std::vector& vec) return out; } +template::value, bool> = false> +std::ostream& operator<<(std::ostream& out, const T& complex) +{ + std::ios::fmtflags old_flags = out.flags(); + std::streamsize prec = out.precision(); + out << std::fixed << std::setprecision(3); + + char sign = (complex.imaginary >= 0) ? '+' : '-'; + out << complex.real << " " << sign << " " << std::abs(complex.imaginary) << "i"; + + out.flags(old_flags); + out.precision(prec); + return out; +} + namespace po = boost::program_options; bool replace(std::string& str, const std::string& from, const std::string& to) @@ -219,6 +258,12 @@ void print_capnp_node(TreeReader* tree, NodeReader* node, const std::string& ind case UDA_TYPE_DOUBLE: print_capnp_data(node, shape, indent); break; + case UDA_TYPE_COMPLEX: + print_capnp_data(node, shape, indent); + break; + case UDA_TYPE_DCOMPLEX: + print_capnp_data(node, shape, indent); + break; } } @@ -383,6 +428,41 @@ void conflicting_options(const boost::program_options::variables_map & vm, } } +void print_result(const uda::Result& res) { + if (res.isTree()) { + print_tree(res.tree(), ""); + } else if (res.uda_type() == UDA_TYPE_CAPNP) { +#ifdef CAPNP_ENABLED + print_capnp(res.raw_data(), res.size()); +#else + std::cout << "Cap'n Proto not enabled - cannot display data\n"; +#endif + } else { + print_data(res.data(), res.uda_type()); + } +} + +void process_request(uda::Client& client, const std::string& request, const std::string& source) { + std::cout << "request: " << request << "\n"; + const auto& res = client.get(request, source); //throws + print_result(res); +} + +void process_batch_requests(uda::Client& client, const std::vector& requests, const std::string& source) { + size_t count = 0; + for (const auto& request : requests) { + try { + process_request(client, request, source); + count++; + } catch (const std::exception& ex) { + std::cout << "error: " << ex.what() << "\n"; + } + } + if (count == 0 and !requests.empty()) { + throw CLIException("All requests in batch failed"); + } +} + int main(int argc, const char** argv) { po::options_description desc("Allowed options"); @@ -391,6 +471,7 @@ int main(int argc, const char** argv) ("host,h", po::value()->default_value("localhost"), "server host name") ("port,p", po::value()->default_value(56565), "server port") ("request", po::value(), "request") + ("batch-file,f", po::value(), "file of batch requests, one per line") ("source", po::value()->default_value(""), "source") ("ping", po::bool_switch(), "ping the server"); @@ -402,27 +483,28 @@ int main(int argc, const char** argv) po::store(po::command_line_parser(argc, argv).options(desc).positional(p).run(), vm); po::notify(vm); - conflicting_options(vm, "ping", "request"); - if (!vm["ping"].as() && vm.count("request") == 0) { - throw po::error("either 'ping' or 'request' must be provided"); - } - } catch (po::error& err) { - if (vm["help"].as()) { - std::cout << "Usage: " << argv[0] << " [options] request\n"; - std::cout << desc << "\n"; - return 1; - } else { - std::cout << "Error: " << err.what() << "\n\n"; + if (vm.count("help") && vm["help"].as()) { std::cout << "Usage: " << argv[0] << " [options] request\n"; std::cout << desc << "\n"; - return -1; + return 0; } - }; - if (vm["help"].as()) { + conflicting_options(vm, "ping", "request"); + conflicting_options(vm, "ping", "batch-file"); + conflicting_options(vm, "request", "batch-file"); + if (!vm["ping"].as() && vm.count("request") == 0 && vm.count("batch-file") == 0) { + throw po::error("either 'ping', 'request' or 'batch-file' must be provided"); + } + } catch (const po::unknown_option& err) { + std::cout << "Error: " << err.what() << "\n\n"; + std::cout << "Usage: " << argv[0] << " [options] request\n"; + std::cout << desc << "\n"; + return -1; + } catch (po::error& err) { + std::cout << "Error: " << err.what() << "\n\n"; std::cout << "Usage: " << argv[0] << " [options] request\n"; std::cout << desc << "\n"; - return 1; + return -1; } if (vm.count("host")) { @@ -433,39 +515,34 @@ int main(int argc, const char** argv) uda::Client::setServerPort(static_cast(vm["port"].as())); } - std::string request; + std::vector requests; if (vm["ping"].as()) { - request = "HELP::ping()"; + requests.emplace_back("HELP::ping()"); + } else if (vm.count("request") > 0) { + if (const auto request = vm["request"].as(); request == "-") { + std::string line; + while (std::getline(std::cin, line)) { + requests.push_back(line); + } + } else { + requests.emplace_back(request); + } } else { - request = vm["request"].as(); - } - - if (request == "-") { - std::stringstream ss; + std::ifstream batch_file(vm["batch-file"].as()); std::string line; - while (std::getline(std::cin, line)) { - ss << line; + while (std::getline(batch_file, line)) { + requests.push_back(line); } - request = ss.str(); } - std::cout << "request: " << request << "\n"; std::string source = vm["source"].as(); uda::Client client; try { - auto& res = client.get(request, source); - - if (res.isTree()) { - print_tree(res.tree(), ""); - } else if (res.uda_type() == UDA_TYPE_CAPNP) { -#ifdef CAPNP_ENABLED - print_capnp(res.raw_data(), res.size()); -#else - std::cout << "Cap'n Proto not enabled - cannot display data\n"; -#endif + if (requests.size() == 1) { + process_request(client, requests[0], source); } else { - print_data(res.data(), res.uda_type()); + process_batch_requests(client, requests, source); } client.close(); @@ -476,4 +553,4 @@ int main(int argc, const char** argv) } return 0; -} \ No newline at end of file +} diff --git a/source/client/accAPI.cpp b/source/client/accAPI.cpp index 0d41ac0dc..b811cef86 100755 --- a/source/client/accAPI.cpp +++ b/source/client/accAPI.cpp @@ -16,6 +16,7 @@ #endif #include +#include #include #include #include @@ -98,7 +99,7 @@ int getThreadId(thread_t id) // Lock the thread and set the previous STATE void udaLockThread() { - CLIENT_FLAGS* client_flags = udaClientFlags(); + // CLIENT_FLAGS* client_flags = udaClientFlags(); static unsigned int mutex_initialised = 0; if (!mutex_initialised) { @@ -153,7 +154,7 @@ void udaLockThread() //putIdamClientEnvironment(&idamState[id].environment); putIdamThreadClientBlock(&idamState[id].client_block); putIdamThreadServerBlock(&idamState[id].server_block); - client_flags->flags = idamState[id].client_block.clientFlags; + // client_flags->flags = idamState[id].client_block.clientFlags; putIdamThreadLastHandle(idamState[id].lastHandle); } else { putIdamThreadLastHandle(-1); @@ -238,6 +239,9 @@ void putIdamThreadLastHandle(int handle) void acc_freeDataBlocks() { + for (auto& data_block : data_blocks) { + freeDataBlock(&data_block); + } data_blocks.clear(); putIdamThreadLastHandle(-1); } @@ -461,7 +465,7 @@ void udaSetProperty(const char* property) } else { if (STR_IEQUALS(property, "verbose")) udaSetLogLevel(UDA_LOG_INFO); if (STR_IEQUALS(property, "debug")) udaSetLogLevel(UDA_LOG_DEBUG); - if (STR_IEQUALS(property, "altData")) client_flags->flags = client_flags->flags | CLIENTFLAG_ALTDATA; + if (STR_IEQUALS(property, "altData")) client_flags->flags |= CLIENTFLAG_ALTDATA; if (!strncasecmp(property, "altRank", 7)) { strncpy(name, property, 55); name[55] = '\0'; @@ -475,9 +479,11 @@ void udaSetProperty(const char* property) } } } - if (STR_IEQUALS(property, "reuseLastHandle")) client_flags->flags = client_flags->flags | CLIENTFLAG_REUSELASTHANDLE; - if (STR_IEQUALS(property, "freeAndReuseLastHandle")) client_flags->flags = client_flags->flags | CLIENTFLAG_FREEREUSELASTHANDLE; - if (STR_IEQUALS(property, "fileCache")) client_flags->flags = client_flags->flags | CLIENTFLAG_FILECACHE; + if (STR_IEQUALS(property, "reuseLastHandle")) client_flags->flags |= CLIENTFLAG_REUSELASTHANDLE; + if (STR_IEQUALS(property, "freeAndReuseLastHandle")) client_flags->flags |= CLIENTFLAG_FREEREUSELASTHANDLE; + if (STR_IEQUALS(property, "fileCache")) client_flags->flags |= CLIENTFLAG_FILECACHE; + if (STR_IEQUALS(property, "dbOnly")) client_flags->flags |= CLIENTFLAG_DB_ONLY; + if (STR_IEQUALS(property, "idleTimeout")) client_flags->flags |= CLIENTFLAG_IDLE_TIMEOUT; } } @@ -513,6 +519,8 @@ int udaGetProperty(const char* property) if (STR_IEQUALS(property, "debug")) return udaGetLogLevel() == UDA_LOG_DEBUG; if (STR_IEQUALS(property, "altData")) return (int)(client_flags->flags & CLIENTFLAG_ALTDATA); if (STR_IEQUALS(property, "fileCache")) return (int)(client_flags->flags & CLIENTFLAG_FILECACHE); + if (STR_IEQUALS(property, "dbOnly")) return (int)(client_flags->flags & CLIENTFLAG_DB_ONLY); + if (STR_IEQUALS(property, "idleTimeout")) return (int)(client_flags->flags & CLIENTFLAG_IDLE_TIMEOUT); } return 0; } @@ -551,6 +559,8 @@ void udaResetProperty(const char* property) client_flags->flags &= !CLIENTFLAG_FREEREUSELASTHANDLE; } if (STR_IEQUALS(property, "fileCache")) client_flags->flags &= !CLIENTFLAG_FILECACHE; + if (STR_IEQUALS(property, "dbOnly")) client_flags->flags &= !CLIENTFLAG_DB_ONLY; + if (STR_IEQUALS(property, "idleTimeout")) client_flags->flags &= !CLIENTFLAG_IDLE_TIMEOUT; } } @@ -926,7 +936,9 @@ unsigned int getIdamCachePermission(int handle) unsigned int getIdamTotalDataBlockSize(int handle) { if (handle < 0 || (unsigned int)handle >= data_blocks.size()) return 0; - return data_blocks[handle].totalDataBlockSize; + // return data_blocks[handle].totalDataBlockSize; + auto data_block = data_blocks[handle]; + return countDataBlockSize(&data_block, &data_block.client_block); } //! returns the atomic or structure type id of the data object @@ -1159,7 +1171,6 @@ char* getIdamSyntheticData(int handle) int status = getIdamDataStatus(handle); if (handle < 0 || (unsigned int)handle >= data_blocks.size()) return nullptr; if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !client_flags->get_bad) return nullptr; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || client_flags->get_bad)) return nullptr; if (!client_flags->get_synthetic || data_blocks[handle].error_model == ERROR_MODEL_UNKNOWN) { return data_blocks[handle].data; } @@ -1179,7 +1190,6 @@ char* getIdamData(int handle) int status = getIdamDataStatus(handle); if (handle < 0 || (unsigned int)handle >= data_blocks.size()) return nullptr; if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !client_flags->get_bad) return nullptr; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || client_flags->get_bad)) return nullptr; if (!client_flags->get_synthetic) { return data_blocks[handle].data; } else { @@ -1518,7 +1528,6 @@ void getIdamDoubleData(int handle, double* fp) int status = getIdamDataStatus(handle); if (handle < 0 || (unsigned int)handle >= data_blocks.size()) return; if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !client_flags->get_bad) return; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || client_flags->get_bad)) return; if (data_blocks[handle].data_type == UDA_TYPE_DOUBLE) { if (!client_flags->get_synthetic) @@ -1659,7 +1668,6 @@ void getIdamFloatData(int handle, float* fp) return; } if (status == MIN_STATUS && !data_blocks[handle].client_block.get_bad && !client_flags->get_bad) return; - if (status != MIN_STATUS && (data_blocks[handle].client_block.get_bad || client_flags->get_bad)) return; if (data_blocks[handle].data_type == UDA_TYPE_FLOAT) { if (!client_flags->get_synthetic) diff --git a/source/client/makeClientRequestBlock.cpp b/source/client/makeClientRequestBlock.cpp index c1f2a8618..0c6b75d9e 100755 --- a/source/client/makeClientRequestBlock.cpp +++ b/source/client/makeClientRequestBlock.cpp @@ -154,14 +154,7 @@ int makeClientRequestBlock(const char** signals, const char** sources, int count } void freeClientRequestBlock(REQUEST_BLOCK* request_block) { - if(request_block != nullptr && request_block->requests != nullptr) { - for (int i = 0; i < request_block->num_requests; i++) { - freeNameValueList(&request_block->requests[i].nameValueList); - freeClientPutDataBlockList(&request_block->requests[i].putDataBlockList); - } - free(request_block->requests); - request_block->requests = nullptr; - } + freeRequestBlock(request_block); } int shotRequestTest(const char* source) diff --git a/source/client/udaClient.cpp b/source/client/udaClient.cpp index 61e33170a..c78aef4a4 100755 --- a/source/client/udaClient.cpp +++ b/source/client/udaClient.cpp @@ -59,7 +59,7 @@ LOGSTRUCTLIST* g_log_struct_list = nullptr; //---------------------------------------------------------------------------------------------------------------------- CLIENT_BLOCK client_block; -SERVER_BLOCK server_block; +thread_local SERVER_BLOCK server_block; time_t tv_server_start = 0; time_t tv_server_end = 0; @@ -331,7 +331,7 @@ fetchMeta(XDR* client_input, DATA_SYSTEM* data_system, SYSTEM_CONFIG* system_con CLIENT_FLAGS* udaClientFlags() { - static CLIENT_FLAGS client_flags = {}; + static thread_local CLIENT_FLAGS client_flags = {}; return &client_flags; } @@ -376,7 +376,10 @@ int idamClient(REQUEST_BLOCK* request_block, int* indices) unsigned int* private_flags = udaPrivateFlags(); CLIENT_FLAGS* client_flags = udaClientFlags(); client_flags->alt_rank = 0; - client_flags->user_timeout = TIMEOUT; + // currently no way of knowing if this has been set by udaSetProperty except that it probably won't be 0 + if (client_flags->user_timeout == 0) { + client_flags->user_timeout = TIMEOUT; + } time_t protocol_time; // Time a Conversation Occured @@ -397,10 +400,6 @@ int idamClient(REQUEST_BLOCK* request_block, int* indices) //------------------------------------------------------------------------- // Initialise the Error Stack before Accessing Data - if (tv_server_start != 0) { - freeIdamErrorStack(&server_block.idamerrorstack); // Free Previous Stack Heap - } - initServerBlock(&server_block, 0); // Reset previous Error Messages from the Server & Free Heap initUdaErrorStack(); @@ -1113,6 +1112,11 @@ int idamClient(REQUEST_BLOCK* request_block, int* indices) // Normal Exit: Return to Client std::copy(data_block_indices.begin(), data_block_indices.end(), indices); + + // reset the timer if only idle time is considered for server timeout instead of total lifetime + if (client_flags->flags & CLIENTFLAG_IDLE_TIMEOUT) { + time(&tv_server_start); + } return 0; //------------------------------------------------------------------------------ @@ -1424,7 +1428,6 @@ void udaFree(int handle) } // closeIdamError(&server_block.idamerrorstack); - freeIdamErrorStack(&server_block.idamerrorstack); initDataBlock(data_block); data_block->handle = -1; // Flag this as ready for re-use } @@ -1441,14 +1444,6 @@ void udaFreeAll() uda::cache::free_cache(); #endif - for (int i = 0; i < udaGetCurrentDataBlockIndex(); ++i) { -#ifndef FATCLIENT - freeDataBlock(getIdamDataBlock(i)); -#else - freeDataBlock(getIdamDataBlock(i)); -#endif - } - acc_freeDataBlocks(); #ifndef FATCLIENT diff --git a/source/client/udaGetAPI.cpp b/source/client/udaGetAPI.cpp index fc43ed4ed..b10273d94 100755 --- a/source/client/udaGetAPI.cpp +++ b/source/client/udaGetAPI.cpp @@ -244,7 +244,7 @@ int idamGetAPIWithHost(const char* data_object, const char* data_source, const c muntrace(); #endif - freeClientRequestBlock(&request_block); + freeRequestBlock(&request_block); // Unlock the thread udaUnlockThread(); return handle; @@ -352,6 +352,7 @@ int idamGetBatchAPIWithHost(const char** signals, const char** sources, int coun muntrace(); #endif + freeRequestBlock(&request_block); // Unlock the thread udaUnlockThread(); return err; diff --git a/source/client2/accAPI.h b/source/client2/accAPI.h index 92256b85b..df7316851 100755 --- a/source/client2/accAPI.h +++ b/source/client2/accAPI.h @@ -3,7 +3,6 @@ #ifdef __cplusplus # include -# include #else # include # include diff --git a/source/client2/client.cpp b/source/client2/client.cpp index 88345f45a..d29a1948f 100644 --- a/source/client2/client.cpp +++ b/source/client2/client.cpp @@ -1022,8 +1022,6 @@ void uda::client::Client::concat_errors(UDA_ERROR_STACK* error_stack) unsigned int iold = error_stack->nerrors; unsigned int inew = error_stack_.size() + error_stack->nerrors; - error_stack->idamerror = (UDA_ERROR*)realloc((void*)error_stack->idamerror, (inew * sizeof(UDA_ERROR))); - for (unsigned int i = iold; i < inew; i++) { error_stack->idamerror[i] = error_stack_[i - iold]; } diff --git a/source/clientserver/errorLog.cpp b/source/clientserver/errorLog.cpp index 78c60f8fc..cee6f90eb 100755 --- a/source/clientserver/errorLog.cpp +++ b/source/clientserver/errorLog.cpp @@ -67,9 +67,9 @@ void initUdaErrorStack() udaerrorstack.clear(); } -void initErrorRecords(const UDA_ERROR_STACK* errorstack) +void initErrorRecords(UDA_ERROR_STACK* errorstack) { - for (unsigned int i = 0; i < errorstack->nerrors; i++) { + for (unsigned int i = 0; i < UDA_MAX_ERRORS; i++) { errorstack->idamerror[i].type = 0; errorstack->idamerror[i].code = 0; errorstack->idamerror[i].location[0] = '\0'; @@ -141,8 +141,9 @@ void concatUdaError(UDA_ERROR_STACK* errorstackout) unsigned int iold = errorstackout->nerrors; unsigned int inew = udaerrorstack.size() + errorstackout->nerrors; - - errorstackout->idamerror = (UDA_ERROR*)realloc((void*)errorstackout->idamerror, (inew * sizeof(UDA_ERROR))); + if (inew > UDA_MAX_ERRORS) { + inew = UDA_MAX_ERRORS; + } for (unsigned int i = iold; i < inew; i++) { errorstackout->idamerror[i] = udaerrorstack[i - iold]; @@ -154,10 +155,8 @@ void freeIdamErrorStack(UDA_ERROR_STACK* errorstack) { // "FIX" : this is causing segfaults when using multiple clients (eg. get and put) // apparently due to both trying to free the same memory. Needs fixing properly. - // free(errorstack->idamerror); errorstack->nerrors = 0; - errorstack->idamerror = nullptr; } // Free Stack Heap diff --git a/source/clientserver/errorLog.h b/source/clientserver/errorLog.h index 68e908817..ffda003d3 100755 --- a/source/clientserver/errorLog.h +++ b/source/clientserver/errorLog.h @@ -21,7 +21,7 @@ extern "C" { LIBRARY_API int udaNumErrors(void); LIBRARY_API void udaErrorLog(CLIENT_BLOCK client_block, REQUEST_BLOCK request_block, UDA_ERROR_STACK* error_stack); LIBRARY_API void initUdaErrorStack(void); -LIBRARY_API void initErrorRecords(const UDA_ERROR_STACK* errorstack); +LIBRARY_API void initErrorRecords(UDA_ERROR_STACK* errorstack); LIBRARY_API void printIdamErrorStack(void); LIBRARY_API void addIdamError(int type, const char* location, int code, const char* msg); LIBRARY_API void concatUdaError(UDA_ERROR_STACK* errorstackout); diff --git a/source/clientserver/initStructs.cpp b/source/clientserver/initStructs.cpp index e0a9c7b4d..678d303bb 100755 --- a/source/clientserver/initStructs.cpp +++ b/source/clientserver/initStructs.cpp @@ -96,7 +96,6 @@ void initServerBlock(SERVER_BLOCK* str, int version) str->msg[0] = '\0'; str->pid = (int)getpid(); str->idamerrorstack.nerrors = 0; - str->idamerrorstack.idamerror = nullptr; str->OSName[0] = '\0'; // Operating System Name str->DOI[0] = '\0'; // Digital Object Identifier (server configuration) diff --git a/source/clientserver/protocol.cpp b/source/clientserver/protocol.cpp index 3b6b4d91d..8c7994eee 100755 --- a/source/clientserver/protocol.cpp +++ b/source/clientserver/protocol.cpp @@ -76,8 +76,6 @@ int protocol_serv(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALL if (server_block->idamerrorstack.nerrors > 0) { // No Data to Receive? - server_block->idamerrorstack.idamerror = (UDA_ERROR*) malloc( - server_block->idamerrorstack.nerrors * sizeof(UDA_ERROR)); initErrorRecords(&server_block->idamerrorstack); if (!xdr_server2(xdrs, server_block)) { diff --git a/source/clientserver/protocol2.cpp b/source/clientserver/protocol2.cpp index 429fdbd48..ab3646f51 100755 --- a/source/clientserver/protocol2.cpp +++ b/source/clientserver/protocol2.cpp @@ -308,8 +308,6 @@ static int handle_server_block(XDR* xdrs, int direction, const void* str, int pr if (server_block->idamerrorstack.nerrors > 0) { // No Data to Receive? - server_block->idamerrorstack.idamerror = (UDA_ERROR*)malloc( - server_block->idamerrorstack.nerrors * sizeof(UDA_ERROR)); initErrorRecords(&server_block->idamerrorstack); if (!xdr_server2(xdrs, server_block)) { diff --git a/source/clientserver/udaDefines.h b/source/clientserver/udaDefines.h index c415229f8..b81d69de7 100755 --- a/source/clientserver/udaDefines.h +++ b/source/clientserver/udaDefines.h @@ -88,6 +88,8 @@ extern "C" { #define CLIENTFLAG_REUSELASTHANDLE 32u // 0010 0000 Reuse the last issued handle value (for this thread) - assume application has freed heap #define CLIENTFLAG_FREEREUSELASTHANDLE 64u // 0100 0000 Free the heap associated with the last issued handle and reuse the handle value #define CLIENTFLAG_FILECACHE 128u // 1000 0000 Access data from and save data to local cache files +#define CLIENTFLAG_DB_ONLY 256u // 0001 0000 0000 Read fails for signals not in DB. No file lookup fallback +#define CLIENTFLAG_IDLE_TIMEOUT 512u // 0010 0000 0000 Switch between timeout mechanisms. default to total lifetime, this switch enables idletime instead. //-------------------------------------------------------- // Error Models diff --git a/source/clientserver/udaStructs.cpp b/source/clientserver/udaStructs.cpp index 752c1ad01..2e27125bc 100644 --- a/source/clientserver/udaStructs.cpp +++ b/source/clientserver/udaStructs.cpp @@ -1,6 +1,7 @@ #include "udaStructs.h" #include +#include #include "udaTypes.h" @@ -12,28 +13,25 @@ void freePutDataBlockList(PUTDATA_BLOCK_LIST* putDataBlockList) // initPutDataBlockList(putDataBlockList); } -//void freeRequestData(REQUEST_DATA* request_data) -//{ -// freeNameValueList(&request_data->nameValueList); -// freePutDataBlockList(&request_data->putDataBlockList); -//} - void freeRequestBlock(REQUEST_BLOCK* request_block) { -// for (int i = 0; i < request_block->num_requests; ++i) { -// freeRequestData(&request_block->requests[0]); -// } -// free(request_block->requests); -// request_block->num_requests = 0; -// request_block->requests = nullptr; + if(request_block == nullptr) { + return; + } + if (request_block->requests != nullptr) { + for (int i = 0; i < request_block->num_requests; i++) { + freeNameValueList(&request_block->requests[i].nameValueList); + freePutDataBlockList(&request_block->requests[i].putDataBlockList); + } + free(request_block->requests); + request_block->requests = nullptr; + } + request_block->num_requests = 0; } void freeClientPutDataBlockList(PUTDATA_BLOCK_LIST* putDataBlockList) { - if (putDataBlockList->putDataBlock != nullptr && putDataBlockList->blockListSize > 0) { - free(putDataBlockList->putDataBlock); - } -// initPutDataBlockList(putDataBlockList); + freePutDataBlockList(putDataBlockList); } void freeDataBlock(DATA_BLOCK* data_block) @@ -210,3 +208,66 @@ void freeReducedDataBlock(DATA_BLOCK* data_block) #endif } +unsigned int countDataBlockListSize(const DATA_BLOCK_LIST* data_block_list, CLIENT_BLOCK* client_block) +{ + unsigned int total = 0; + for (int i = 0; i < data_block_list->count; ++i) { + total += countDataBlockSize(&data_block_list->data[i], client_block); + } + return total; +} + +unsigned int countDataBlockSize(const DATA_BLOCK* data_block, CLIENT_BLOCK* client_block) +{ + int factor; + DIMS dim; + unsigned int count = sizeof(DATA_BLOCK); + + count += (unsigned int)(getSizeOf((UDA_TYPE)data_block->data_type) * data_block->data_n); + + if (data_block->error_type != UDA_TYPE_UNKNOWN) { + count += (unsigned int)(getSizeOf((UDA_TYPE)data_block->error_type) * data_block->data_n); + } + if (data_block->errasymmetry) { + count += (unsigned int)(getSizeOf((UDA_TYPE)data_block->error_type) * data_block->data_n); + } + + if (data_block->rank > 0) { + for (unsigned int k = 0; k < data_block->rank; k++) { + count += sizeof(DIMS); + dim = data_block->dims[k]; + if (!dim.compressed) { + count += (unsigned int)(getSizeOf((UDA_TYPE)dim.data_type) * dim.dim_n); + factor = 1; + if (dim.errasymmetry) factor = 2; + if (dim.error_type != UDA_TYPE_UNKNOWN) { + count += (unsigned int)(factor * getSizeOf((UDA_TYPE)dim.error_type) * dim.dim_n); + } + } else {; + switch (dim.method) { + case 0: + count += +2 * sizeof(double); + break; + case 1: + for (unsigned int i = 0; i < dim.udoms; i++) { + count += (unsigned int)(*((long*)dim.sams + i) * getSizeOf((UDA_TYPE)dim.data_type)); + } + break; + case 2: + count += dim.udoms * getSizeOf((UDA_TYPE)dim.data_type); + break; + case 3: + count += dim.udoms * getSizeOf((UDA_TYPE)dim.data_type); + break; + } + } + } + } + + if (client_block->get_meta) { + count += sizeof(DATA_SYSTEM) + sizeof(SYSTEM_CONFIG) + sizeof(DATA_SOURCE) + sizeof(SIGNAL) + + sizeof(SIGNAL_DESC); + } + + return count; +} diff --git a/source/clientserver/udaStructs.h b/source/clientserver/udaStructs.h index 9c2c08e84..af61873e7 100755 --- a/source/clientserver/udaStructs.h +++ b/source/clientserver/udaStructs.h @@ -306,9 +306,11 @@ typedef struct UdaError { char msg[STRING_LENGTH]; // Message } UDA_ERROR; +#define UDA_MAX_ERRORS 30 + typedef struct UdaErrorStack { unsigned int nerrors; // Number of Errors - UDA_ERROR* idamerror; // Array of Errors + UDA_ERROR idamerror[UDA_MAX_ERRORS]; // Array of Errors } UDA_ERROR_STACK; typedef struct ServerBlock { @@ -440,6 +442,10 @@ void freeRequestBlock(REQUEST_BLOCK* request_block); void freePutDataBlockList(PUTDATA_BLOCK_LIST* putDataBlockList); +LIBRARY_API unsigned int countDataBlockListSize(const DATA_BLOCK_LIST* data_block_list, CLIENT_BLOCK* client_block); +LIBRARY_API unsigned int countDataBlockSize(const DATA_BLOCK* data_block, CLIENT_BLOCK* client_block); + + #ifdef __cplusplus } #endif diff --git a/source/common/uda_env_options.hpp b/source/common/uda_env_options.hpp new file mode 100644 index 000000000..9167aaed0 --- /dev/null +++ b/source/common/uda_env_options.hpp @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace uda::common::env_config { + +const std::vector truthy_values = {"1", "true", "yes", "on"}; +const std::vector falsey_values = {"0", "false", "no", "off"}; + +inline bool strings_match(std::string_view val, const std::vector& accepted_values) { + std::string value(val); + std::transform(value.begin(), value.end(), value.begin(), + [] (unsigned char c) {return std::tolower(c); }); + + // case insensitive comparison + return std::any_of(accepted_values.begin(), accepted_values.end(), + [&] (std::string v){ + std::transform(v.begin(), v.end(), v.begin(), + [] (unsigned char c) {return std::tolower(c); }); + return value == v; }); +} + +inline bool match_custom_values(std::string_view var_name, const std::vector& accepted_values, + bool default_value=false) { + const char* value = std::getenv(var_name.data()); + if (value == nullptr) { + return default_value; + } + return strings_match(value, accepted_values); +} + +inline std::optional +get_custom_param(std::string_view var_name, const std::vector& accepted_values) { + const char* value = std::getenv(var_name.data()); + if (value == nullptr) { + return {}; + } + if (strings_match(value, accepted_values)) { + return value; + } + return {}; +} + +inline bool evaluate_bool_param(std::string_view var_name, bool default_value=false) { + + const char* val = std::getenv(var_name.data()); + if (val == nullptr) { + return default_value; + } + if (strings_match(val, truthy_values)) { + return true; + } + if (strings_match(val, falsey_values)) { + return false; + } + return default_value; +} + +} // namespace diff --git a/source/common/uuid.hpp b/source/common/uuid.hpp new file mode 100644 index 000000000..d6d6684db --- /dev/null +++ b/source/common/uuid.hpp @@ -0,0 +1,12 @@ +#include +#include +#include +#include + +namespace uda::common::uuid { +inline std::string generate_random_uuid() +{ + boost::uuids::uuid uuid = boost::uuids::random_generator()(); + return boost::uuids::to_string(uuid); +} +} // namespace diff --git a/source/etc/machine.d/mast.l.cfg b/source/etc/machine.d/mast.l.cfg index 7905d3b38..acdae71f4 100644 --- a/source/etc/machine.d/mast.l.cfg +++ b/source/etc/machine.d/mast.l.cfg @@ -12,8 +12,6 @@ export UDA_PORT2=${UDA_PORT} DATA_ARCHIVE=/net/raidsrvr/data -export UDA_BYTES_PLUGIN_ALLOWED_PATHS="/net/raidsrvr/data;/common/uda-scratch;/projects/physics/omfit" - export MAST_DATA=${DATA_ARCHIVE}/MAST_Data export MAST_IMAGES=${DATA_ARCHIVE}/MAST_IMAGES export MAST_NEW=${DATA_ARCHIVE}/MAST_NEW @@ -29,3 +27,5 @@ export UDA_METADATA_PLUGIN=MASTU_DB export UDA_TESTDB_SQLDBNAME=uda # SQL Database Name export UDA_PLUGIN_DEBUG_SINGLEFILE=1 + +export UDA_ALLOWED_PATHS="/net/raidsrvr/data/;/common/uda-scratch/;/projects/physics/omfit/;/projects/codes/MAST-U/" \ No newline at end of file diff --git a/source/etc/uda@.service.in b/source/etc/uda@.service.in index f9492d78b..e886baf50 100644 --- a/source/etc/uda@.service.in +++ b/source/etc/uda@.service.in @@ -3,7 +3,7 @@ Description=@USER@-uda service [Service] ExecStart=@CMAKE_INSTALL_PREFIX@/etc/udaserver.sh -Type=forking +Type=exec User=@USER@ #Group=yes UMask=002 diff --git a/source/etc/udaserver.cfg.in b/source/etc/udaserver.cfg.in index 1c7bece14..be6edf637 100755 --- a/source/etc/udaserver.cfg.in +++ b/source/etc/udaserver.cfg.in @@ -40,3 +40,10 @@ export UDA_PLUGIN_DEBUG_SINGLEFILE=1 export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$UDA_ROOT/lib:$UDA_ROOT/lib/plugins" export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:@EXTRA_LD_LIBRARY_PATHS@" export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH + +#------------------------------------------------------------------------------------------------------ +# Permitted filepaths for the server to access formatted as a comma-delimited list +# by deafult all file access will be prohibited if this is set to empty or +# the environment variable is unset. Use semi-colons (;) to deliminate multiple allowed paths +# set the path to "/" to allow everything to be accessed +export UDA_ALLOWED_PATHS= \ No newline at end of file diff --git a/source/logging/accessLog.cpp b/source/logging/accessLog.cpp index 0cebf34c4..19bdd08e4 100755 --- a/source/logging/accessLog.cpp +++ b/source/logging/accessLog.cpp @@ -23,6 +23,7 @@ #include #include +#include #if defined(SERVERBUILD) || defined(FATCLIENT) # include # include @@ -31,70 +32,6 @@ #endif -unsigned int countDataBlockListSize(const DATA_BLOCK_LIST* data_block_list, CLIENT_BLOCK* client_block) -{ - unsigned int total = 0; - for (int i = 0; i < data_block_list->count; ++i) { - total += countDataBlockSize(&data_block_list->data[i], client_block); - } - return total; -} - -unsigned int countDataBlockSize(const DATA_BLOCK* data_block, CLIENT_BLOCK* client_block) -{ - int factor; - DIMS dim; - unsigned int count = sizeof(DATA_BLOCK); - - count += (unsigned int)(getSizeOf((UDA_TYPE)data_block->data_type) * data_block->data_n); - - if (data_block->error_type != UDA_TYPE_UNKNOWN) { - count += (unsigned int)(getSizeOf((UDA_TYPE)data_block->error_type) * data_block->data_n); - } - if (data_block->errasymmetry) { - count += (unsigned int)(getSizeOf((UDA_TYPE)data_block->error_type) * data_block->data_n); - } - - if (data_block->rank > 0) { - for (unsigned int k = 0; k < data_block->rank; k++) { - count += sizeof(DIMS); - dim = data_block->dims[k]; - if (!dim.compressed) { - count += (unsigned int)(getSizeOf((UDA_TYPE)dim.data_type) * dim.dim_n); - factor = 1; - if (dim.errasymmetry) factor = 2; - if (dim.error_type != UDA_TYPE_UNKNOWN) { - count += (unsigned int)(factor * getSizeOf((UDA_TYPE)dim.error_type) * dim.dim_n); - } - } else {; - switch (dim.method) { - case 0: - count += +2 * sizeof(double); - break; - case 1: - for (unsigned int i = 0; i < dim.udoms; i++) { - count += (unsigned int)(*((long*)dim.sams + i) * getSizeOf((UDA_TYPE)dim.data_type)); - } - break; - case 2: - count += dim.udoms * getSizeOf((UDA_TYPE)dim.data_type); - break; - case 3: - count += dim.udoms * getSizeOf((UDA_TYPE)dim.data_type); - break; - } - } - } - } - - if (client_block->get_meta) { - count += sizeof(DATA_SYSTEM) + sizeof(SYSTEM_CONFIG) + sizeof(DATA_SOURCE) + sizeof(SIGNAL) + - sizeof(SIGNAL_DESC); - } - - return count; -} - #if defined(SERVERBUILD) || defined(FATCLIENT) void udaAccessLog(int init, CLIENT_BLOCK client_block, REQUEST_BLOCK request_block, SERVER_BLOCK server_block, @@ -243,10 +180,6 @@ void udaAccessLog(int init, CLIENT_BLOCK client_block, REQUEST_BLOCK request_blo auto str = fmt.str(); udaLog(UDA_LOG_ACCESS, "%s\n", str.c_str()); - -// udaServerRedirectStdStreams(0); -// udaProvenancePlugin(&client_block, &request, nullptr, nullptr, pluginlist, str.c_str(), environment); -// udaServerRedirectStdStreams(1); } } diff --git a/source/logging/accessLog.h b/source/logging/accessLog.h index d84decd2d..85b296971 100755 --- a/source/logging/accessLog.h +++ b/source/logging/accessLog.h @@ -11,9 +11,6 @@ extern "C" { #endif -LIBRARY_API unsigned int countDataBlockListSize(const DATA_BLOCK_LIST* data_block_list, CLIENT_BLOCK* client_block); -LIBRARY_API unsigned int countDataBlockSize(const DATA_BLOCK* data_block, CLIENT_BLOCK* client_block); - LIBRARY_API void udaAccessLog(int init, CLIENT_BLOCK client_block, REQUEST_BLOCK request_block, SERVER_BLOCK server_block, unsigned int total_datablock_size); @@ -22,4 +19,4 @@ udaAccessLog(int init, CLIENT_BLOCK client_block, REQUEST_BLOCK request_block, S } #endif -#endif // UDA_LOGGING_ACCESSLOG_H \ No newline at end of file +#endif // UDA_LOGGING_ACCESSLOG_H diff --git a/source/plugins/CMakeLists.txt b/source/plugins/CMakeLists.txt index 52b03f920..05e3de388 100755 --- a/source/plugins/CMakeLists.txt +++ b/source/plugins/CMakeLists.txt @@ -3,6 +3,7 @@ find_package( LibXml2 REQUIRED ) find_package( fmt REQUIRED ) +find_package( Boost REQUIRED ) if( WIN32 OR MINGW ) find_package( XDR REQUIRED ) @@ -14,6 +15,13 @@ else() endif() endif() +set ( STDFS_VAR "" ) +if( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 9.1 ) + set( STDFS_VAR "stdc++fs" ) +elseif ( "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 9.0 ) + set( STDFS_VAR "c++fs" ) +endif() + configure_file( "${CMAKE_SOURCE_DIR}/source/etc/udaPlugins.conf.in" "${CMAKE_CURRENT_BINARY_DIR}/udaPlugins.conf" @@ -56,6 +64,7 @@ add_definitions( -DSERVERBUILD ) include_directories( ${CMAKE_SOURCE_DIR}/source ${LIBXML2_INCLUDE_DIR} + ${Boost_INCLUDE_DIRS} ) if( MINGW OR WIN32 ) @@ -67,6 +76,7 @@ endif() set( SOURCE_FILES managePluginFiles.cpp udaPlugin.cpp + utils.cpp ) set( HEADER_FILES @@ -74,7 +84,8 @@ set( HEADER_FILES pluginStructs.h udaPlugin.h udaPluginFiles.h - testplugin/teststructs.h testplugin/teststructs.cpp) + utils.h +) add_library( plugins-objects OBJECT ${SOURCE_FILES} ${HEADER_FILES} ) add_library( plugins-static STATIC $ ) @@ -102,6 +113,7 @@ if( BUILD_SHARED_LIBS ) target_link_libraries( plugins-shared PRIVATE server-static ${LINK_LIB} + ${STDFS_VAR} ) endif() diff --git a/source/plugins/bytes/CMakeLists.txt b/source/plugins/bytes/CMakeLists.txt index 23904720a..ac1e9de01 100644 --- a/source/plugins/bytes/CMakeLists.txt +++ b/source/plugins/bytes/CMakeLists.txt @@ -24,7 +24,6 @@ uda_plugin( EXAMPLE "BYTES::read()" LIBNAME bytes_plugin SOURCES bytesPlugin.cpp readBytesNonOptimally.cpp md5Sum.cpp - CONFIG_FILE bytesPlugin.cfg EXTRA_INCLUDE_DIRS ${LIBXML2_INCLUDE_DIR} EXTRA_LINK_LIBS diff --git a/source/plugins/bytes/bytesPlugin.cfg.in b/source/plugins/bytes/bytesPlugin.cfg.in deleted file mode 100755 index 3bc67c9e6..000000000 --- a/source/plugins/bytes/bytesPlugin.cfg.in +++ /dev/null @@ -1 +0,0 @@ -export UDA_BYTES_PLUGIN_ALLOWED_PATHS= \ No newline at end of file diff --git a/source/plugins/bytes/bytesPlugin.cpp b/source/plugins/bytes/bytesPlugin.cpp index da407fdd8..618be3d23 100644 --- a/source/plugins/bytes/bytesPlugin.cpp +++ b/source/plugins/bytes/bytesPlugin.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include "readBytesNonOptimally.h" @@ -27,6 +28,19 @@ namespace filesystem = std::filesystem; #define BYTEFILEOPENERROR 100004 #define BYTEFILEHEAPERROR 100005 +namespace { +class FileDeleter +{ +public: + void operator()(FILE* file) const + { + if (file) { + fclose(file); + } + } +}; +} // anon namespace + class BytesPlugin { public: @@ -60,7 +74,7 @@ class BytesPlugin int size(IDAM_PLUGIN_INTERFACE* plugin_interface); private: - using file_ptr = std::unique_ptr; + using file_ptr = std::unique_ptr; bool init_ = false; std::unordered_map file_map_ = {}; @@ -171,36 +185,6 @@ int BytesPlugin::max_interface_version(IDAM_PLUGIN_INTERFACE* plugin_interface) //---------------------------------------------------------------------------------------- // Add functionality here .... -// Check if path starts with pre-approved file path -// Raises Plugin Error if not -int check_allowed_path(const char* expandedPath) { - std::string full_path; - try { - full_path = filesystem::canonical(expandedPath).string(); - } catch (filesystem::filesystem_error& e) { - UDA_LOG(UDA_LOG_DEBUG, "Filepath [%s] not found! Error: %s\n", full_path.c_str(), e.what()); - RAISE_PLUGIN_ERROR("Provided File Path Not Found!\n"); - } - const char* env_str = std::getenv("UDA_BYTES_PLUGIN_ALLOWED_PATHS"); - std::vector allowed_paths; - if (env_str) { - // gotta check if environment variable exists before using it - boost::split(allowed_paths, env_str, boost::is_any_of(";")); - } - bool good_path = false; - for (const auto& allowed_path : allowed_paths) { - if (full_path.rfind(allowed_path, 0) != std::string::npos) { - good_path = true; - break; - } - } - if (!good_path) { - UDA_LOG(UDA_LOG_DEBUG, "Bad Path Provided %s\n", expandedPath); - RAISE_PLUGIN_ERROR("Bad File Path Provided\n"); - } - return 0; -} - int check_path(const Environment* environment, const std::string& path) { int err = 0; @@ -274,7 +258,7 @@ int BytesPlugin::read(IDAM_PLUGIN_INTERFACE* plugin_interface) FILE* file = nullptr; if (file_map_.count(tmp_path) == 0) { - file_ptr ptr = {fopen(tmp_path, "rb"), fclose}; + file_ptr ptr(fopen(tmp_path, "rb")); file = ptr.get(); file_map_.emplace(tmp_path, std::move(ptr)); } else { diff --git a/source/plugins/keyvalue/keyvaluePlugin.cpp b/source/plugins/keyvalue/keyvaluePlugin.cpp index 31ee0831c..68669a566 100755 --- a/source/plugins/keyvalue/keyvaluePlugin.cpp +++ b/source/plugins/keyvalue/keyvaluePlugin.cpp @@ -171,7 +171,7 @@ int uda::keyvalue::Plugin::help(IDAM_PLUGIN_INTERFACE* plugin_interface) int uda::keyvalue::Plugin::version(IDAM_PLUGIN_INTERFACE* plugin_interface) { - setReturnDataString(plugin_interface->data_block, UDA_BUILD_VERSION, "Plugin version number"); + return setReturnDataString(plugin_interface->data_block, UDA_BUILD_VERSION, "Plugin version number"); } // Plugin Build Date diff --git a/source/plugins/managePluginFiles.cpp b/source/plugins/managePluginFiles.cpp index 8c6101e81..a1870c2e4 100755 --- a/source/plugins/managePluginFiles.cpp +++ b/source/plugins/managePluginFiles.cpp @@ -13,7 +13,7 @@ #include #define UDA_PLUGIN_FILE_ALLOC 10 -#define MAX_OPEN_PLUGIN_FILE_DESC 50 +#define MAX_OPEN_PLUGIN_FILE_DESC 5 // Initialise the File List and allocate heap for the list diff --git a/source/plugins/testplugin/testplugin.cpp b/source/plugins/testplugin/testplugin.cpp index eb158a289..05089de3c 100755 --- a/source/plugins/testplugin/testplugin.cpp +++ b/source/plugins/testplugin/testplugin.cpp @@ -142,6 +142,8 @@ static int do_emptytest(IDAM_PLUGIN_INTERFACE* plugin_interface); #ifdef CAPNP_ENABLED static int do_capnp_test(IDAM_PLUGIN_INTERFACE* plugin_interface); +static int do_capnp_complex_test(IDAM_PLUGIN_INTERFACE* plugin_interface); + static int do_nested_capnp_test(IDAM_PLUGIN_INTERFACE* plugin_interface); static int do_long_capnp_test(IDAM_PLUGIN_INTERFACE* plugin_interface); @@ -348,6 +350,8 @@ extern int testplugin(IDAM_PLUGIN_INTERFACE* plugin_interface) #ifdef CAPNP_ENABLED } else if (STR_IEQUALS(request->function, "capnp")) { err = do_capnp_test(plugin_interface); + } else if (STR_IEQUALS(request->function, "capnp_complex")) { + err = do_capnp_complex_test(plugin_interface); } else if (STR_IEQUALS(request->function, "capnp_nested")) { err = do_nested_capnp_test(plugin_interface); } else if (STR_IEQUALS(request->function, "capnp_long")) { @@ -3983,6 +3987,49 @@ int do_capnp_test(IDAM_PLUGIN_INTERFACE* plugin_interface) return 0; } +int do_capnp_complex_test(IDAM_PLUGIN_INTERFACE* plugin_interface) +{ + auto tree = uda_capnp_new_tree(); + auto root = uda_capnp_get_root(tree); + uda_capnp_set_node_name(root, "root"); + uda_capnp_add_children(root, 3); + + auto child = uda_capnp_get_child(tree, root, 0); + uda_capnp_set_node_name(child, "complex_array"); + + std::vector complex_vec(30); + for (float i = 0; i < 30; ++i) { + complex_vec[i] = COMPLEX{i / 10.0f, i - 10.0f}; + } + uda_capnp_add_array_complex(child, complex_vec.data(), complex_vec.size()); + + child = uda_capnp_get_child(tree, root, 1); + uda_capnp_set_node_name(child, "dcomplex_array"); + + std::vector dcomplex_vec(100); + for (double i = 0; i < 100; ++i) { + dcomplex_vec[i] = DCOMPLEX{ i/3, i/5}; + } + uda_capnp_add_array_dcomplex(child, dcomplex_vec.data(), dcomplex_vec.size()); + + child = uda_capnp_get_child(tree, root, 2); + uda_capnp_set_node_name(child, "complex_scalar"); + + uda_capnp_add_complex(child, COMPLEX{5,3}); + + auto buffer = uda_capnp_serialise(tree); + + DATA_BLOCK* data_block = plugin_interface->data_block; + initDataBlock(data_block); + + data_block->data_n = static_cast(buffer.size); + data_block->data = buffer.data; + data_block->dims = nullptr; + data_block->data_type = UDA_TYPE_CAPNP; + + return 0; +} + int do_nested_capnp_test(IDAM_PLUGIN_INTERFACE* plugin_interface) { auto tree = uda_capnp_new_tree(); diff --git a/source/plugins/udaPlugin.cpp b/source/plugins/udaPlugin.cpp index b07df3ad4..79f70bf4d 100644 --- a/source/plugins/udaPlugin.cpp +++ b/source/plugins/udaPlugin.cpp @@ -40,7 +40,6 @@ IDAM_PLUGIN_INTERFACE* udaCreatePluginInterface(const char* request) plugin_interface->housekeeping = 0; plugin_interface->changePlugin = 0; plugin_interface->error_stack.nerrors = 0; - plugin_interface->error_stack.idamerror = nullptr; return plugin_interface; } diff --git a/source/plugins/utils.cpp b/source/plugins/utils.cpp new file mode 100644 index 000000000..c5af9ff74 --- /dev/null +++ b/source/plugins/utils.cpp @@ -0,0 +1,70 @@ +#include "utils.h" + +#include +#include +#include + +#include "udaPlugin.h" + +#if defined __has_include +# if !__has_include() +# include +namespace filesystem = std::experimental::filesystem; +# else +# include +namespace filesystem = std::filesystem; +# endif +#else +# include +namespace filesystem = std::filesystem; +#endif + +#include +#include + +namespace { + +bool starts_with(const std::string& string, const std::string& search_string) { + if (string.size() < search_string.size()) { + return false; + } + const auto compare_string = string.substr(0, search_string.size()); + return compare_string == search_string; +} + +} + +// Check if path starts with pre-approved file path +// Raises Plugin Error if not +int check_allowed_path(const char* expanded_path) { + std::string full_path; + try { + full_path = filesystem::canonical(expanded_path).string(); + } catch (filesystem::filesystem_error& e) { + UDA_LOG(UDA_LOG_DEBUG, "Filepath [%s] not found! Error: %s\n", full_path.c_str(), e.what()); + RAISE_PLUGIN_ERROR("Provided File Path Not Found!"); + } + const char* env_str = std::getenv("UDA_ALLOWED_PATHS"); + + std::vector allowed_paths; + if (env_str != nullptr && env_str[0] != '\0') { + // Checking if environment variable exists before using it + boost::split(allowed_paths, env_str, boost::is_any_of(";")); + } else { + UDA_LOG(UDA_LOG_WARN, "UDA_ALLOWED_PATHS not found or empty, rejecting all paths\n"); + } + + bool good_path = false; + for (const auto& allowed_path: allowed_paths) { + if (starts_with(full_path, allowed_path)) { + good_path = true; + break; + } + } + if (!good_path) { + UDA_LOG(UDA_LOG_ERROR, "Bad Path Provided %s\n", expanded_path); + std::string error_msg("Bad Path Provided " + std::string(expanded_path)); + RAISE_PLUGIN_ERROR(error_msg.c_str()); + } + return 0; +} \ No newline at end of file diff --git a/source/plugins/utils.h b/source/plugins/utils.h new file mode 100644 index 000000000..0ebbb3f5e --- /dev/null +++ b/source/plugins/utils.h @@ -0,0 +1,16 @@ +#ifndef UDA_PLUGINS_UTILS_H +#define UDA_PLUGINS_UTILS_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +LIBRARY_API int check_allowed_path(const char* expanded_path); + +#ifdef __cplusplus +} +#endif + +#endif // UDA_PLUGINS_UTILS_H diff --git a/source/serialisation/capnp_serialisation.cpp b/source/serialisation/capnp_serialisation.cpp index 41da25aa4..5ed3c6554 100644 --- a/source/serialisation/capnp_serialisation.cpp +++ b/source/serialisation/capnp_serialisation.cpp @@ -10,13 +10,36 @@ #include #include #include +#include +#include #include #include "schema.capnp.h" +//NOTE: redefinition of UDA complex types here to avoid dragging in unnecessary dependencies. +// To be removed when headers are cleaned up in uda v3.0 +typedef struct DComplex { + double real; + double imaginary; +} DCOMPLEX; + +typedef struct Complex { + float real; + float imaginary; +} COMPLEX; + namespace { +template +struct is_uda_complex : std::false_type {}; + +template<> +struct is_uda_complex : std::true_type {}; + +template<> +struct is_uda_complex : std::true_type {}; + class PackedMessageStreamReader : public capnp::MessageReader { public: PackedMessageStreamReader(const char *bytes, size_t size, capnp::ReaderOptions options) @@ -57,6 +80,8 @@ const char* to_string(::TreeNode::Type type) case ::TreeNode::Type::FLT64: return "flt64"; case ::TreeNode::Type::STRING: return "string"; case ::TreeNode::Type::VOID: return "void"; + case ::TreeNode::Type::COMPLEX: return "complex"; + case ::TreeNode::Type::DCOMPLEX: return "dcomplex"; } return ""; } @@ -77,6 +102,21 @@ std::ostream& operator<<(std::ostream& out, const typename capnp::List::value, bool> = false> +std::ostream& operator<<(std::ostream& out, const T& complex) +{ + std::ios::fmtflags old_flags = out.flags(); + std::streamsize prec = out.precision(); + out << std::fixed << std::setprecision(3); + + char sign = (complex.imaginary >= 0) ? '+' : '-'; + out << complex.real << " " << sign << " " << std::abs(complex.imaginary) << "i"; + + out.flags(old_flags); + out.precision(prec); + return out; +} + template std::ostream& operator<<(std::ostream& out, gsl::span span) { @@ -161,6 +201,8 @@ void print_node(std::ostream& out, const ::TreeNode::Reader& tree, const std::st case ::TreeNode::Type::UINT32: print_data(out, array, indent); break; case ::TreeNode::Type::UINT64: print_data(out, array, indent); break; case ::TreeNode::Type::STRING: print_data(out, array, indent); break; + case ::TreeNode::Type::COMPLEX: print_data(out, array, indent); break; + case ::TreeNode::Type::DCOMPLEX: print_data(out, array, indent); break; case ::TreeNode::Type::VOID: out << indent << " data: \n"; break; } @@ -296,6 +338,8 @@ template <> TreeNode::Type TreeNodeTypeConverter::type = TreeNode::Typ template <> TreeNode::Type TreeNodeTypeConverter::type = TreeNode::Type::UINT64; template <> TreeNode::Type TreeNodeTypeConverter::type = TreeNode::Type::FLT32; template <> TreeNode::Type TreeNodeTypeConverter::type = TreeNode::Type::FLT64; +template <> TreeNode::Type TreeNodeTypeConverter::type = TreeNode::Type::COMPLEX; +template <> TreeNode::Type TreeNodeTypeConverter::type = TreeNode::Type::DCOMPLEX; // maximum size of data to send in each capnp "data" blob - capnp has a implicit limit of 512MB of data // for blobs so this size should not exceed that. @@ -342,6 +386,16 @@ void uda_capnp_add_md_array(NodeBuilder* node, const T* data_ptr, size_t* shape_ data.setEos(true); } +void uda_capnp_add_md_array_complex(NodeBuilder* node, const COMPLEX* data, size_t* shape_array, size_t rank) +{ + return uda_capnp_add_md_array(node, data, shape_array, rank); +} + +void uda_capnp_add_md_array_dcomplex(NodeBuilder* node, const DCOMPLEX* data, size_t* shape_array, size_t rank) +{ + return uda_capnp_add_md_array(node, data, shape_array, rank); +} + void uda_capnp_add_md_array_f32(NodeBuilder* node, const float* data, size_t* shape_array, size_t rank) { return uda_capnp_add_md_array(node, data, shape_array, rank); @@ -435,6 +489,16 @@ void uda_capnp_add_array(NodeBuilder* node, const T* data_ptr, size_t size) data.setEos(true); } +void uda_capnp_add_array_complex(NodeBuilder* node, const COMPLEX* data, size_t size) +{ + return uda_capnp_add_array(node, data, size); +} + +void uda_capnp_add_array_dcomplex(NodeBuilder* node, const DCOMPLEX* data, size_t size) +{ + return uda_capnp_add_array(node, data, size); +} + void uda_capnp_add_array_f32(NodeBuilder* node, const float* data, size_t size) { return uda_capnp_add_array(node, data, size); @@ -505,6 +569,16 @@ void uda_capnp_add_scalar(NodeBuilder* node, T scalar) data.setEos(true); } +void uda_capnp_add_complex(NodeBuilder* node, COMPLEX data) +{ + uda_capnp_add_scalar(node, data); +} + +void uda_capnp_add_dcomplex(NodeBuilder* node, DCOMPLEX data) +{ + uda_capnp_add_scalar(node, data); +} + void uda_capnp_add_f32(NodeBuilder* node, float data) { uda_capnp_add_scalar(node, data); @@ -629,6 +703,8 @@ int uda_capnp_read_type(NodeReader* node) case TreeNode::Type::FLT32: return UDA_TYPE_FLOAT; case TreeNode::Type::FLT64: return UDA_TYPE_DOUBLE; case TreeNode::Type::STRING: return UDA_TYPE_STRING; + case TreeNode::Type::COMPLEX: return UDA_TYPE_COMPLEX; + case TreeNode::Type::DCOMPLEX: return UDA_TYPE_DCOMPLEX; default: return UDA_TYPE_UNKNOWN; } } diff --git a/source/serialisation/capnp_serialisation.h b/source/serialisation/capnp_serialisation.h index 5f784e771..2c97e215c 100644 --- a/source/serialisation/capnp_serialisation.h +++ b/source/serialisation/capnp_serialisation.h @@ -10,6 +10,9 @@ extern "C" { #endif +typedef struct Complex COMPLEX; +typedef struct DComplex DCOMPLEX; + typedef struct Buffer { char* data; size_t size; @@ -52,6 +55,8 @@ void uda_capnp_set_node_name(NodeBuilder* node, const char* name); void uda_capnp_add_children(NodeBuilder* node, size_t num_children); NodeBuilder* uda_capnp_get_child(TreeBuilder* tree, NodeBuilder* node, size_t index); +void uda_capnp_add_array_complex(NodeBuilder* node, const COMPLEX* data, size_t size); +void uda_capnp_add_array_dcomplex(NodeBuilder* node, const DCOMPLEX* data, size_t size); void uda_capnp_add_array_f32(NodeBuilder* node, const float* data, size_t size); void uda_capnp_add_array_f64(NodeBuilder* node, const double* data, size_t size); void uda_capnp_add_array_i8(NodeBuilder* node, const int8_t* data, size_t size); @@ -64,6 +69,8 @@ void uda_capnp_add_array_u32(NodeBuilder* node, const uint32_t* data, size_t siz void uda_capnp_add_array_u64(NodeBuilder* node, const uint64_t* data, size_t size); void uda_capnp_add_array_char(NodeBuilder* node, const char* data, size_t size); +void uda_capnp_add_md_array_complex(NodeBuilder* node, const COMPLEX* data, size_t* shape_array, size_t rank); +void uda_capnp_add_md_array_dcomplex(NodeBuilder* node, const DCOMPLEX* data, size_t* shape_array, size_t rank); void uda_capnp_add_md_array_f32(NodeBuilder* node, const float* data, size_t* shape_array, size_t rank); void uda_capnp_add_md_array_f64(NodeBuilder* node, const double* data, size_t* shape_array, size_t rank); void uda_capnp_add_md_array_i8(NodeBuilder* node, const int8_t* data, size_t* shape_array, size_t rank); @@ -76,6 +83,8 @@ void uda_capnp_add_md_array_u32(NodeBuilder* node, const uint32_t* data, size_t* void uda_capnp_add_md_array_u64(NodeBuilder* node, const uint64_t* data, size_t* shape_array, size_t rank); void uda_capnp_add_md_array_char(NodeBuilder* node, const char* data, size_t* shape_array, size_t rank); +void uda_capnp_add_complex(NodeBuilder* node, COMPLEX data); +void uda_capnp_add_dcomplex(NodeBuilder* node, DCOMPLEX data); void uda_capnp_add_f32(NodeBuilder* node, float data); void uda_capnp_add_f64(NodeBuilder* node, double data); void uda_capnp_add_i8(NodeBuilder* node, int8_t data); @@ -95,4 +104,4 @@ void uda_capnp_print_tree_reader(TreeReader* tree); } #endif -#endif // UDA_CLIENTSERVER_CAPNP_SERIALISATION_H \ No newline at end of file +#endif // UDA_CLIENTSERVER_CAPNP_SERIALISATION_H diff --git a/source/serialisation/schema.capnp b/source/serialisation/schema.capnp index cd95fe5f7..ca682483a 100644 --- a/source/serialisation/schema.capnp +++ b/source/serialisation/schema.capnp @@ -29,5 +29,7 @@ struct TreeNode { flt64 @9; string @10; void @11; + complex @12; + dcomplex @13; } } diff --git a/source/server/CMakeLists.txt b/source/server/CMakeLists.txt index 0a17a0c02..0980a087f 100755 --- a/source/server/CMakeLists.txt +++ b/source/server/CMakeLists.txt @@ -4,6 +4,7 @@ find_package( LibXml2 REQUIRED ) find_package( OpenSSL REQUIRED ) find_package( fmt REQUIRED ) +find_package( Boost REQUIRED ) if( WIN32 OR MINGW ) find_package( XDR REQUIRED ) @@ -61,6 +62,7 @@ include_directories( ${LIBXML2_INCLUDE_DIR} ${OPENSSL_INCLUDE_DIR} ${fmt_SOURCE_DIR}/include + ${Boost_INCLUDE_DIRS} ) add_definitions( -DSERVERBUILD ) @@ -76,7 +78,7 @@ add_library( server-objects OBJECT ${SRC_FILES} ${HEADER_FILES} ) -target_link_libraries( server-objects PRIVATE fmt::fmt ) +target_link_libraries( server-objects PRIVATE fmt::fmt Boost::boost ) add_library( fatserver-objects OBJECT fatServer.cpp diff --git a/source/server/serverGetData.cpp b/source/server/serverGetData.cpp index 4f1b9c19a..70ce02280 100755 --- a/source/server/serverGetData.cpp +++ b/source/server/serverGetData.cpp @@ -1089,7 +1089,6 @@ int read_data(REQUEST_DATA* request, CLIENT_BLOCK client_block, idam_plugin_interface.userdefinedtypelist = userdefinedtypelist; idam_plugin_interface.logmalloclist = logmalloclist; idam_plugin_interface.error_stack.nerrors = 0; - idam_plugin_interface.error_stack.idamerror = nullptr; int plugin_id; @@ -1125,7 +1124,7 @@ int read_data(REQUEST_DATA* request, CLIENT_BLOCK client_block, #ifndef FATCLIENT // Redirect Output to temporary file if no file handles passed - int reset = 0; + bool reset = false; int rc; if ((rc = udaServerRedirectStdStreams(reset)) != 0) { UDA_THROW_ERROR(rc, "Error Redirecting Plugin Message Output"); @@ -1142,7 +1141,7 @@ int read_data(REQUEST_DATA* request, CLIENT_BLOCK client_block, #ifndef FATCLIENT // Reset Redirected Output - reset = 1; + reset = true; if ((rc = udaServerRedirectStdStreams(reset)) != 0) { UDA_THROW_ERROR(rc, "Error Resetting Redirected Plugin Message Output"); } @@ -1154,13 +1153,6 @@ int read_data(REQUEST_DATA* request, CLIENT_BLOCK client_block, UDA_LOG(UDA_LOG_DEBUG, "returned from plugin called\n"); - // Save Provenance with socket stream protection - - udaServerRedirectStdStreams(0); - udaProvenancePlugin(&client_block, request, data_source, signal_desc, pluginlist, nullptr, - getServerEnvironment()); - udaServerRedirectStdStreams(1); - // If no structures to pass back (only regular data) then free the user defined type list if (data_block->opaque_block == nullptr) { @@ -1243,13 +1235,5 @@ int read_data(REQUEST_DATA* request, CLIENT_BLOCK client_block, data_block->client_block = client_block; - //---------------------------------------------------------------------------- - // Save Provenance with socket stream protection - - udaServerRedirectStdStreams(0); - udaProvenancePlugin(&client_block, request, data_source, signal_desc, pluginlist, nullptr, - getServerEnvironment()); - udaServerRedirectStdStreams(1); - return 0; } diff --git a/source/server/serverPlugin.cpp b/source/server/serverPlugin.cpp index baabda17f..2f06cb56b 100755 --- a/source/server/serverPlugin.cpp +++ b/source/server/serverPlugin.cpp @@ -117,125 +117,107 @@ void printPluginList(FILE* fd, const PLUGINLIST* plugin_list) } } -int udaServerRedirectStdStreams(int reset) +int udaServerRedirectStdStreams(bool reset, bool cleanup) { // Any OS messages will corrupt xdr streams so re-divert IO from plugin libraries to a temporary file // Multi platform compliance - //static FILE* originalStdFH = nullptr; - //static FILE* originalErrFH = nullptr; - static int originalStdFH = 0; - static int originalErrFH = 0; - static FILE* mdsmsgFH = nullptr; + static int original_std_fd = 0; + static int original_err_fd = 0; + static FILE* plugin_redirect_fh = nullptr; + static bool is_specified_redirect_file = false; static char mksdir_template[MAXPATH] = { 0 }; - static char tempFile[MAXPATH] = { 0 }; + static char temp_file[MAXPATH] = { 0 }; - static bool singleFile = false; + if (cleanup) { + UDA_LOG(UDA_LOG_DEBUG, "Closing redirect file\n"); - if (!reset) { - if (!singleFile) { - const char* env = getenv("UDA_PLUGIN_DEBUG_SINGLEFILE"); // Use a single file for all plugin data requests - if (env != nullptr) { - singleFile = true; // Define UDA_PLUGIN_DEBUG to retain the file + errno = 0; + int rc = fclose(plugin_redirect_fh); + if (rc) { + int err = errno; + UDA_THROW_ERROR(err, strerror(err)); + } + + plugin_redirect_fh = nullptr; + + if (!is_specified_redirect_file) { + UDA_LOG(UDA_LOG_DEBUG, "Removing temporary file\n"); + + errno = 0; + rc = remove(temp_file); // Delete the temporary file + if (rc) { + int err = errno; + UDA_THROW_ERROR(err, strerror(err)); } } - if (mdsmsgFH != nullptr && singleFile) { + mksdir_template[0] = '\0'; + temp_file[0] = '\0'; + + return 0; + } + + if (!reset) { + if (plugin_redirect_fh != nullptr) { // Multi platform compliance - //stdout = mdsmsgFH; // Redirect all IO to a temporary file - //stderr = mdsmsgFH; - dup2(fileno(mdsmsgFH), fileno(stdout)); - dup2(fileno(mdsmsgFH), fileno(stderr)); + //stdout = plugin_redirect_fh; // Redirect all IO to a temporary file + //stderr = plugin_redirect_fh; + dup2(fileno(plugin_redirect_fh), fileno(stdout)); + dup2(fileno(plugin_redirect_fh), fileno(stderr)); return 0; } // Multi platform compliance - //originalStdFH = stdout; // Retain current values - //originalErrFH = stderr; - originalStdFH = dup(fileno(stdout)); - originalErrFH = dup(fileno(stderr)); - mdsmsgFH = nullptr; + //original_std_fd = stdout; // Retain current values + //original_err_fd = stderr; + original_std_fd = dup(fileno(stdout)); + original_err_fd = dup(fileno(stderr)); UDA_LOG(UDA_LOG_DEBUG, "Redirect standard output to temporary file\n"); - if (mksdir_template[0] == '\0') { - const char* env = getenv("UDA_PLUGIN_REDIVERT"); + const char* redirect_file = getenv("UDA_PLUGIN_REDIRECT_FILE"); + errno = 0; - if (env == nullptr) { - if ((env = getenv("UDA_WORK_DIR")) != nullptr) { - snprintf(mksdir_template, MAXPATH, "%s/idamPLUGINXXXXXX", env); + if (redirect_file == nullptr) { + if (mksdir_template[0] == '\0') { + is_specified_redirect_file = false; + const char* redirect_dir = getenv("UDA_PLUGIN_REDIRECT_DIR"); + if (redirect_dir != nullptr) { + snprintf(mksdir_template, MAXPATH, "%s/idamPLUGINXXXXXX", redirect_dir); } else { strcpy(mksdir_template, "/tmp/idamPLUGINXXXXXX"); } - } else { - strcpy(mksdir_template, env); } - } - - strcpy(tempFile, mksdir_template); - - // Open the message Trap - - errno = 0; - int fd = mkstemp(tempFile); - if (fd < 0 || errno != 0) { - int err = (errno != 0) ? errno : 994; - UDA_THROW_ERROR(err, "Unable to Obtain a Temporary File Name"); + strcpy(temp_file, mksdir_template); + int fd = mkstemp(temp_file); + if (fd < 0 || errno != 0) { + int err = (errno != 0) ? errno : 994; + UDA_THROW_ERROR(err, "Unable to Obtain a Temporary File Name"); + } + plugin_redirect_fh = fdopen(fd, "a"); + } else { + strcpy(temp_file, redirect_file); + plugin_redirect_fh = fopen(temp_file, "w"); } - mdsmsgFH = fdopen(fd, "a"); - - if (mdsmsgFH == nullptr || errno != 0) { + // Open the message Trap + if (plugin_redirect_fh == nullptr || errno != 0) { UDA_THROW_ERROR(999, "Unable to Trap Plugin Error Messages."); } // Multi platform compliance - //stdout = mdsmsgFH; // Redirect to a temporary file - //stderr = mdsmsgFH; - dup2(fileno(mdsmsgFH), fileno(stdout)); - dup2(fileno(mdsmsgFH), fileno(stderr)); + dup2(fileno(plugin_redirect_fh), fileno(stdout)); + dup2(fileno(plugin_redirect_fh), fileno(stderr)); } else { - if (mdsmsgFH != nullptr) { - UDA_LOG(UDA_LOG_DEBUG, "Resetting original file handles and removing temporary file\n"); - - if (!singleFile) { - if (mdsmsgFH != nullptr) { - errno = 0; - int rc = fclose(mdsmsgFH); - if (rc) { - int err = errno; - UDA_THROW_ERROR(err, strerror(err)); - } - } - mdsmsgFH = nullptr; - if (getenv("UDA_PLUGIN_DEBUG") == nullptr) { - errno = 0; - int rc = remove(tempFile); // Delete the temporary file - if (rc) { - int err = errno; - UDA_THROW_ERROR(err, strerror(err)); - } - tempFile[0] = '\0'; - } - } - - // Multi platform compliance - //stdout = originalStdFH; - //stderr = originalErrFH; - dup2(originalStdFH, fileno(stdout)); - dup2(originalErrFH, fileno(stderr)); - - } else { - + if (plugin_redirect_fh != nullptr) { UDA_LOG(UDA_LOG_DEBUG, "Resetting original file handles\n"); // Multi platform compliance - //stdout = originalStdFH; - //stderr = originalErrFH; - dup2(originalStdFH, fileno(stdout)); - dup2(originalErrFH, fileno(stderr)); + dup2(original_std_fd, fileno(stdout)); + dup2(original_err_fd, fileno(stderr)); } } @@ -410,7 +392,7 @@ int udaProvenancePlugin(CLIENT_BLOCK* client_block, REQUEST_DATA* original_reque makeRequestData(&request, *plugin_list, environment); - int err, rc, reset; + int err, rc; DATA_BLOCK data_block; IDAM_PLUGIN_INTERFACE idam_plugin_interface; @@ -446,11 +428,10 @@ int udaProvenancePlugin(CLIENT_BLOCK* client_block, REQUEST_DATA* original_reque idam_plugin_interface.userdefinedtypelist = &userdefinedtypelist; idam_plugin_interface.logmalloclist = &logmalloclist; idam_plugin_interface.error_stack.nerrors = 0; - idam_plugin_interface.error_stack.idamerror = nullptr; // Redirect Output to temporary file if no file handles passed - reset = 0; + bool reset = false; if ((err = udaServerRedirectStdStreams(reset)) != 0) { UDA_THROW_ERROR(err, "Error Redirecting Plugin Message Output"); } @@ -481,7 +462,7 @@ int udaProvenancePlugin(CLIENT_BLOCK* client_block, REQUEST_DATA* original_reque // Reset Redirected Output - reset = 1; + reset = true; if ((rc = udaServerRedirectStdStreams(reset)) != 0 || err != 0) { if (rc != 0) { addIdamError(UDA_CODE_ERROR_TYPE, __func__, rc, "Error Resetting Redirected Plugin Message Output"); @@ -560,7 +541,7 @@ int udaServerMetaDataPlugin(const PLUGINLIST* plugin_list, int plugin_id, REQUES SIGNAL_DESC* signal_desc, SIGNAL* signal_rec, DATA_SOURCE* data_source, const ENVIRONMENT* environment) { - int err, reset, rc; + int err, rc; IDAM_PLUGIN_INTERFACE idam_plugin_interface; // Check the Interface Compliance @@ -593,11 +574,10 @@ int udaServerMetaDataPlugin(const PLUGINLIST* plugin_list, int plugin_id, REQUES idam_plugin_interface.userdefinedtypelist = &userdefinedtypelist; idam_plugin_interface.logmalloclist = &logmalloclist; idam_plugin_interface.error_stack.nerrors = 0; - idam_plugin_interface.error_stack.idamerror = nullptr; // Redirect Output to temporary file if no file handles passed - reset = 0; + bool reset = false; if ((err = udaServerRedirectStdStreams(reset)) != 0) { UDA_THROW_ERROR(err, "Error Redirecting Plugin Message Output"); } @@ -608,7 +588,7 @@ int udaServerMetaDataPlugin(const PLUGINLIST* plugin_list, int plugin_id, REQUES // Reset Redirected Output - reset = 1; + reset = true; if ((rc = udaServerRedirectStdStreams(reset)) != 0 || err != 0) { if (rc != 0) { addIdamError(UDA_CODE_ERROR_TYPE, __func__, rc, "Error Resetting Redirected Plugin Message Output"); diff --git a/source/server/serverPlugin.h b/source/server/serverPlugin.h index 70de2b1fa..50e3dc914 100755 --- a/source/server/serverPlugin.h +++ b/source/server/serverPlugin.h @@ -15,7 +15,7 @@ extern "C" { LIBRARY_API void allocPluginList(int count, PLUGINLIST* plugin_list); LIBRARY_API void freePluginList(PLUGINLIST* plugin_list); LIBRARY_API void initPluginData(PLUGIN_DATA* plugin); -LIBRARY_API int udaServerRedirectStdStreams(int reset); +LIBRARY_API int udaServerRedirectStdStreams(bool reset, bool cleanup=false); LIBRARY_API int udaServerPlugin(REQUEST_DATA* request, DATA_SOURCE* data_source, SIGNAL_DESC* signal_desc, const PLUGINLIST* plugin_list, const ENVIRONMENT* environment); LIBRARY_API int udaProvenancePlugin(CLIENT_BLOCK* client_block, REQUEST_DATA* original_request, diff --git a/source/server/udaServer.cpp b/source/server/udaServer.cpp index b87216bb1..9edae1f12 100755 --- a/source/server/udaServer.cpp +++ b/source/server/udaServer.cpp @@ -7,6 +7,7 @@ #endif #include +#include #include #include @@ -20,6 +21,7 @@ #include #include #include +#include #include "closeServerSockets.h" #include "createXDRStream.h" @@ -45,6 +47,7 @@ constexpr int server_version = 10; static int protocol_version = 10; static int legacy_server_version = 6; +const static std::string server_uuid = uda::common::uuid::generate_random_uuid(); static USERDEFINEDTYPELIST* user_defined_type_list = nullptr; // User Defined Structure Types from Data Files & Plugins static LOGMALLOCLIST* log_malloc_list = nullptr; // List of all Heap Allocations for Data: Freed after data is dispatched @@ -74,14 +77,13 @@ static int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_bloc DATA_BLOCK_LIST* data_block_list, int* fatal, int* server_closedown, uda::cache::UdaCache* cache, LOGSTRUCTLIST* log_struct_list, XDR* server_input, const unsigned int* total_datablock_size, - int server_tot_block_time, int* server_timeout); + int* server_timeout); static int doServerLoop(REQUEST_BLOCK* request_block, DATA_BLOCK_LIST* data_block_list, CLIENT_BLOCK* client_block, SERVER_BLOCK* server_block, METADATA_BLOCK* metadata_block, ACTIONS* actions_desc, ACTIONS* actions_sig, int* fatal, uda::cache::UdaCache* cache, LOGSTRUCTLIST* log_struct_list, XDR* server_input, XDR* server_output, unsigned int* total_datablock_size, - int server_tot_block_time, - int* server_timeout); + int* server_tot_block_time, int* server_timeout); static int reportToClient(SERVER_BLOCK* server_block, DATA_BLOCK_LIST* data_block_list, CLIENT_BLOCK* client_block, int trap1Err, @@ -133,6 +135,7 @@ int udaServer(CLIENT_BLOCK client_block) initUdaErrorStack(); initServerBlock(&server_block, server_version); + snprintf(server_block.DOI, STRING_LENGTH, "%s", server_uuid.c_str()); initActions(&actions_desc); // There may be a Sequence of Actions to Apply initActions(&actions_sig); initRequestBlock(&request_block); @@ -159,7 +162,7 @@ int udaServer(CLIENT_BLOCK client_block) int fatal = 0; doServerLoop(&request_block, &data_block_list, &client_block, &server_block, &metadata_block, &actions_desc, &actions_sig, &fatal, cache, &log_struct_list, server_input, server_output, - &total_datablock_size, server_tot_block_time, &server_timeout); + &total_datablock_size, &server_tot_block_time, &server_timeout); } err = doServerClosedown(&client_block, &request_block, &data_block_list, server_tot_block_time, server_timeout); @@ -376,7 +379,7 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV METADATA_BLOCK* metadata_block, ACTIONS* actions_desc, ACTIONS* actions_sig, DATA_BLOCK_LIST* data_block_list, int* fatal, int* server_closedown, uda::cache::UdaCache* cache, LOGSTRUCTLIST* log_struct_list, XDR* server_input, const unsigned int* total_datablock_size, - int server_tot_block_time, int* server_timeout) + int* server_timeout) { UDA_LOG(UDA_LOG_DEBUG, "Start of Server Error Trap #1 Loop\n"); @@ -403,11 +406,6 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV if ((err = protocol2(server_input, protocol_id, XDR_RECEIVE, nullptr, log_malloc_list, user_defined_type_list, client_block, protocol_version, log_struct_list, 0, malloc_source)) != 0) { - if (server_tot_block_time >= 1000 * *server_timeout) { - *fatal = 1; - UDA_THROW_ERROR(999, "Server Time Out"); - } - UDA_LOG(UDA_LOG_DEBUG, "Problem Receiving Client Data Block\n"); addIdamError(UDA_CODE_ERROR_TYPE, __func__, err, "Protocol 10 Error (Receiving Client Block)"); @@ -819,7 +817,7 @@ int handleRequest(REQUEST_BLOCK* request_block, CLIENT_BLOCK* client_block, SERV int doServerLoop(REQUEST_BLOCK* request_block, DATA_BLOCK_LIST* data_block_list, CLIENT_BLOCK* client_block, SERVER_BLOCK* server_block, METADATA_BLOCK* metadata_block, ACTIONS* actions_desc, ACTIONS* actions_sig, int* fatal, uda::cache::UdaCache* cache, LOGSTRUCTLIST* log_struct_list, - XDR* server_input, XDR* server_output, unsigned int* total_datablock_size, int server_tot_block_time, + XDR* server_input, XDR* server_output, unsigned int* total_datablock_size, int* server_tot_block_time, int* server_timeout) { int err = 0; @@ -842,7 +840,10 @@ int doServerLoop(REQUEST_BLOCK* request_block, DATA_BLOCK_LIST* data_block_list, int server_closedown = 0; err = handleRequest(request_block, client_block, server_block, metadata_block, actions_desc, actions_sig, data_block_list, fatal, &server_closedown, cache, log_struct_list, server_input, - total_datablock_size, server_tot_block_time, server_timeout); + total_datablock_size, server_timeout); + + // Reset server block time to zero so that we only kill the server after TIMEOUT minutes of inactivity + *server_tot_block_time = 0; if (server_closedown) { break; @@ -902,6 +903,7 @@ int doServerLoop(REQUEST_BLOCK* request_block, DATA_BLOCK_LIST* data_block_list, UDA_LOG(UDA_LOG_DEBUG, "initServerBlock\n"); initServerBlock(server_block, server_version); + snprintf(server_block->DOI, STRING_LENGTH, "%s", server_uuid.c_str()); //---------------------------------------------------------------------------- // Server Wait Loop @@ -917,6 +919,8 @@ int doServerClosedown(CLIENT_BLOCK* client_block, REQUEST_BLOCK* request_block, //---------------------------------------------------------------------------- // Server Destruct..... + udaServerRedirectStdStreams(false, true); + UDA_LOG(UDA_LOG_DEBUG, "Server Shutting Down\n"); if (server_tot_block_time > 1000 * server_timeout) { UDA_LOG(UDA_LOG_DEBUG, "Server Timeout after %d secs\n", server_timeout); diff --git a/source/server/writer.cpp b/source/server/writer.cpp index 6691413c3..ef04fa520 100755 --- a/source/server/writer.cpp +++ b/source/server/writer.cpp @@ -131,6 +131,9 @@ int server_write(void* iohandle, char* buf, int count) while (BytesSent < count) { while (((rc = (int)write(serverSocket, buf, count)) == -1) && (errno == EINTR)) {} + if (rc < 0) { + break; + } BytesSent += rc; buf += rc; } diff --git a/source/server2/get_data.cpp b/source/server2/get_data.cpp index 3a2ddb186..66a710f3b 100644 --- a/source/server2/get_data.cpp +++ b/source/server2/get_data.cpp @@ -1052,7 +1052,6 @@ int uda::Server::read_data(RequestData* request, DATA_BLOCK* data_block) plugin_interface.userdefinedtypelist = user_defined_type_list_; plugin_interface.logmalloclist = log_malloc_list_; plugin_interface.error_stack.nerrors = 0; - plugin_interface.error_stack.idamerror = nullptr; int plugin_request = REQUEST_READ_UNKNOWN; @@ -1211,4 +1210,4 @@ int uda::Server::read_data(RequestData* request, DATA_BLOCK* data_block) uda::serverRedirectStdStreams(1); return 0; -} \ No newline at end of file +} diff --git a/source/server2/server.cpp b/source/server2/server.cpp index 5dcc5e665..ffa935170 100644 --- a/source/server2/server.cpp +++ b/source/server2/server.cpp @@ -4,19 +4,18 @@ #include #include -#include "clientserver/initStructs.h" -#include "server_environment.hpp" -#include "logging/logging.h" #include "clientserver/errorLog.h" -#include "clientserver/udaErrors.h" +#include "clientserver/initStructs.h" +#include "clientserver/printStructs.h" #include "clientserver/protocol.h" #include "clientserver/xdrlib.h" -#include "clientserver/printStructs.h" #include "logging/accessLog.h" +#include "logging/logging.h" +#include "server_environment.hpp" +#include "server_exceptions.h" #include "server_plugin.h" #include "server_processing.h" #include "structures/struct.h" -#include "server_exceptions.h" #ifdef SSLAUTHENTICATION # include "authentication/udaServerSSL.h" #endif @@ -702,11 +701,13 @@ int uda::Server::handle_request() return err; } -unsigned int count_data_block_list_size(const std::vector& data_blocks, ClientBlock* client_block) +unsigned int count_data_block_list_size(std::vector& data_blocks, ClientBlock* client_block) { unsigned int total = 0; - for (const auto& data_block : data_blocks) { - total += countDataBlockSize(&data_block, client_block); + for (auto& data_block : data_blocks) { + unsigned int count = countDataBlockSize(&data_block, client_block); + data_block.totalDataBlockSize = count; + total += count; } return total; } diff --git a/source/server2/server_plugin.cpp b/source/server2/server_plugin.cpp index 58c9f9e44..52184d0ba 100755 --- a/source/server2/server_plugin.cpp +++ b/source/server2/server_plugin.cpp @@ -361,7 +361,6 @@ int uda::provenancePlugin(ClientBlock* client_block, RequestData* original_reque plugin_interface.userdefinedtypelist = &userdefinedtypelist; plugin_interface.logmalloclist = &logmalloclist; plugin_interface.error_stack.nerrors = 0; - plugin_interface.error_stack.idamerror = nullptr; // Redirect Output to temporary file if no file handles passed @@ -511,7 +510,7 @@ int uda::call_metadata_plugin(const PluginData& plugin, RequestData* request_blo idam_plugin_interface.userdefinedtypelist = &userdefinedtypelist; idam_plugin_interface.logmalloclist = &logmalloclist; idam_plugin_interface.error_stack.nerrors = 0; - idam_plugin_interface.error_stack.idamerror = nullptr; + //RC idam_plugin_interface.error_stack.idamerror = nullptr; // Redirect Output to temporary file if no file handles passed diff --git a/source/structures/struct.cpp b/source/structures/struct.cpp index c60da5d5f..dd6e1f534 100755 --- a/source/structures/struct.cpp +++ b/source/structures/struct.cpp @@ -205,7 +205,7 @@ void expandImage(char* buffer, char defnames[MAXELEMENTS][MAXELEMENTNAME], int* char work[STRING_LENGTH]; char* p1, * p2, * p3; - if (buffer[0] != '\t' || buffer[0] != ' ') { + if (buffer[0] != '\t' && buffer[0] != ' ') { strcpy(expand, "\t"); // Tab out the structure contents } else { expand[0] = '\0'; } len = (int)strlen(expand); diff --git a/source/wrappers/c++/client.cpp b/source/wrappers/c++/client.cpp index 2f00165f7..35a0485bb 100755 --- a/source/wrappers/c++/client.cpp +++ b/source/wrappers/c++/client.cpp @@ -35,6 +35,8 @@ void uda::Client::setProperty(Property prop, bool value) case PROP_VERBOSE: name = "verbose"; break; case PROP_DEBUG: name = "debug"; break; case PROP_ALTDATA: name = "altdata"; break; + case PROP_DB_ONLY: name = "dbonly"; break; + case PROP_IDLE_TIMEOUT: name = "idletimeout"; break; case PROP_TIMEOUT: case PROP_ALTRANK: @@ -70,6 +72,8 @@ void uda::Client::setProperty(Property prop, int value) case PROP_VERBOSE: case PROP_DEBUG: case PROP_ALTDATA: + case PROP_DB_ONLY: + case PROP_IDLE_TIMEOUT: throw InvalidUseException("Cannot set integer value for boolean property"); case PROP_TIMEOUT: @@ -112,6 +116,8 @@ int uda::Client::property(Property prop) case PROP_ALTDATA: return udaGetProperty("altdata"); case PROP_TIMEOUT: return udaGetProperty("timeout"); case PROP_ALTRANK: return udaGetProperty("altrank"); + case PROP_DB_ONLY: return udaGetProperty("dbonly"); + case PROP_IDLE_TIMEOUT: return udaGetProperty("idletimeout"); default: throw UDAException("Unknown property"); diff --git a/source/wrappers/c++/client.hpp b/source/wrappers/c++/client.hpp index a71c1f7db..7ae1d747b 100755 --- a/source/wrappers/c++/client.hpp +++ b/source/wrappers/c++/client.hpp @@ -110,7 +110,9 @@ enum Property PROP_VERBOSE, PROP_DEBUG, PROP_ALTDATA, - PROP_ALTRANK + PROP_ALTRANK, + PROP_DB_ONLY, + PROP_IDLE_TIMEOUT }; enum ErrorCodes diff --git a/source/wrappers/python/CMakeLists.txt b/source/wrappers/python/CMakeLists.txt index 29249195d..72e221315 100755 --- a/source/wrappers/python/CMakeLists.txt +++ b/source/wrappers/python/CMakeLists.txt @@ -5,11 +5,23 @@ install( FILES ${CYTHON_FILES} pyuda/cpyuda/uda.pxd DESTINATION python_installer file( GLOB INSTALL_FILES pyuda/*.py ) install( FILES ${INSTALL_FILES} DESTINATION python_installer/pyuda ) -install( FILES ${CMAKE_CURRENT_LIST_DIR}/uda/__init__.py DESTINATION python_installer/uda ) configure_file( ${CMAKE_CURRENT_LIST_DIR}/pyuda/_version.py.in ${CMAKE_CURRENT_BINARY_DIR}/pyuda/_version.py @ONLY ) install( FILES ${CMAKE_CURRENT_BINARY_DIR}/pyuda/_version.py DESTINATION python_installer/pyuda ) +### extra install steps for the dummy "uda" repo which will mirror "pyuda" on pypi +set( DUMMY_REPO_FILES + uda/README.md + uda/pyproject.toml + ${CMAKE_CURRENT_BINARY_DIR}/uda/setup.py +) + +install( FILES ${CMAKE_CURRENT_LIST_DIR}/uda/uda/__init__.py DESTINATION python_installer/uda/uda ) +configure_file( ${CMAKE_CURRENT_LIST_DIR}/uda/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/uda/setup.py @ONLY ) + +install( FILES ${DUMMY_REPO_FILES} DESTINATION python_installer/uda ) +#### + find_package( OpenSSL REQUIRED ) find_package( LibXml2 REQUIRED ) find_package( LibMemcached QUIET ) diff --git a/source/wrappers/python/README.md b/source/wrappers/python/README.md index ac796c68a..b581c2587 100644 --- a/source/wrappers/python/README.md +++ b/source/wrappers/python/README.md @@ -17,3 +17,6 @@ client = pyuda.Client() signal_object = client.get(signal, source) ``` + +## Old release versions +Releases 2.7.6-2.8.1 were uploaded to the uda project on pypi [here](https://pypi.org/project/uda). The project moved here under the pyuda name for release 2.8.2. diff --git a/source/wrappers/python/pyproject.toml b/source/wrappers/python/pyproject.toml index 04727bc58..8696391c3 100644 --- a/source/wrappers/python/pyproject.toml +++ b/source/wrappers/python/pyproject.toml @@ -3,6 +3,7 @@ requires = [ "setuptools>=42", "oldest-supported-numpy; python_version <'3.9'", "numpy>=2.0; python_version >='3.9'", + "Cython>=0.29, <3.1; python_version=='3.8'", "Cython>=0.29", "six" ] @@ -15,8 +16,9 @@ source = "https://github.com/ukaea/UDA" tracker = "https://github.com/ukaea/UDA/issues" [project] -name = "uda" +name = "pyuda" dynamic = ["version"] +description = "Universal Data Access (UDA)" readme = {file = 'README.md', content-type = "text/markdown"} license = {text = "Apache-2.0 license"} dependencies = ["numpy>1.7", "six", "progress"] diff --git a/source/wrappers/python/pyuda/__init__.py b/source/wrappers/python/pyuda/__init__.py index 3312982b3..b23feb5c2 100644 --- a/source/wrappers/python/pyuda/__init__.py +++ b/source/wrappers/python/pyuda/__init__.py @@ -20,5 +20,5 @@ Properties = cpyuda.Properties -__all__ = (UDAException, ProtocolException, ServerException, InvalidUseException, - Client, Signal, Video, Dim, Properties, DEBUG, WARNING, INFO, ERROR) +__all__ = ("UDAException", "ProtocolException", "ServerException", "InvalidUseException", + "Client", "Signal", "Video", "Dim", "Properties", "DEBUG", "WARNING", "INFO", "ERROR") diff --git a/source/wrappers/python/pyuda/_client.py b/source/wrappers/python/pyuda/_client.py index 718ebc628..7ae3087e4 100644 --- a/source/wrappers/python/pyuda/_client.py +++ b/source/wrappers/python/pyuda/_client.py @@ -19,6 +19,7 @@ from enum import Enum except ImportError: Enum = object +from warnings import warn class ClientMeta(type): @@ -45,6 +46,16 @@ def server(cls, value): cpyuda.set_server_host_name(value) +def _target(server, port, queue): + Client.port = port + Client.server = server + try: + result = cpyuda.get_data("help::help()", "") + queue.put(result.is_string()) + except cpyuda.ServerException as ex: + print("cpyuda.ServerException: %s" % ex) + + class Client(with_metaclass(ClientMeta, object)): """ A class representing the UDA client. @@ -216,8 +227,42 @@ def get_property(cls, prop): def set_property(cls, prop, value=None): cpyuda.set_property(prop, value) - def close_connection(self): + @classmethod + def close_connection(cls): cpyuda.close_connection() - def reset_connection(self): + @classmethod + def reset_connection(cls): cpyuda.close_connection() + + @classmethod + def test_connection(cls, timeout=1): + import multiprocessing + queue = multiprocessing.Queue() + p = multiprocessing.Process(target=_target, args=(cls.server, cls.port, queue)) + p.start() + p.join(timeout) + if p.is_alive(): + p.terminate() + p.join() + raise TimeoutError("Connection test timed out after %1.2f seconds" + % timeout) + if queue.empty(): + return False + return queue.get() + + @classmethod + def query_server_version(cls): + result = cpyuda.get_data("help::version()", "") + if not result.is_string(): + warn("Server versions before 2.8.1 do not report their software version through this interface") + return None + return result.data() + + @classmethod + def query_server_info(cls): + software_version = cls.query_server_version() + protocol_version = cpyuda.get_server_protocol_version() + uuid = cpyuda.get_server_uuid() + return {"software_version": software_version, "protocol_version": protocol_version, + "uuid": uuid, "server": cls.server, "port": cls.port} diff --git a/source/wrappers/python/pyuda/_dim.py b/source/wrappers/python/pyuda/_dim.py index 69ae618f8..e77b1585c 100644 --- a/source/wrappers/python/pyuda/_dim.py +++ b/source/wrappers/python/pyuda/_dim.py @@ -1,5 +1,5 @@ from __future__ import (division, print_function, absolute_import) - +import copy class Dim(object): @@ -20,3 +20,18 @@ def units(self): def __repr__(self): return "".format(self.label) if self.label else "" + + def __deepcopy__(self, memo): + return DataOwningDim(data=copy.deepcopy(self.data, memo), + label=copy.deepcopy(self.label, memo), + units=copy.deepcopy(self.units, memo)) + + +class DataOwningDim: + def __init__(self, data=None, label='', units=''): + self.data = data + self.label = label + self.units = units + + def __repr__(self): + return "".format(self.label) if self.label else "" diff --git a/source/wrappers/python/pyuda/_signal.py b/source/wrappers/python/pyuda/_signal.py index 20ec900bf..ded25c024 100644 --- a/source/wrappers/python/pyuda/_signal.py +++ b/source/wrappers/python/pyuda/_signal.py @@ -3,6 +3,7 @@ import json import base64 import numpy as np +import copy import cpyuda @@ -59,7 +60,28 @@ def default(self, obj): return super().default(obj) -class Signal(Data): +class SignalDataImpl(Data): + def plot(self): + import matplotlib.pyplot as plt + + dim = self.dims[0] + + plt.plot(dim.data, self.data) + plt.xlabel('{0} ({1})'.format(dim.label, dim.units)) + plt.ylabel('{0} ({1})'.format(self.label, self.units)) + plt.show() + + def widget(self): + raise NotImplementedError("widget function not implemented for Signal objects") + + def jsonify(self, indent=None): + return json.dumps(self, cls=SignalEncoder, indent=indent) + + def __repr__(self): + return "".format(self.label) if self.label else "" + + +class Signal(SignalDataImpl): def __init__(self, cresult): self._cresult = cresult @@ -128,6 +150,10 @@ def time_index(self): def meta(self): return self._cresult.meta() + @property + def data_block_size(self): + return self._cresult.data_block_size() + def _import_dims(self): self._dims = [] @@ -202,22 +228,65 @@ def set_time_last(self): if self.time_index == 0: self.reverse_dimension_order() - def plot(self): - import matplotlib.pyplot as plt - - dim = self.dims[0] - - plt.plot(dim.data, self.data) - plt.xlabel('{0} ({1})'.format(dim.label, dim.units)) - plt.ylabel('{0} ({1})'.format(self.label, self.units)) - plt.show() - - def widget(self): - raise NotImplementedError("widget function not implemented for Signal objects") - - def jsonify(self, indent=None): - return json.dumps(self, cls=SignalEncoder, indent=indent) - - def __repr__(self): - return "".format(self.label) if self.label else "" - + def clone(self): + """ + Copy all signal data from cpyuda into a DataOwningSignal object + """ + return copy.deepcopy(self) + + def __deepcopy__(self, memo): + """ + Copy all signal data from cpyuda into a DataOwningSignal object + """ + return DataOwningSignal(data=copy.deepcopy(self.data, memo), + errors=copy.deepcopy(self.errors, memo), + label=copy.deepcopy(self.label, memo), + units=copy.deepcopy(self.units, memo), + description=copy.deepcopy(self.description, memo), + rank=copy.deepcopy(self.rank, memo), + dims=copy.deepcopy(self.dims, memo), + shape=copy.deepcopy(self.shape, memo), + time_index=copy.deepcopy(self.time_index, memo), + meta=copy.deepcopy(self.meta, memo)) + + def __reduce__(self): + """ + Overwriting __reduce__ method for pickling pyuda signal objects. + This does deep copies of all the data views held by a signal object + and constructs a data owning signal object which is picklable and can + be loaded from disk without state initialisation errors in cpyuda. + """ + return (DataOwningSignal, (copy.deepcopy(self.data), + copy.deepcopy(self.errors), + copy.deepcopy(self.label), + copy.deepcopy(self.units), + copy.deepcopy(self.description), + copy.deepcopy(self.rank), + copy.deepcopy(self.dims), + copy.deepcopy(self.shape), + copy.deepcopy(self.time_index), + copy.deepcopy(self.meta))) + + +class DataOwningSignal(SignalDataImpl): + """ + Class to hold a copy of a pyuda Signal object where all data arrays are owned + by the object instead of being views of memory held by the uda c-library + """ + + def __init__(self, data=None, errors=None, label='', units='', description='', + rank=None, dims=None, shape=None, time_index=None, meta=None): + self.data = data + self.errors = errors + self.label = label + self.units = units + self.descritpion = description + self.rank = rank + self.dims = dims + self.shape = shape + self.time_index = time_index + self.meta = meta + if self.time_index is not None: + self.time = self.dims[self.time_index] + else: + self.time = None diff --git a/source/wrappers/python/pyuda/_string.py b/source/wrappers/python/pyuda/_string.py index f3c4cfd13..65761bfed 100644 --- a/source/wrappers/python/pyuda/_string.py +++ b/source/wrappers/python/pyuda/_string.py @@ -4,21 +4,11 @@ from ._utils import cdata_to_numpy_array from ._data import Data +import copy import json -class String(Data): - - def __init__(self, cresult): - self._cresult = cresult - self._data = None - - @property - def str(self): - if self._data is None: - self._data = self._cresult.data() - return self._data - +class StringDataImpl(Data): def __str__(self): return self.str @@ -35,4 +25,40 @@ def jsonify(self, indent=None): 'value': self.str }, } - return json.dumps(obj, indent=indent) \ No newline at end of file + return json.dumps(obj, indent=indent) + + +class String(StringDataImpl): + + def __init__(self, cresult): + self._cresult = cresult + self._data = None + + @property + def str(self): + if self._data is None: + self._data = self._cresult.data() + return self._data + + def clone(self): + return copy.deepcopy(self) + + def __deepcopy__(self, memo): + """ + Copy all signal data from cpyuda into a DataOwningString object + """ + return DataOwningString(str=copy.deepcopy(self.str, memo)) + + def __reduce__(self): + """ + Overwriting __reduce__ method for pickling pyuda string signal objects. + This does deep copies of all the data views held by a signal object + and constructs a data owning object which is picklable and can + be loaded from disk without state initialisation errors in cpyuda. + """ + return (DataOwningString, (copy.deepcopy(self.str),)) + + +class DataOwningString(StringDataImpl): + def __init__(self, str=None): + self.str = str diff --git a/source/wrappers/python/pyuda/_structured.py b/source/wrappers/python/pyuda/_structured.py index 0bd23244e..b9041862f 100644 --- a/source/wrappers/python/pyuda/_structured.py +++ b/source/wrappers/python/pyuda/_structured.py @@ -3,6 +3,7 @@ from ._utils import (cdata_scalar_to_value, cdata_vector_to_value, cdata_array_to_value) from ._data import Data +import copy import json import itertools import numpy as np @@ -32,40 +33,36 @@ def default(self, obj): return super().default(obj) -class StructuredData(Data): +class StructuredDataImpl(Data): + def __repr__(self): + return "".format(self.name) - _translation_table = None + def plot(self): + raise NotImplementedError("plot function not implemented for StructuredData objects") - def __init__(self, cnode): - self._cnode = cnode - self._children = None - self._name = self._cnode.name() or 'ROOT' - self._imported_attrs = [] - if StructuredData._translation_table is None: - StructuredData._setup_translation_table() - self._import_data() + def widget(self): + raise NotImplementedError("widget function not implemented for StructuredData objects") - @classmethod - def _setup_translation_table(cls): - cls._translation_table = ''.join(chr(i) for i in range(256)) - identifier_chars = tuple(itertools.chain(range(ord('0'), ord('9')+1), - range(ord('a'), ord('z')+1), - range(ord('A'), ord('Z')+1))) - identifier_chars += ('_',) - cls._translation_table = u''.join(c if ord(c) in identifier_chars else '_' for c in cls._translation_table) + def _todict(self): + obj = {} + for name in self._imported_attrs: + obj[name] = getattr(self, name) + if len(self.children) > 0: + obj['children'] = [] + for child in self.children: + obj['children'].append(child._todict()) + return obj + + def jsonify(self, indent=None): + obj = self._todict() + return json.dumps(obj, cls=StructuredDataEncoder, indent=indent) - def _import_data(self): - data = self._cnode.data() - for (name, value) in data.items(): - if value is not None: - attr_name = self._check_name(name) - self._imported_attrs.append(attr_name) - setattr(self, attr_name, value) +class TreeDisplayer: def _display(self, depth, level): if depth is not None and level > depth: return - print(('|' * level + '['), self._name, ']') + print(('|' * level + '['), self.name, ']') for name in self._imported_attrs: print(('|' * (level + 1) + '->'), name) for child in self.children: @@ -92,6 +89,37 @@ def __getitem__(self, item): else: return [child["/".join(tokens[1:])] for child in found] + +class StructuredData(StructuredDataImpl, TreeDisplayer): + + _translation_table = None + + def __init__(self, cnode): + self._cnode = cnode + self._children = None + self._name = self._cnode.name() or 'ROOT' + self._imported_attrs = [] + if StructuredData._translation_table is None: + StructuredData._setup_translation_table() + self._import_data() + + @classmethod + def _setup_translation_table(cls): + cls._translation_table = ''.join(chr(i) for i in range(256)) + identifier_chars = tuple(itertools.chain(range(ord('0'), ord('9')+1), + range(ord('a'), ord('z')+1), + range(ord('A'), ord('Z')+1))) + identifier_chars += ('_',) + cls._translation_table = u''.join(c if ord(c) in identifier_chars else '_' for c in cls._translation_table) + + def _import_data(self): + data = self._cnode.data() + for (name, value) in data.items(): + if value is not None: + attr_name = self._check_name(name) + self._imported_attrs.append(attr_name) + setattr(self, attr_name, value) + @property def children(self): if self._children is None: @@ -118,25 +146,41 @@ def _check_name(self, name): name += '_' return name - def __repr__(self): - return "".format(self._name) - - def plot(self): - raise NotImplementedError("plot function not implemented for StructuredData objects") - - def widget(self): - raise NotImplementedError("widget function not implemented for StructuredData objects") - - def _todict(self): - obj = {} - for name in self._imported_attrs: - obj[name] = getattr(self, name) - if len(self.children) > 0: - obj['children'] = [] - for child in self.children: - obj['children'].append(child._todict()) - return obj - - def jsonify(self, indent=None): - obj = self._todict() - return json.dumps(obj, cls=StructuredDataEncoder, indent=indent) + def clone(self): + """ + Copy all signal data from cpyuda into a DataOwningSignal object + """ + return copy.deepcopy(self) + + def __deepcopy__(self, memo): + """ + Copy all signal data from cpyuda into a DataOwningSignal object + """ + imported_attrs = {attr: copy.deepcopy(getattr(self, attr), memo) + for attr in self._imported_attrs} + + return DataOwningStructuredData(children=copy.deepcopy(self.children, memo), + name=copy.deepcopy(self.name, memo), + imported_attrs=imported_attrs) + + def __reduce__(self): + """ + Overwriting __reduce__ method for pickling pyuda signal objects. + This does deep copies of all the data views held by a signal object + and constructs a data owning signal object which is picklable and can + be loaded from disk without state initialisation errors in cpyuda. + """ + imported_attrs = {attr: copy.deepcopy(getattr(self, attr)) + for attr in self._imported_attrs} + return (DataOwningStructuredData, (copy.deepcopy(self.children), + copy.deepcopy(self.name), + imported_attrs)) + + +class DataOwningStructuredData(StructuredDataImpl, TreeDisplayer): + def __init__(self, children=None, name=None, imported_attrs=None): + self.children = children + self.name = name + if imported_attrs is not None: + for key, val in imported_attrs.items(): + setattr(self, key, val) diff --git a/source/wrappers/python/pyuda/_video.py b/source/wrappers/python/pyuda/_video.py index 3a03cb2a1..d89ee0cb9 100644 --- a/source/wrappers/python/pyuda/_video.py +++ b/source/wrappers/python/pyuda/_video.py @@ -2,6 +2,7 @@ from ._data import Data +import copy import json import numpy as np import base64 @@ -85,3 +86,10 @@ def widget(self): def jsonify(self, indent=None): raise NotImplementedError("jsonify has not been implement for Video objects") + + def clone(self): + """ + Return a deepcopy of a video object + """ + return copy.deepcopy(self) + diff --git a/source/wrappers/python/pyuda/cpyuda/client.pyx b/source/wrappers/python/pyuda/cpyuda/client.pyx index 122a706cf..b0b1c2183 100644 --- a/source/wrappers/python/pyuda/cpyuda/client.pyx +++ b/source/wrappers/python/pyuda/cpyuda/client.pyx @@ -30,6 +30,8 @@ _properties = { "reuselasthandle": ("REUSE_LAST_HANDLE", False), "freeandreuselasthandle": ("FREE_AND_REUSE_LAST_HANDLE", False), "filecache": ("FILE_CACHE", False), + "dbonly": ("DB_ONLY", False), + "idletimeout": ("IDLE_TIMEOUT", False), } if PY_MAJOR_VERSION >= 3.0: @@ -87,6 +89,14 @@ def close_connection(): uda.closeAllConnections() +def get_server_protocol_version(): + return uda.getIdamServerVersion() + + +def get_server_uuid(): + return uda.getIdamServerDOI().decode() + + def get_data(signal, source): handle = uda.idamGetAPI(signal.encode(), source.encode()) cdef const char* err_msg diff --git a/source/wrappers/python/pyuda/cpyuda/result.pyx b/source/wrappers/python/pyuda/cpyuda/result.pyx index ca874e54f..a4d4c926a 100644 --- a/source/wrappers/python/pyuda/cpyuda/result.pyx +++ b/source/wrappers/python/pyuda/cpyuda/result.pyx @@ -151,3 +151,7 @@ cdef class Result: def time_dim(self, data_type): cdef int order = uda.getIdamOrder(int(self._handle)) return Dim(self._handle, order, data_type) + + def data_block_size(self): + return uda.getIdamTotalDataBlockSize(int(self._handle)) + diff --git a/source/wrappers/python/pyuda/cpyuda/uda.pxd b/source/wrappers/python/pyuda/cpyuda/uda.pxd index 5a1b447ce..c4757fa94 100644 --- a/source/wrappers/python/pyuda/cpyuda/uda.pxd +++ b/source/wrappers/python/pyuda/cpyuda/uda.pxd @@ -7,6 +7,8 @@ cdef extern from "client/udaClient.h": const char* getUdaBuildDate(); void udaFree(int handle); CLIENT_FLAGS* udaClientFlags(); + int getIdamServerVersion(); + const char* getIdamServerDOI(); ctypedef struct CLIENT_FLAGS @@ -135,6 +137,7 @@ cdef extern from "client/accAPI.h": int getIdamOrder(int handle); NTREE* getIdamDataTree(int handle); LOGMALLOCLIST* getIdamLogMallocList(int handle); + unsigned int getIdamTotalDataBlockSize(int handle); cdef extern from "clientserver/initStructs.h": void initIdamPutDataBlock(PUTDATA_BLOCK* str); diff --git a/source/wrappers/python/setup.py.in b/source/wrappers/python/setup.py.in index e865cbc25..9505abdda 100644 --- a/source/wrappers/python/setup.py.in +++ b/source/wrappers/python/setup.py.in @@ -13,7 +13,7 @@ default_options['compile_time_env'] = {'CAPNP': @CAPNP_FLAG@} extra_link_args = [] extra_compile_args = ['-std=c++17'] -root = os.environ.get('UDA_DIR', '@CMAKE_INSTALL_PREFIX@') +root = '@CMAKE_INSTALL_PREFIX@' print('Detected platform: ' + sys.platform) print('Detected system: ' + sysconfig.get_platform()) @@ -80,20 +80,26 @@ ext = Extension( ) -def get_version(v): +def get_version(v, full_version=''): if v.count('.') > 2: v = ".Post".join(v.rsplit('.', 1)) - print("setup version = %s" % v) + # non-tagged commits should only go to testpypi + # where the hash postfix is allowed in the version name + # (local commits PEP 440) + # this can avoid clashing "postX" versions on different branches + # note hash is int-encoded (base 36) to permit upload + if '-' in full_version: + git_hash = full_version.split('-')[1] + v += ".dev%d" % int(git_hash, 36) + # print("setup version = %s" % v) return v setup( - name='uda', - version=get_version('@PROJECT_VERSION@'), - description='Unified Data Access (UDA)', - long_description='Unified Data Access (UDA)', + name='pyuda', + version=get_version('@PROJECT_VERSION@', '@FULL_VERSION@'), author='Jonathan Hollocombe', author_email='jonathan.hollocombe@ukaea.uk', - packages=['pyuda', 'uda'], + packages=['pyuda'], ext_modules=cythonize([ext]), ) diff --git a/source/wrappers/python/uda/README.md b/source/wrappers/python/uda/README.md new file mode 100644 index 000000000..51078d671 --- /dev/null +++ b/source/wrappers/python/uda/README.md @@ -0,0 +1,34 @@ +# This package is deprecated + +Please use ['pyuda'](https://pypi.org/project/pyuda/) instead. This package is now for compatibility only and may be removed in the future. Installing this 'uda' package will now only install a shim which references the underlying 'pyuda' package. + +Install the new package directly using: +``` +pip install pyuda +``` + +Previous release versions (2.7.6-2.8.1) are still hosted here. The first release under the new pyuda name is 2.8.2. + +## Pyuda package description + +See ['pyuda'](https://pypi.org/project/pyuda/) for most up to date descriptions. Below is the package information as of release 2.8.1. + +pyuda is the python interface to the uda client library. It is used for remote access to scientific and experimental data from a number of international labs hosting uda data servers. + +- **Website:** https://ukaea.github.io/UDA/ +- **Documentation:** https://ukaea.github.io/UDA/ +- **Source Code:** https://github.com/ukaea/UDA +- **Issue Tracker:** https://github.com/ukaea/UDA/issues + + +## Quick-start + +```py +import pyuda + +pyuda.Client.server = "" +pyuda.Client.port = +client = pyuda.Client() +signal_object = client.get(signal, source) + +``` diff --git a/source/wrappers/python/uda/__init__.py b/source/wrappers/python/uda/__init__.py deleted file mode 100644 index d16731cc6..000000000 --- a/source/wrappers/python/uda/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import (division, print_function, absolute_import) - -from logging import DEBUG, WARNING, INFO, ERROR - -from pyuda import cpyuda - -from pyuda._client import Client -from pyuda._signal import Signal -from pyuda._video import Video -from pyuda._dim import Dim -from pyuda._structured import StructuredData -from pyuda._json import SignalEncoder, SignalDecoder -from pyuda._version import __version__, __version_info__ - - -UDAException = cpyuda.UDAException -ProtocolException = cpyuda.ProtocolException -ServerException = cpyuda.ServerException -InvalidUseException = cpyuda.InvalidUseException -Properties = cpyuda.Properties - - -__all__ = (UDAException, ProtocolException, ServerException, InvalidUseException, - Client, Signal, Video, Dim, Properties, DEBUG, WARNING, INFO, ERROR) diff --git a/source/wrappers/python/uda/pyproject.toml b/source/wrappers/python/uda/pyproject.toml new file mode 100644 index 000000000..8de0c27ee --- /dev/null +++ b/source/wrappers/python/uda/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = [ + "setuptools>=42" +] +build-backend = "setuptools.build_meta" + +[project.urls] +homepage = "https://ukaea.github.io/UDA/" +documentation = "https://ukaea.github.io/UDA/" +source = "https://github.com/ukaea/UDA" +tracker = "https://github.com/ukaea/UDA/issues" + +[project] +name = "uda" +dynamic = ["version"] +description = "Universal Data Access (UDA)" +readme = {file = 'README.md', content-type = "text/markdown"} +license = {text = "Apache-2.0 license"} +dependencies = ["pyuda"] +requires-python = ">= 3.5" + +classifiers = [ + 'Intended Audience :: Science/Research', + 'Programming Language :: C', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3 :: Only', + 'Operating System :: POSIX :: Linux', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: Microsoft :: Windows', +] + +keywords = ["deprecated"] + +[project.optional-dependencies] +plot = ["pyplot"] +full = ["pyplot", "rich"] diff --git a/source/wrappers/python/uda/setup.py.in b/source/wrappers/python/uda/setup.py.in new file mode 100644 index 000000000..a84c08e11 --- /dev/null +++ b/source/wrappers/python/uda/setup.py.in @@ -0,0 +1,26 @@ +#!/usr/bin/env python +from setuptools import setup + + +def get_version(v, full_version=''): + if v.count('.') > 2: + v = ".Post".join(v.rsplit('.', 1)) + # non-tagged commits should only go to testpypi + # where the hash postfix is allowed in the version name + if '-' in full_version: + git_hash = full_version.split('-')[1] + # hash is int encoded (base 36) to allow upload + v += ".dev%d" % int(git_hash, 36) + print("setup version = %s" % v) + return v + + +version = get_version('@PROJECT_VERSION@', '@FULL_VERSION@') + + +setup( + name='uda', + version=version, + install_requires=["pyuda==%s" % version], + packages=['uda'], +) diff --git a/source/wrappers/python/uda/uda/__init__.py b/source/wrappers/python/uda/uda/__init__.py new file mode 100644 index 000000000..52a360edf --- /dev/null +++ b/source/wrappers/python/uda/uda/__init__.py @@ -0,0 +1,15 @@ +print("NOTE: this package has been renamed as \"pyuda\". The uda package name is now just a wrapper around \"pyuda\"; uda will be deprecated.") +import warnings +warnings.warn("The \"uda\" package name is deprecated. Please use \"pyuda\" when pip installing the package or importing into python", DeprecationWarning) + +import pyuda as _pyuda + +# Copy public attributes into the oldname namespace +globals().update({ + name: getattr(_pyuda, name) + for name in getattr(_pyuda, '__all__', dir(_pyuda)) + if not name.startswith('_') +}) + +# Optionally set __all__ to the same as newname +__all__ = getattr(_pyuda, '__all__', []) diff --git a/source/wrappers/python/win.setup.py.in b/source/wrappers/python/win.setup.py.in index cb56dd952..e5b08b36c 100644 --- a/source/wrappers/python/win.setup.py.in +++ b/source/wrappers/python/win.setup.py.in @@ -75,18 +75,24 @@ ext = Extension( ) -def get_version(v): +def get_version(v, full_version=''): if v.count('.') > 2: v = ".Post".join(v.rsplit('.', 1)) - print("setup version = %s" % v) + # non-tagged commits should only go to testpypi + # where the hash postfix is allowed in the version name + # (local commits PEP 440) + # this can avoid clashing "postX" versions on different branches + # note hash is int-encoded (base 36) to permit upload + if '-' in full_version: + git_hash = full_version.split('-')[1] + v += ".dev%d" % int(git_hash, 36) + # print("setup version = %s" % v) return v setup( name='ukaea_pyuda_test', - version=get_version('@PROJECT_VERSION@'), - description='Unified Data Access (UDA)', - long_description='Unified Data Access (UDA)', + version=get_version('@PROJECT_VERSION@', '@FULL_VERSION@'), author='Jonathan Hollocombe', author_email='jonathan.hollocombe@ukaea.uk', packages=['pyuda', 'uda'], diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1a89a16a9..0cbade343 100755 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -158,7 +158,8 @@ macro( BUILD_TEST NAME SOURCE ) endmacro( BUILD_TEST ) add_subdirectory( plugins ) -add_subdirectory( imas ) +# add_subdirectory( imas ) +add_subdirectory( unit_tests ) add_definitions( -D__USE_XOPEN2K8 ) diff --git a/test/plugins/test_testplugin.cpp b/test/plugins/test_testplugin.cpp index f09ec7a91..1da965e11 100755 --- a/test/plugins/test_testplugin.cpp +++ b/test/plugins/test_testplugin.cpp @@ -1597,8 +1597,13 @@ TEST_CASE( "Run errortest - test error reporting", "[plugins][TESTPLUGIN]" ) uda::Client client; +#ifdef __APPLE__ + REQUIRE_THROWS_WITH( client.get("TESTPLUGIN::errortest(test=1)", ""), "[testplugin]: Test #1 of Error State Management\n" ); + REQUIRE_THROWS_WITH( client.get("TESTPLUGIN::errortest(test=2)", ""), "[testplugin]: Test #2 of Error State Management\n" ); +#else REQUIRE_THROWS_WITH( client.get("TESTPLUGIN::errortest(test=1)", ""), "[testplugin]: Test #1 of Error State Management\n[testplugin]: Test #1 of Error State Management\n" ); REQUIRE_THROWS_WITH( client.get("TESTPLUGIN::errortest(test=2)", ""), "[testplugin]: Test #2 of Error State Management\n[testplugin]: Test #2 of Error State Management\n" ); +#endif #ifndef FATCLIENT // This test hard crashes the server code so can't be run in fat-client mode @@ -2032,4 +2037,57 @@ TEST_CASE( "Test capnp serialisation", "[plugins][TESTPLUGIN]" ) REQUIRE( array[10] == Approx(1.0) ); REQUIRE( array[29] == Approx(2.9) ); } -#endif // CAPNP_ENABLED \ No newline at end of file + +TEST_CASE( "Test capnp complex number serialisation", "[plugins][TESTPLUGIN]" ) +{ +#include "setup.inc" + + int handle = idamGetAPI("TESTPLUGIN::capnp_complex()", ""); + REQUIRE( handle >= 0 ); + + int ec = getIdamErrorCode(handle); + REQUIRE( ec == 0 ); + + const char* error = getIdamErrorMsg(handle); + std::string error_string = error == nullptr ? "" : error; + REQUIRE( error_string.empty() ); + + auto data_type = getIdamDataType(handle); + REQUIRE( data_type == UDA_TYPE_CAPNP ); + + auto data = getIdamData(handle); + REQUIRE( data != nullptr ); + + auto data_n = getIdamDataNum(handle); + REQUIRE( data_n >= 0 ); + + auto tree = uda_capnp_deserialise(data, data_n); + + auto root = uda_capnp_read_root(tree); + auto node = uda_capnp_read_child(tree, root, "complex_array"); + REQUIRE( node != nullptr ); + + auto maybe_rank = uda_capnp_read_rank(node); + REQUIRE( maybe_rank.has_value ); + REQUIRE( maybe_rank.value == 1 ); + + auto rank = maybe_rank.value; + + size_t shape[1]; + bool ok = uda_capnp_read_shape(node, shape); + REQUIRE( ok ); + REQUIRE( shape[0] == 30 ); + + COMPLEX array[30]; + ok = uda_capnp_read_data(node, reinterpret_cast(&array)); + REQUIRE( ok ); + + REQUIRE( array[0].real == Approx(0.0) ); + REQUIRE( array[10].real == Approx(1.0) ); + REQUIRE( array[29].real == Approx(2.9) ); + + REQUIRE( array[0].imaginary == Approx(-10.0) ); + REQUIRE( array[9].real == Approx(-1.0) ); + REQUIRE( array[3].real == Approx(6.0) ); +} +#endif // CAPNP_ENABLED diff --git a/test/unit_tests/CMakeLists.txt b/test/unit_tests/CMakeLists.txt new file mode 100644 index 000000000..c141b94d6 --- /dev/null +++ b/test/unit_tests/CMakeLists.txt @@ -0,0 +1,10 @@ +Include( FetchContent ) + +FetchContent_Declare( + Catch2 + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.8.0 +) + +FetchContent_MakeAvailable( Catch2 ) +add_subdirectory( common ) diff --git a/test/unit_tests/common/CMakeLists.txt b/test/unit_tests/common/CMakeLists.txt new file mode 100644 index 000000000..f7bfd1687 --- /dev/null +++ b/test/unit_tests/common/CMakeLists.txt @@ -0,0 +1,4 @@ +add_executable( test_uda_env uda_env_options.test.cpp ) +target_include_directories( test_uda_env PRIVATE ${CMAKE_SOURCE_DIR}/source ) +target_link_libraries( test_uda_env PRIVATE Catch2::Catch2WithMain ) +add_test( NAME TestUdaEnvOptions COMMAND test_uda_env ) diff --git a/test/unit_tests/common/uda_env_options.test.cpp b/test/unit_tests/common/uda_env_options.test.cpp new file mode 100644 index 000000000..13560a2b8 --- /dev/null +++ b/test/unit_tests/common/uda_env_options.test.cpp @@ -0,0 +1,134 @@ +// catch includes +#include +#include + +// std includes +#include +#include +#include + +// uda includes +#include + +namespace { +class EnvFixture +{ + public: + EnvFixture(std::string_view name, std::string_view val) + :name(name), value(val) + { + setenv(this->name.c_str(), value.c_str(), 1); + } + + ~EnvFixture() + { + unsetenv(name.c_str()); + } + + std::string name; + std::string value; +}; +} // anon namespace + +namespace uda_env = uda::common::env_config; + +SCENARIO( "uda options stored in env vars can be interpreted", "[uda-env]" ) +{ + GIVEN( "an existing environment variable" ) + { + AND_GIVEN( "The variable is truthy" ) + { + auto value = GENERATE("1", "true", "yes", "on", "ON", "TRUE", "YeS"); + std::string name = std::string("UDA_ENV_TEST_VAR_EXISTS_") + value; + EnvFixture env(name, value); + + THEN( "check evaluates to true" ) + { + REQUIRE( uda_env::evaluate_bool_param(name.c_str()) ); + } + } + AND_GIVEN( "The variable is falsey" ) + { + auto value = GENERATE("0", "false", "no", "off", "OFF","FALSE", "FaLSe"); + std::string name = std::string("UDA_ENV_TEST_VAR_EXISTS_") + value; + EnvFixture env(name, value); + + THEN( "string match fails for list of truthy values" ) + { + REQUIRE( ! uda_env::match_custom_values(name.c_str(), uda_env::truthy_values) ); + } + THEN( "string match passes for list of falsey values" ) + { + REQUIRE( uda_env::match_custom_values(name.c_str(), uda_env::falsey_values) ); + } + AND_THEN( "check evaluates to false" ) + { + REQUIRE( ! uda_env::evaluate_bool_param(name.c_str()) ); + } + } + AND_GIVEN( "The variable has an unexpected value" ) + { + auto value = GENERATE("fake", "not_in_list", "TYPO"); + std::string name = std::string("UDA_ENV_TEST_VAR_EXISTS_") + value; + EnvFixture env(name, value); + + THEN( "string match fails for lists of both truthy and falsey values" ) + { + REQUIRE( ! uda_env::strings_match(name, uda_env::falsey_values) ); + REQUIRE( ! uda_env::strings_match(name, uda_env::truthy_values) ); + AND_THEN( "check evaluates false" ) + { + REQUIRE( ! uda_env::evaluate_bool_param(name.c_str()) ); + } + } + } + AND_GIVEN( "A custom list of accepted values" ) + { + std::vector accepted_values {"ENUM1", "ENUM2", "ENUM3"}; + + WHEN( "the value is in the accepted list") + { + auto value = GENERATE("ENUM1", "ENUM2", "ENUM3"); + std::string name = std::string("UDA_ENV_TEST_VAR2_EXISTS_") + value; + EnvFixture env(name, value); + THEN( "string match passes for list of accepted values" ) + { + REQUIRE( uda_env::match_custom_values(name.c_str(), accepted_values) ); + } + } + WHEN( "the value is not in the accepted list") + { + auto value = GENERATE("fake", "not_in_list", "TYPO"); + std::string name = std::string("UDA_ENV_TEST_VAR2_EXISTS_") + value; + EnvFixture env(name, value); + THEN( "string match fails for list of accepted values" ) + { + REQUIRE( ! uda_env::match_custom_values(name.c_str(), accepted_values) ); + } + } + } + } + + GIVEN( "a non-existent environment variable" ) + { + const char* name = "UDA_ENV_TEST_VAR_UNSET"; + unsetenv(name); + WHEN( "A default value of true is used" ) + { + bool default_value = true; + THEN( "check evaluates true" ) + { + REQUIRE( uda_env::evaluate_bool_param(name, default_value) ); + } + } + WHEN( "A default value of false is used" ) + { + bool default_value = false; + THEN( "check evaluates false" ) + { + REQUIRE( ! uda_env::evaluate_bool_param(name, default_value) ); + } + } + + } +}